[gelbooru] inherit from BooruExtractor class

Breaks pool functionality when using API calls (for now),
but reduces code clutter and enables the `tags` option.

gallery_dl/extractor/booru.py

@@ -129,8 +129,10 @@ class MoebooruPageMixin():
 
 class GelbooruPageMixin():
     """Pagination for Gelbooru-like sites"""
+    page_start = 0
+
     def reset_page(self):
-        self.params["pid"] = self.page_start - 1
+        self.params["pid"] = self.page_start
 
     def update_page(self, data):
         self.params["pid"] += 1

gallery_dl/extractor/gelbooru.py

@@ -8,55 +8,46 @@
 
 """Extract images from https://gelbooru.com/"""
 
-from .common import SharedConfigExtractor, Message
+from . import booru
-from .. import text, util, exception
+from .common import Message
-import xml.etree.ElementTree as ET
+from .. import text, exception
 
 
-class GelbooruExtractor(SharedConfigExtractor):
+class GelbooruExtractor(booru.XmlParserMixin,
+                        booru.GelbooruPageMixin,
+                        booru.BooruExtractor):
     """Base class for gelbooru extractors"""
-    basecategory = "booru"
     category = "gelbooru"
-    filename_fmt = "{category}_{id}_{md5}.{extension}"
+    api_url = "https://gelbooru.com/index.php"
-    api_url = "https://gelbooru.com/index.php?page=dapi&s=post&q=index"
+    post_url = "https://gelbooru.com/index.php?page=post&s=view&id={}"
+
+    def __init__(self, match):
+        super().__init__(match)
 
-    def __init__(self):
-        SharedConfigExtractor.__init__(self)
-        self.start_post = 0
         self.use_api = self.config("api", True)
         if self.use_api:
-            self.get_post_data = self.get_post_data_api
+            self.params.update({"page": "dapi", "s": "post", "q": "index"})
+        else:
+            self.items = self.items_noapi
 
-    def items(self):
+    def items_noapi(self):
         data = self.get_metadata()
 
         yield Message.Version, 1
         yield Message.Directory, data
 
-        for post in util.advance(self.get_posts(), self.start_post):
+        for post in self.get_posts():
-            if isinstance(post, str):
+            post = self.get_post_data(post)
-                post = self.get_post_data(post)
-            for key in ("id", "width", "height", "score", "change"):
-                post[key] = text.parse_int(post[key])
             url = post["file_url"]
             post.update(data)
             yield Message.Url, url, text.nameext_from_url(url, post)
 
-    def skip(self, num):
-        self.start_post += num
-        return num
-
-    def get_metadata(self):
-        """Return general metadata"""
-        return {}
-
     def get_posts(self):
         """Return an iterable containing all relevant post objects"""
 
     def get_post_data(self, post_id):
         """Extract metadata of a single post"""
-        page = self.request("https://gelbooru.com/index.php?page=post&s=view"
+        page = self.request(self.post_url.format(post_id)).text
-                            "&id=" + post_id).text
         data = text.extract_all(page, (
             (None        , '<meta name="keywords"', ''),
             ("tags"      , ' imageboard, ', '"'),
@@ -78,19 +69,11 @@ class GelbooruExtractor(SharedConfigExtractor):
             [tag.replace(" ", "_") for tag in data["tags"].split(", ")])
         return data
 
-    def get_post_data_api(self, post_id):
-        """Request metadata of a single post from Gelbooru's API"""
-        return ET.fromstring(
-            self.request(self.api_url + "&id=" + post_id).text)[0].attrib
 
-
+class GelbooruTagExtractor(booru.TagMixin, GelbooruExtractor):
-class GelbooruTagExtractor(GelbooruExtractor):
     """Extractor for images from gelbooru.com based on search-tags"""
-    subcategory = "tag"
-    directory_fmt = ["{category}", "{search_tags}"]
-    archive_fmt = "t_{search_tags}_{id}"
     pattern = [r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?"
-               r"\?page=post&s=list&tags=([^&]+)"]
+               r"\?page=post&s=list&tags=(?P<tags>[^&#]+)"]
     test = [
         ("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", {
             "count": 5,
@@ -102,42 +85,13 @@ class GelbooruTagExtractor(GelbooruExtractor):
     ]
 
     def __init__(self, match):
-        GelbooruExtractor.__init__(self)
+        super().__init__(match)
-        self.tags = text.unquote(match.group(1).replace("+", " "))
+        if not self.use_api:
-        self.per_page = 100 if self.use_api else 42
+            self.per_page = 42
-        self.start_page = 0
-
-    def skip(self, num):
-        pages, posts = divmod(num, self.per_page)
-        self.start_page += pages
-        self.start_post += posts
-        return num
-
-    def get_metadata(self):
-        return {"search_tags": self.tags}
 
     def get_posts(self):
-        if self.use_api:
-            return self._get_posts_api()
-        return self._get_posts_manual()
-
-    def _get_posts_api(self):
-        params = {
-            # 'pid' is page-id; first page has index 0
-            "tags": self.tags, "limit": self.per_page, "pid": self.start_page}
-        while True:
-            root = ET.fromstring(
-                self.request(self.api_url, params=params).text)
-            for item in root:
-                yield item.attrib
-            if len(root) < self.per_page:
-                return
-            params["pid"] += 1
-
-    def _get_posts_manual(self):
         url = "https://gelbooru.com/index.php?page=post&s=list"
-        # 'pid' is post-id; values for 'pid' must be multiples of 42
+        params = {"tags": self.tags, "pid": self.page_start * self.per_page}
-        params = {"tags": self.tags, "pid": self.start_page * self.per_page}
 
         while True:
             page = self.request(url, params=params).text
@@ -150,20 +104,12 @@ class GelbooruTagExtractor(GelbooruExtractor):
 
 class GelbooruPoolExtractor(GelbooruExtractor):
     """Extractor for image-pools from gelbooru.com"""
-    subcategory = "pool"
-    directory_fmt = ["{category}", "pool", "{pool}"]
-    archive_fmt = "p_{pool}_{id}"
     pattern = [r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?"
                r"\?page=pool&s=show&id=(\d+)"]
     test = [("https://gelbooru.com/index.php?page=pool&s=show&id=761", {
         "count": 6,
     })]
 
-    def __init__(self, match):
-        GelbooruExtractor.__init__(self)
-        self.pool_id = match.group(1)
-        self.posts = None
-
     def get_metadata(self):
         page = self.request("https://gelbooru.com/index.php?page=pool&s=show"
                             "&id=" + self.pool_id).text
@@ -183,20 +129,14 @@ class GelbooruPoolExtractor(GelbooruExtractor):
         return self.posts
 
 
-class GelbooruPostExtractor(GelbooruExtractor):
+class GelbooruPostExtractor(booru.PostMixin, GelbooruExtractor):
     """Extractor for single images from gelbooru.com"""
-    subcategory = "post"
-    archive_fmt = "{id}"
     pattern = [r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?"
-               r"\?page=post&s=view&id=(\d+)"]
+               r"\?page=post&s=view&id=(?P<post>\d+)"]
     test = [("https://gelbooru.com/index.php?page=post&s=view&id=313638", {
         "content": "5e255713cbf0a8e0801dc423563c34d896bb9229",
         "count": 1,
     })]
 
-    def __init__(self, match):
-        GelbooruExtractor.__init__(self)
-        self.post_id = match.group(1)
-
     def get_posts(self):
-        return (self.post_id,)
+        return (self.post,)