[bobx] add gallery and model extractors

docs/supportedsites.rst

@@ -13,6 +13,7 @@ Archive of Sins      https://archiveofsins.com/          Threads
 Archived.Moe         https://archived.moe/               Threads
 ArtStation           https://www.artstation.com/         |Images from Use-0|
 Behance              https://www.behance.net/            Images from Users, Galleries
+BobX                 http://www.bobx.com/dark/           Galleries, Models
 Danbooru             https://danbooru.donmai.us/         Pools, Popular Images, Posts, Tag-Searches
 Desuarchive          https://desuarchive.org/            Threads
 DeviantArt           https://www.deviantart.com/         |Collections, De-1|                                Optional (OAuth)

gallery_dl/extractor/__init__.py

@@ -20,6 +20,7 @@ modules = [
     "artstation",
     "b4k",
     "behance",
+    "bobx",
     "danbooru",
     "desuarchive",
     "deviantart",

gallery_dl/extractor/bobx.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extract images from http://www.bobx.com/dark/"""
+
+from .common import Extractor, Message
+from .. import text
+
+
+class BobxExtractor(Extractor):
+    """Base class for bobx extractors"""
+    category = "bobx"
+    root = "http://www.bobx.com"
+    per_page = 80
+
+
+class BobxGalleryExtractor(BobxExtractor):
+    """Extractor for individual image galleries on bobx.com"""
+    subcategory = "gallery"
+    directory_fmt = ["{category}", "{model}", "{title}"]
+    filename_fmt = "{model}_{image_id}_{num:>03}.{extension}"
+    archive_fmt = "{image_id}"
+    pattern = [r"(?:https?://)?(?:www\.)?bobx\.com"
+               r"/([^/]+/[^/]+/photoset/[\w-]+)-\d+-\d+-\d+\.html"]
+    test = [
+        (("http://www.bobx.com/idol/mikoto-hibi"
+          "/photoset/wpb-2018-_11-0-2-8.html"), {
+            "url": "93972d6a661f6627e963d62c9d15531e6b36a389",
+            "keyword": "03505f6d3cdab7b5579431bfe8622eeffd36f533",
+            "content": "3f176b7fe752524cec21a763aa55567e41181e07",
+        }),
+        (("http://www.bobx.com/idol/nashiko-momotsuki"
+          "/photoset/wpb-net-_221---2018-08---magic-of-summer-0-10-10.html"), {
+            "url": "f5d6c0cd0881ae6f504c21a90d86e3464dc54e8e",
+            "keyword": "43395ac200deaaa50627da666bd02c8f1f86a59d",
+        }),
+    ]
+
+    def __init__(self, match):
+        BobxExtractor.__init__(self)
+        self.path = match.group(1)
+
+    def items(self):
+        num = 0
+        while True:
+            url = "{}/{}-{}-10-8.html".format(self.root, self.path, num)
+            page = self.request(url, encoding="utf-8").text
+
+            if num == 0:
+                data = self.metadata(page)
+                yield Message.Version, 1
+                yield Message.Directory, data
+                data["num"] = 0
+
+            for url in self.images(page):
+                url = text.urljoin(self.root, url.replace("-preview-", "-"))
+                data = text.nameext_from_url(url, data)
+                data["image_id"] = text.parse_int(
+                    data["name"].rpartition("-")[2])
+                data["num"] += 1
+                yield Message.Url, url, data
+
+            num += self.per_page
+            if num >= data["count"]:
+                return
+
+    @staticmethod
+    def metadata(page):
+        """Collect metadata for extractor-job"""
+        info = text.extract(page, "<title>", "</title>")[0]
+        model, _, info = info.partition(" in ")
+        info, _, count = info.rpartition(" of ")
+        title = info.rpartition(" - @")[0]
+        return {
+            "title": text.unquote(title),
+            "model": text.unquote(model),
+            "count": text.parse_int(count),
+        }
+
+    @staticmethod
+    def images(page):
+        """Extract all image-urls"""
+        page = text.extract(page, "<table CELLPADDING=", "<script ")[0]
+        return text.extract_iter(page, '<img src="/thumbnail', '"')
+
+
+class BobxModelExtractor(BobxExtractor):
+    """Extractor for a model's image galleries on bobx.com"""
+    subcategory = "model"
+    pattern = [r"(?:https?://)?(?:www\.)?bobx\.com/([^/]+/[^/?&#]+)/?$"]
+    test = [("http://www.bobx.com/idol/nashiko-momotsuki/", {
+        "url": "4294c30465ba17c2ed755aa01762c2cb01edc70f",
+    })]
+
+    def __init__(self, match):
+        BobxExtractor.__init__(self)
+        self.url = "{}/{}/".format(self.root, match.group(1))
+
+    def items(self):
+        page = self.request(self.url).text
+        skip = True
+
+        yield Message.Version, 1
+        for part in text.extract_iter(page, '="photoset/', '"'):
+            # skip every other entry
+            skip = not skip
+            if skip:
+                continue
+            yield Message.Queue, "{}photoset/{}".format(self.url, part), {}

scripts/build_supportedsites.py

@@ -14,6 +14,7 @@ CATEGORY_MAP = {
     "archiveofsins"  : "Archive of Sins",
     "artstation"     : "ArtStation",
     "b4k"            : "arch.b4k.co",
+    "bobx"           : "BobX",
     "deviantart"     : "DeviantArt",
     "dokireader"     : "Doki Reader",
     "dynastyscans"   : "Dynasty Reader",