[hentaihand] update to new site layout

docs/supportedsites.rst

@@ -44,7 +44,7 @@ Hentai Cafe          https://hentai.cafe/                Chapters, Manga
 Hentai Foundry       https://www.hentai-foundry.com/     |hentaifoundry-C|
 Hentai2Read          https://hentai2read.com/            Chapters, Manga
 HentaiFox            https://hentaifox.com/              Galleries, Search Results
-HentaiHand           https://hentaihand.com/             Galleries, Search Results, Tag Searches
+HentaiHand           https://hentaihand.com/             Galleries, Tag Searches
 HentaiHere           https://hentaihere.com/             Chapters, Manga
 Hentainexus          https://hentainexus.com/            Galleries, Search Results
 Hiperdex             https://hiperdex.com/               Artists, Chapters, Manga

gallery_dl/extractor/hentaihand.py

@@ -10,74 +10,61 @@
 
 from .common import GalleryExtractor, Extractor, Message
 from .. import text, util
-import collections
+import json
 
 
 class HentaihandGalleryExtractor(GalleryExtractor):
     """Extractor for image galleries on hentaihand.com"""
     category = "hentaihand"
     root = "https://hentaihand.com"
-    pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com"
+    pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)"
-               r"/(?:comi|view)c/(\d+)")
     test = (
-        ("https://hentaihand.com/comic/272772/kouda-tomohiro-chiyomi-bl", {
+        (("https://hentaihand.com/en/comic/kouda-tomohiro-chiyomi-"
-            "pattern": r"https://i.hentaihand.com/.*/images/full/\d+.jpg$",
+          "blizzard-comic-aun-2016-12-english-nanda-sore-scans"), {
+            "pattern": r"https://cdn.hentaihand.com/.*/images/304546/\d+.jpg$",
             "count": 19,
             "keyword": {
-                "artists"   : ["kouda tomohiro"],
+                "artists"   : ["Kouda Tomohiro"],
-                "categories": ["manga"],
+                "date"      : "dt:2020-02-06 00:00:00",
-                "date"      : "Feb. 6, 2020, 3:19 p.m.",
+                "gallery_id": 304546,
-                "gallery_id": 272772,
                 "lang"      : "en",
                 "language"  : "English",
-                "relationships": ["family", "step family"],
+                "relationships": ["Family", "Step family"],
                 "tags"      : list,
                 "title"     : r"re:\[Kouda Tomohiro\] Chiyomi Blizzard",
-                "title_jp"  : r"re:\[幸田朋弘\] ちよみブリザード",
+                "title_alt" : r"re:\[幸田朋弘\] ちよみブリザード",
+                "type"      : "Manga",
             },
         }),
-        ("https://hentaihand.com/viewc/272772/kouda-tomohiro-chiyomi-bl"),
     )
 
     def __init__(self, match):
-        self.gallery_id = match.group(1)
+        self.slug = match.group(1)
-        url = "{}/comic/{}".format(self.root, self.gallery_id)
+        url = "{}/api/comics/{}".format(self.root, self.slug)
         GalleryExtractor.__init__(self, match, url)
 
     def metadata(self, page):
-        extr = text.extract_from(page)
+        info = json.loads(page)
-
-        title_en = text.unescape(extr("<h1>", "<"))
-        title_jp = text.unescape(extr("<h2>", "<"))
-        tags = extr('<section id="tags"', "</section>")
-
         data = {
-            "gallery_id" : text.parse_int(self.gallery_id),
+            "gallery_id" : text.parse_int(info["id"]),
-            "title"      : title_en or title_jp,
+            "title"      : info["title"],
-            "title_en"   : title_en,
+            "title_alt"  : info["alternative_title"],
-            "title_jp"   : title_jp,
+            "slug"       : self.slug,
-
+            "type"       : info["category"]["name"],
-            # impossible to parse with strptime()
+            "language"   : info["language"]["name"],
-            "date"       : extr('datetime="', '"'),
+            "lang"       : util.language_to_code(info["language"]["name"]),
+            "tags"       : [t["slug"] for t in info["tags"]],
+            "date"       : text.parse_datetime(
+                info["uploaded_at"], "%Y-%m-%d"),
         }
-
+        for key in ("artists", "authors", "groups", "characters",
-        tdict = collections.defaultdict(list)
+                    "relationships", "parodies"):
-        for path in text.extract_iter(tags, 'href="/', '"'):
+            data[key] = [v["name"] for v in info[key]]
-            kind, _, name = path.partition("/")
-            tdict[kind].append(name.replace("+", " "))
-        data.update(tdict)
-
-        if "languages" in data:
-            data["language"] = data["languages"][-1].capitalize()
-            data["lang"] = util.language_to_code(data["language"])
-            del data["languages"]
         return data
 
     def images(self, _):
-        url = "{}/viewc/{}/1".format(self.root, self.gallery_id)
+        info = self.request(self.gallery_url + "/images").json()
-        page = self.request(url).text
+        return [(img["source_url"], img) for img in info["images"]]
-        images = text.extract(page, "var images", ";")[0]
-        return [(img, None) for img in text.extract_iter(images, "'", "'")]
 
 
 class HentaihandTagExtractor(Extractor):
@@ -86,49 +73,49 @@ class HentaihandTagExtractor(Extractor):
     subcategory = "tag"
     root = "https://hentaihand.com"
     pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com"
-               r"(/(?:parody|characters|tags|artists|groups|languages"
+               r"/\w+/(parody|character|tag|artist|group|language"
-               r"|categories|relationships)/[^#]+)")
+               r"|category|relationship)/([^/?&#]+)")
     test = (
-        ("https://hentaihand.com/artists/tony+taka", {
+        ("https://hentaihand.com/en/artist/himuro", {
             "pattern": HentaihandGalleryExtractor.pattern,
-            "count": ">= 50",
+            "count": ">= 18",
         }),
-        ("https://hentaihand.com/artists/tony+taka/popular?page=2"),
+        ("https://hentaihand.com/en/tag/full-color"),
-        ("https://hentaihand.com/tags/full+color"),
+        ("https://hentaihand.com/fr/language/japanese"),
-        ("https://hentaihand.com/languages/japanese"),
+        ("https://hentaihand.com/zh/category/manga"),
-        ("https://hentaihand.com/categories/manga"),
     )
 
     def __init__(self, match):
         Extractor.__init__(self, match)
-        self.path, _, query = match.group(1).partition("?")
+        self.type, self.key = match.groups()
-        self.query = text.parse_query(query)
-        self.query["page"] = text.parse_int(self.query.get("page"), 1)
 
     def items(self):
-        yield Message.Version, 1
+        if self.type[-1] == "y":
-        url = self.root + self.path
+            tpl = self.type[:-1] + "ies"
-        params = self.query.copy()
+        else:
-        data = {"_extractor": HentaihandGalleryExtractor}
+            tpl = self.type + "s"
 
+        url = "{}/api/{}/{}".format(self.root, tpl, self.key)
+        tid = self.request(url, notfound=self.type).json()["id"]
+
+        url = self.root + "/api/comics"
+        params = {
+            "per_page": "18",
+            tpl       : tid,
+            "page"    : 1,
+            "q"       : "",
+            "sort"    : "uploaded_at",
+            "order"   : "desc",
+            "duration": "day",
+        }
         while True:
-            page = self.request(url, params=params).text
+            info = self.request(url, params=params).json()
 
-            for path in text.extract_iter(page, '<a href="/comic/', '"'):
+            for gallery in info["data"]:
-                yield Message.Queue, self.root + "/comic/" + path, data
+                gurl = "{}/en/comic/{}".format(self.root, gallery["slug"])
+                gallery["_extractor"] = HentaihandGalleryExtractor
+                yield Message.Queue, gurl, gallery
 
-            pos = page.find(">(current)<")
+            if params["page"] >= info["last_page"]:
-            if pos < 0 or page.find('class="page-link" href="', pos) < 0:
+                return
-                break
             params["page"] += 1
-
-
-class HentaihandSearchExtractor(HentaihandTagExtractor):
-    """Extractor for search results on hentaihand.com"""
-    subcategory = "search"
-    pattern = r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com(/search/?[^#]+)"
-    test = ("https://hentaihand.com/search?q=color", {
-        "pattern": HentaihandGalleryExtractor.pattern,
-        "range": "1-50",
-        "count": 50,
-    })

test/test_results.py

@@ -31,7 +31,6 @@ TRAVIS_SKIP = {
 
 # temporary issues, etc.
 BROKEN = {
-    "hentaihand",
     "imagevenue",
     "photobucket",
     "worldthree",