[hbrowse] extract hmanga metadata & general maintenance

gallery_dl/extractor/hbrowse.py

@@ -13,66 +13,87 @@ from .. import text
 import json
 
 
-class HbrowseMangaExtractor(MangaExtractor):
+class HbrowseExtractor(Extractor):
-    """Extractor for manga from hbrowse.com"""
+    """Base class for hbrowse extractors"""
     category = "hbrowse"
+    root = "http://www.hbrowse.com"
+
+    @staticmethod
+    def _parse_page(page, data):
+        text.extract_all(page, (
+            ('manga' , '<td class="listLong">', '</td>'),
+            ('artist', '<td class="listLong">', '</td>'),
+            ('total' , '<td class="listLong">', ' '),
+            ('origin', '<td class="listLong">', '</td>'),
+        ), values=data)
+
+        data["manga"] = text.unescape(data["manga"])
+        data["total"] = int(data["total"])
+        data["artist"] = text.remove_html(data["artist"])
+        data["origin"] = text.remove_html(data["origin"])
+
+
+class HbrowseMangaExtractor(MangaExtractor, HbrowseExtractor):
+    """Extractor for manga from hbrowse.com"""
     pattern = [r"(?:https?://)?((?:www\.)?hbrowse\.com/\d+)/?$"]
     reverse = False
     test = [("http://www.hbrowse.com/10363", {
         "url": "4d9def5df21c23f8c3d36de2076c189c02ea43bd",
+        "keyword": "aa0c6ba9ba180f18861aa5d608ff7f1966e666f8",
     })]
 
     def chapters(self, page):
+        results = []
+        data = {"manga_id": int(self.url.rstrip("/").rpartition("/")[2])}
+        self._parse_page(page, data)
+
+        pos = 0
         needle = '<td class="listMiddle">\n<a class="listLink" href="'
-        return list(text.extract_iter(page, needle, '"'))
+        while True:
+            url, pos = text.extract(page, needle, '"', pos)
+            if not url:
+                return results
+            title, pos = text.extract(page, '>View ', '<', pos)
+            results.append((url, {
+                "chapter": int(url.rpartition("/")[2][1:]),
+                "title": title, **data
+            }))
 
 
-class HbrowseChapterExtractor(Extractor):
+class HbrowseChapterExtractor(HbrowseExtractor):
     """Extractor for manga-chapters from hbrowse.com"""
-    category = "hbrowse"
     subcategory = "chapter"
-    directory_fmt = ["{category}", "{gallery_id} {title}", "c{chapter:>05}"]
+    directory_fmt = ["{category}", "{manga_id} {manga}", "c{chapter:>05}"]
-    filename_fmt = ("{category}_{gallery_id}_{chapter:>05}_"
+    filename_fmt = ("{category}_{manga_id}_{chapter:>05}_"
                     "{num:>03}.{extension}")
-    pattern = [r"(?:https?://)?(?:www\.)?hbrowse\.com/(\d+)/(c\d+)"]
+    pattern = [r"(?:https?://)?(?:www\.)?hbrowse\.com/(\d+)/c(\d+)"]
     test = [("http://www.hbrowse.com/10363/c00000", {
         "url": "634f4800858913f097bc3b62a8fedaf74b5254bd",
-        "keyword": "f0f96cefda19e5aee1a19454f63ffe3a425602ab",
+        "keyword": "730bd33de2a0a0fb4e0b6dcdafedcaeee1060047",
         "content": "44578ebbe176c2c27434966aef22945787e2781e",
     })]
-    url_base = "http://www.hbrowse.com"
 
     def __init__(self, match):
-        Extractor.__init__(self)
+        HbrowseExtractor.__init__(self)
         self.gid, self.chapter = match.groups()
-        self.path = "/{}/{}/".format(self.gid, self.chapter)
+        self.path = "/{}/c{}/".format(self.gid, self.chapter)
 
     def items(self):
-        page = self.request(self.url_base + self.path).text
+        page = self.request(self.root + self.path).text
         data = self.get_job_metadata(page)
         yield Message.Version, 1
         yield Message.Directory, data
-        for num, url in enumerate(self.get_image_urls(page), 1):
+        for data["num"], url in enumerate(self.get_image_urls(page), 1):
-            data["num"] = num
             yield Message.Url, url, text.nameext_from_url(url, data)
 
     def get_job_metadata(self, page):
         """Collect metadata for extractor-job"""
-        data = {
+        data = {"manga_id": int(self.gid), "chapter": int(self.chapter)}
-            'gallery_id': self.gid,
+        self._parse_page(page, data)
-            "chapter": int(self.chapter[1:]),
+        return data
-        }
-        return text.extract_all(page, (
-            ('title'      , '<td class="listLong">', '</td>'),
-            (None         , '<td class="listLong">', ''),
-            ('artist'     , '>', '<'),
-            ('count_total', '<td class="listLong">', ' '),
-            (None         , '<td class="listLong">', ''),
-            ('origin'     , '>', '<'),
-        ), values=data)[0]
 
     def get_image_urls(self, page):
         """Yield all image-urls for a 'chapter'"""
-        base = self.url_base + "/data" + self.path
+        base = self.root + "/data" + self.path
         json_data = text.extract(page, ';list = ', ',"zzz"')[0] + "]"
         return [base + name for name in json.loads(json_data)]

gallery_dl/extractor/mangazuki.py

@@ -80,7 +80,7 @@ class MangazukiMangaExtractor(MangaExtractor):
             "url": "aab747414191b14e768f4a1eb148448d83ef2e14",
         }),
         ("https://raws.mangazuki.co/series/Rakujitsu-no-Pathos", {
-            "url": "57ac10ce4f4a93a313c80542bbc5bd6fd922b055",
+            "url": "0b85292b096909e8419632f35d3e2680d468c12c",
         }),
     ]
 

test/test_extractors.py

@@ -66,9 +66,9 @@ skip = [
     "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie",
     "archivedmoe", "archiveofsins", "thebarchive",
     # temporary issues
-    "imgtrex",  # 504
+    "luscious",  # "high load"
-    "hentaifoundry",  # SSL cert expired
     "pawoo",
+    "seaotterscans",  # "Name or service not known"
 ]
 # enable selective testing for direct calls
 if __name__ == '__main__' and len(sys.argv) > 1: