[jpgchurch] refactor

gallery_dl/extractor/__init__.py

@@ -73,6 +73,7 @@ modules = [
     "kissgoddess",
     "kohlchan",
     "komikcast",
+    "lensdump",
     "lightroom",
     "lineblog",
     "livedoor",

gallery_dl/extractor/jpgchurch.py

@@ -12,75 +12,37 @@ from .. import text
 BASE_PATTERN = r"(?:https?://)?jpg\.church"
 
 
-class JpgchurchImageExtractor(Extractor):
+class JpgchurchExtractor(Extractor):
-    """Base Extractor for Jpgchurch Images"""
+    """Base class for Jpgchurch extractors"""
-    category = "Jpgchurch"
+    category = "jpgchurch"
-    subcategory = "image"
-    directory_fmt = ("{category}", "{user}")
-    filename_fmt = "{filename}"
-    pattern = BASE_PATTERN + r"/img/([\w\d\-\.]+)"
     root = "https://jpg.church"
-    test = ("https://jpg.church/img/funnymeme.LecXGS",)
+    directory_fmt = ("{category}", "{user}",)
+    archive_fmt = "{filename}"
 
     def __init__(self, match):
         Extractor.__init__(self, match)
-        self.image = match.group(1)
 
     def items(self):
-        data = self.metadata()
         for image in self.images():
-            if "album" in image or "user" in image:
+            yield Message.Directory, image
-                data.update(image)
-            yield Message.Directory, data
             yield Message.Url, image["url"], image
 
-    def metadata(self):
-        """Return general metadata"""
-        return {}
-
     def images(self):
         """Return an iterable containing the image(s)"""
-        url = "{}/img/{}".format(self.root, self.image)
-        return [self._get_images(url)]
-
-    def _get_images(self, url):
-        page = self.request(url).text
-        data = self._extract_image(page)
-        data.update({
-            "user": data["user"].split("/")[-1],
-            "extension": text.ext_from_url(data["url"])
-        })
-        return data
 
     @staticmethod
-    def _extract_image(page):
+    def _extract_user(page):
-        _page = text.extract(
+        return text.extract(page, 'username: "', '"')[0]
-            page,
-            '<div class="header-content-right">', '<span class="user-image')[0]
-        return text.extract_all(_page, (
-            ('url', '<a href="', '" download='),
-            ('filename', '"', '" class'),
-            ('user', '<a href="', '" class="user-image">')))[0]
 
-
+    def _extract_image(self, url):
-class JpgchurchAlbumExtractor(JpgchurchImageExtractor, Extractor):
+        page = self.request(url).text
-    """Extractor for Jpgchurch Albums"""
+        data = {
-    subcategory = "album"
+            "url": text.extract(
-    directory_fmt = ("{category}", "{user}", "{album}",)
+                page, '<meta property="og:image" content="', '" />')[0],
-    pattern = BASE_PATTERN + r"/a(?:lbum)?/([\w\d\-\.]+)"
+        }
-    test = ("https://jpg.church/album/CDilP/?sort=date_desc&page=1",)
+        text.nameext_from_url(data["url"], data)
-
+        data["user"] = self._extract_user(page)
-    def __init__(self, match):
+        return data
-        Extractor.__init__(self, match)
-        self.album = match.group(1).split('.')[-1]
-
-    def metadata(self):
-        return {"album": self.album}
-
-    def images(self):
-        url = "{}/a/{}".format(self.root, self.album)
-        for _url in self._get_album_images(url):
-            yield self._get_images(_url)
 
     def _pagination(self, url):
         """Uses recursion to yield the next page"""
@@ -92,30 +54,114 @@ class JpgchurchAlbumExtractor(JpgchurchImageExtractor, Extractor):
             url = _next
             yield from self._pagination(_next)
 
-    def _get_album_images(self, url):
+    def _get_images(self, url):
-        for _url in self._pagination(url):
+        for url in self._pagination(url):
-            page = self.request(_url).text
+            page = self.request(url).text
-            _page = text.extract_iter(
+            album = text.extract(page, '<a data-text="album-name"', '</h1>')[0]
+            album = text.extract(album, '>', '</a>')[0]
+            page = text.extract_iter(
                 page, '<div class="list-item-image ', 'image-container')
-            for image in _page:
+            for image in page:
-                yield text.extract(image, '<a href="', '" class')[0]
+                image = text.extract(image, '<a href="', '"')[0]
+                data = self._extract_image(image)
+                data["album"] = album
+                yield data
+
+    def _get_albums(self, url):
+        for url in self._pagination(url):
+            page = self.request(url).text
+            album = text.extract(page, '<a data-text="album-name"', '</h1>')[0]
+            album = text.extract(album, '>', '</a>')[0]
+            page = text.extract_iter(
+                page, '<div class="list-item-image ', 'image-container')
+            for image in page:
+                image = text.extract(image, '<a href="', '"')[0]
+                yield image
 
 
-class JpgchurchUserExtractor(JpgchurchAlbumExtractor, Extractor):
+class JpgchurchImageExtractor(JpgchurchExtractor):
-    """Extractor for Jpgchurch Users"""
+    """Extractor for Jpgchurch Images"""
-    subcategory = "user"
+    subcategory = "image"
-    directory_fmt = ("{category}", "{user}",)
+    pattern = BASE_PATTERN + r"/img/([^/?#]+)"
-    pattern = BASE_PATTERN + r"/(?!img|a(?:lbum)?)([\w\d\-\.]+)"
+    test = (
-    test = ("https://jpg.church/exearco",)
+        ("https://jpg.church/img/funnymeme.LecXGS"),
+    )
 
     def __init__(self, match):
-        Extractor.__init__(self, match)
+        JpgchurchExtractor.__init__(self, match)
-        self.user = match.group(1)
+        self.image = match.group(1)
-
-    def metadata(self):
-        return {"user": self.user}
 
     def images(self):
+        url = "{}/img/{}".format(self.root, self.image)
+        yield self._extract_image(url)
+
+
+class JpgchurchAlbumExtractor(JpgchurchExtractor):
+    """Extractor for Jpgchurch Albums"""
+    subcategory = "album"
+    directory_fmt = ("{category}", "{user}", "{album}",)
+    pattern = BASE_PATTERN + r"/a(?:lbum)?/([^/?#]+)(/sub)?"
+    test = (
+        ("https://jpg.church/album/CDilP/?sort=date_desc&page=1", {
+            "count": 2,
+            "pattern": r"^https://[^/]+/.*\.(jpg|png)",
+        }),
+        ("https://jpg.church/a/gunggingnsk.N9OOI", {
+            "count": 114,
+        }),
+        ("https://jpg.church/a/101-200.aNJ6A/", {
+            "count": 100,
+        }),
+        ("https://jpg.church/a/hannahowo.aNTdH/sub", {
+            "count": 606,
+        }),
+    )
+
+    def __init__(self, match):
+        JpgchurchExtractor.__init__(self, match)
+        self.album = match.group(1)
+        self.is_sub = match.group(2)
+
+    def images(self):
+        url = "{}/a/{}".format(self.root, self.album)
+        if self.is_sub:
+            url += "/sub"
+            for album in self._get_albums(url):
+                yield from self._get_images(album)
+        else:
+            yield from self._get_images(url)
+
+
+class JpgchurchUserExtractor(JpgchurchExtractor):
+    """Extractor for Jpgchurch Users"""
+    subcategory = "user"
+    pattern = BASE_PATTERN + r"/(?!img|a(?:lbum)?)([^/?#]+)(/albums)?"
+    test = (
+        ("https://jpg.church/exearco", {
+            "count": 3,
+        }),
+        ("https://jpg.church/exearco/albums", {
+            "count": 1,
+        }),
+    )
+
+    def __init__(self, match):
+        JpgchurchExtractor.__init__(self, match)
+        self.user = match.group(1)
+        self.is_album = match.group(2)
+
+    def items(self):
         url = "{}/{}".format(self.root, self.user)
-        for _url in self._get_album_images(url):
+        if self.is_album:
-            yield self._get_images(_url)
+            url += "/albums"
+            data = {
+                "_extractor": JpgchurchAlbumExtractor,
+            }
+            for album in self._get_albums(url):
+                yield Message.Queue, album, data
+        else:
+            data = {
+                "_extractor": JpgchurchImageExtractor,
+            }
+            for image in self._get_albums(url):
+                yield Message.Queue, image, data