[pixiv] move file/ugoira extraction into a separate function

gallery_dl/extractor/pixiv.py

@@ -27,6 +27,8 @@ class PixivExtractor(Extractor):
     filename_fmt = "{id}_p{num}.{extension}"
     archive_fmt = "{id}{suffix}.{extension}"
     cookies_domain = None
+    url_sanity = ("https://s.pximg.net/common/images"
+                  "/limit_sanity_level_360.png")
 
     def _init(self):
         self.api = PixivAppAPI(self)
@@ -46,8 +48,6 @@ class PixivExtractor(Extractor):
             def transform_tags(work):
                 work["tags"] = [tag["name"] for tag in work["tags"]]
 
-        url_sanity = ("https://s.pximg.net/common/images"
-                      "/limit_sanity_level_360.png")
         ratings = {0: "General", 1: "R-18", 2: "R-18G"}
         meta_user = self.config("metadata")
         meta_bookmark = self.config("metadata-bookmark")
@@ -60,11 +60,7 @@ class PixivExtractor(Extractor):
             if not work["user"]["id"]:
                 continue
 
-            meta_single_page = work["meta_single_page"]
+            files = self._extract_files(work)
-            meta_pages = work["meta_pages"]
-            del work["meta_single_page"]
-            del work["image_urls"]
-            del work["meta_pages"]
 
             if meta_user:
                 work.update(self.api.user_detail(work["user"]["id"]))
@@ -81,69 +77,82 @@ class PixivExtractor(Extractor):
             work.update(metadata)
 
             yield Message.Directory, work
-
+            for work["num"], file in enumerate(files):
-            if work["type"] == "ugoira":
+                url = file["url"]
-                if not self.load_ugoira:
+                work.update(file)
-                    continue
-
-                try:
-                    ugoira = self.api.ugoira_metadata(work["id"])
-                except exception.StopExtraction as exc:
-                    self.log.warning(
-                        "Unable to retrieve Ugoira metatdata (%s - %s)",
-                        work.get("id"), exc.message)
-                    continue
-
-                url = ugoira["zip_urls"]["medium"]
-                work["frames"] = frames = ugoira["frames"]
-                work["date_url"] = self._date_from_url(url)
-                work["_http_adjust_extension"] = False
-
-                if self.load_ugoira == "original":
-                    base, sep, _ = url.rpartition("_ugoira")
-                    base = base.replace(
-                        "/img-zip-ugoira/", "/img-original/", 1) + sep
-
-                    for ext in ("jpg", "png", "gif"):
-                        try:
-                            url = ("{}0.{}".format(base, ext))
-                            self.request(url, method="HEAD")
-                            break
-                        except exception.HttpError:
-                            pass
-                    else:
-                        self.log.warning(
-                            "Unable to find Ugoira frame URLs (%s)",
-                            work.get("id"))
-                        continue
-
-                    for num, frame in enumerate(frames):
-                        url = ("{}{}.{}".format(base, num, ext))
-                        work["num"] = work["_ugoira_frame_index"] = num
-                        work["suffix"] = "_p{:02}".format(num)
-                        text.nameext_from_url(url, work)
-                        yield Message.Url, url, work
-
-                else:
-                    url = url.replace("_ugoira600x600", "_ugoira1920x1080")
-                    yield Message.Url, url, text.nameext_from_url(url, work)
-
-            elif work["page_count"] == 1:
-                url = meta_single_page["original_image_url"]
-                if url == url_sanity:
-                    self.log.warning(
-                        "Unable to download work %s ('sanity_level' warning)",
-                        work["id"])
-                    continue
                 work["date_url"] = self._date_from_url(url)
                 yield Message.Url, url, text.nameext_from_url(url, work)
 
+    def _extract_files(self, work):
+        meta_single_page = work["meta_single_page"]
+        meta_pages = work["meta_pages"]
+        del work["meta_single_page"]
+        del work["image_urls"]
+        del work["meta_pages"]
+
+        files = []
+        if work["type"] == "ugoira":
+            if self.load_ugoira:
+                try:
+                    self._extract_ugoira(work, files)
+                except exception.StopExtraction as exc:
+                    self.log.warning(
+                        "Unable to retrieve Ugoira metatdata (%s - %s)",
+                        work["id"], exc.message)
+
+        elif work["page_count"] == 1:
+            url = meta_single_page["original_image_url"]
+            if url == self.url_sanity:
+                self.log.warning(
+                    "Unable to download work %s ('sanity_level' warning)",
+                    work["id"])
             else:
-                for work["num"], img in enumerate(meta_pages):
+                files.append({"url": url})
-                    url = img["image_urls"]["original"]
+
-                    work["date_url"] = self._date_from_url(url)
+        else:
-                    work["suffix"] = "_p{:02}".format(work["num"])
+            for num, img in enumerate(meta_pages):
-                    yield Message.Url, url, text.nameext_from_url(url, work)
+                files.append({
+                    "url"   : img["image_urls"]["original"],
+                    "suffix": "_p{:02}".format(num),
+                })
+
+        return files
+
+    def _extract_ugoira(self, work, files):
+        ugoira = self.api.ugoira_metadata(work["id"])
+        url = ugoira["zip_urls"]["medium"]
+        work["frames"] = frames = ugoira["frames"]
+        work["date_url"] = self._date_from_url(url)
+        work["_http_adjust_extension"] = False
+
+        if self.load_ugoira == "original":
+            base, sep, _ = url.rpartition("_ugoira")
+            base = base.replace("/img-zip-ugoira/", "/img-original/", 1) + sep
+
+            for ext in ("jpg", "png", "gif"):
+                try:
+                    url = "{}0.{}".format(base, ext)
+                    self.request(url, method="HEAD")
+                    break
+                except exception.HttpError:
+                    pass
+            else:
+                return self.log.warning(
+                    "Unable to find Ugoira frame URLs (%s)", work["id"])
+
+            for num in range(len(frames)):
+                url = "{}{}.{}".format(base, num, ext)
+                files.append(text.nameext_from_url(url, {
+                    "url": url,
+                    "num": num,
+                    "suffix": "_p{:02}".format(num),
+                    "_ugoira_frame_index": num,
+
+                }))
+        else:
+            files.append({
+                "url": url.replace("_ugoira600x600", "_ugoira1920x1080", 1),
+            })
 
     @staticmethod
     def _date_from_url(url, offset=timedelta(hours=9)):
@@ -175,6 +184,9 @@ class PixivExtractor(Extractor):
             "x_restrict"      : 0,
         }
 
+    def _web_to_mobile(self, work):
+        return work
+
     def works(self):
         """Return an iterable containing all relevant 'work' objects"""