[fansly] improve file extraction code (#4401)

- add 'format' option
- handle exceptions

docs/configuration.rst

@@ -2811,6 +2811,18 @@ Description
     `fanbox.comments <extractor.fanbox.comments_>`__
 
 
+extractor.fansly.format
+-----------------------
+Type
+    ``list`` of ``integers``
+Default
+    ``[303, 302, 1, 2, 4]``
+Description
+    Selects the file format to extract.
+
+    When more than one format is given, the first available one is selected.
+
+
 extractor.fansly.token
 ----------------------
 Type

docs/gallery-dl.conf

@@ -330,7 +330,9 @@
         },
         "fansly":
         {
-            "token": ""
+            "token": "",
+
+            "format": [303, 302, 1, 2, 4]
         },
         "flickr":
         {

gallery_dl/extractor/fansly.py

@@ -9,7 +9,7 @@
 """Extractors for https://fansly.com/"""
 
 from .common import Extractor, Message
-from .. import text
+from .. import text, util
 import time
 
 BASE_PATTERN = r"(?:https?://)?(?:www\.)?fansly\.com"
@@ -25,6 +25,7 @@ class FanslyExtractor(Extractor):
 
     def _init(self):
         self.api = FanslyAPI(self)
+        self.formats = self.config("format") or (303, 302, 1, 2, 4)
 
     def items(self):
         for post in self.posts():
@@ -40,49 +41,71 @@ class FanslyExtractor(Extractor):
 
     def _extract_files(self, post):
         files = []
-
         for attachment in post.pop("attachments"):
-            media = attachment["media"]
-            file = {
-                **media,
-                "date": text.parse_timestamp(media["createdAt"]),
-                "date_updated": text.parse_timestamp(media["updatedAt"]),
-            }
+            try:
+                self._extract_attachment(files, post, attachment)
+            except Exception as exc:
+                self.log.debug("", exc_info=exc)
+                self.log.error(
+                    "%s/%s, Failed to extract media (%s: %s)",
+                    post["id"], attachment.get("id"),
+                    exc.__class__.__name__, exc)
+        return files
 
-            width = 0
-            for variant in media["variants"]:
-                if variant["width"] > width:
-                    width = variant["width"]
-                    variant_max = variant
-                if variant["type"] == 303:
-                    break
-            else:
-                # image
-                file["type"] = "image"
-                files.append({
-                    "file": file,
-                    "url" : variant_max["locations"][0]["location"],
-                })
-                continue
+    def _extract_attachment(self, files, post, attachment):
+        media = attachment["media"]
+        variants = {
+            variant["type"]: variant
+            for variant in media.pop("variants", ())
+        }
+        variants[media["type"]] = media
 
-            # video
-            location = variant["locations"][0]
+        for fmt in self.formats:
+            if fmt in variants and (variant := variants[fmt]).get("locations"):
+                break
+        else:
+            return self.log.warning(
+                "%s/%s: Requested format not available",
+                post["id"], attachment["id"])
+
+        mime = variant["mimetype"]
+        location = variant.pop("locations")[0]
+        if "metadata" in variant:
+            try:
+                variant.update(util.json_loads(variant.pop("metadata")))
+            except Exception:
+                pass
+
+        file = {
+            **variant,
+            "format": fmt,
+            "date": text.parse_timestamp(media["createdAt"]),
+            "date_updated": text.parse_timestamp(media["updatedAt"]),
+        }
+
+        if "metadata" in location:
+            # manifest
             meta = location["metadata"]
 
             file["type"] = "video"
             files.append({
                 "file": file,
                 "url": f"ytdl:{location['location']}",
-                "_fallback": (media["locations"][0]["location"],),
-                "_ytdl_manifest": "dash",
+                #  "_fallback": (media["locations"][0]["location"],),
+                "_ytdl_manifest":
+                    "dash" if mime == "application/dash+xml" else "hls",
                 "_ytdl_manifest_cookies": (
                     ("CloudFront-Key-Pair-Id", meta["Key-Pair-Id"]),
                     ("CloudFront-Signature"  , meta["Signature"]),
                     ("CloudFront-Policy"     , meta["Policy"]),
                 ),
             })
-
-        return files
+        else:
+            file["type"] = "image" if mime.startswith("image/") else "video"
+            files.append({
+                "file": file,
+                "url" : location["location"],
+            })
 
 
 class FanslyPostExtractor(FanslyExtractor):
@@ -250,6 +273,7 @@ class FanslyAPI():
                     attachments.extend(
                         media[m["accountMediaId"]]
                         for m in bundle
+                        if m["accountMediaId"] in media
                     )
                 else:
                     self.extractor.log.warning(

test/results/fansly.py

@@ -20,6 +20,40 @@ __tests__ = (
     "#class"   : fansly.FanslyPostExtractor,
 },
 
+{
+    "#url"     : "https://fansly.com/post/545313467469410305",
+    "#comment" : "'This post does not exist or has been deleted.'",
+    "#class"   : fansly.FanslyPostExtractor,
+    "#count"   : 0,
+},
+
+{
+    "#url"     : "https://fansly.com/post/543835794918354944",
+    "#comment" : "one locked image",
+    "#class"   : fansly.FanslyPostExtractor,
+    "#pattern" : r"https://cdn3.fansly.com/364164066794549248/542559086856646656.jpeg\?.+",
+    "#count"   : 1,
+    "#auth"    : False,
+    "#log"     : (
+        "No 'token' provided",
+        "543835794918354944/542560754868432896: Requested format not available",
+    ),
+},
+
+{
+    "#url"     : "https://fansly.com/post/451349524175138816",
+    "#comment" : "locked image + 2 locked videos",
+    "#class"   : fansly.FanslyPostExtractor,
+    "#count"   : 0,
+    "#auth"    : False,
+    "#log"     : (
+        "No 'token' provided",
+        "451349524175138816/451349523013316609: Requested format not available",
+        "451349524175138816/451349523000729600: Requested format not available",
+        "451349524175138816/451349523025899520: Requested format not available",
+    ),
+},
+
 {
     "#url"     : "https://fansly.com/Oliviaus/posts",
     "#class"   : fansly.FanslyCreatorPostsExtractor,