[sexcom] update

- fix 'title' and 'type' of pictures
- remove '#' from the beginning of each tag
- add 'gifs' option

docs/configuration.rst

@@ -4371,6 +4371,16 @@ Description
     Download videos.
 
 
+extractor.sexcom.gifs
+---------------------
+Type
+    ``bool``
+Default
+    ``true``
+Description
+    Download animated images as ``.gif`` instead of ``.webp``
+
+
 extractor.skeb.article
 ----------------------
 Type

docs/gallery-dl.conf

@@ -582,6 +582,10 @@
             "password": "",
             "sleep-request": "0.5-1.5"
         },
+        "sexcom":
+        {
+            "gifs": true
+        },
         "skeb":
         {
             "article"      : false,

gallery_dl/extractor/sexcom.py

@@ -24,6 +24,8 @@ class SexcomExtractor(Extractor):
     root = "https://www.sex.com"
 
     def items(self):
+        self.gifs = self.config("gifs", True)
+
         yield Message.Directory, self.metadata()
         for pin in map(self._parse_pin, self.pins()):
             if not pin:
@@ -38,6 +40,7 @@ class SexcomExtractor(Extractor):
                     pin["date"] = dt
             except Exception:
                 pass
+            pin["tags"] = [t[1:] for t in pin["tags"]]
 
             yield Message.Url, url, pin
 
@@ -73,7 +76,7 @@ class SexcomExtractor(Extractor):
             return self._parse_pin_legacy(response)
         if "/videos/" in response.url:
             return self._parse_pin_video(response)
-        return self._parse_pin_gifs(response)
+        return self._parse_pin_image(response)
 
     def _parse_pin_legacy(self, response):
         extr = text.extract_from(response.text)
@@ -124,20 +127,31 @@ class SexcomExtractor(Extractor):
 
         return data
 
-    def _parse_pin_gifs(self, response):
+    def _parse_pin_image(self, response):
         extr = text.extract_from(response.text)
+        href = extr(' href="', '"').partition("?")[0]
+        title, _, type = extr("<title>", " | ").rpartition(" ")
 
         data = {
             "_http_headers": {"Referer": response.url},
-            "type": "gif",
+            "url": href,
-            "url": extr(' href="', '"').partition("?")[0],
+            "title": text.unescape(title),
-            "title": text.unescape(extr("<title>", " Gif | Sex.com<")),
             "pin_id": text.parse_int(extr(
                 'rel="canonical" href="', '"').rpartition("/")[2]),
             "tags": text.split_html(extr("</h1>", "</section>")),
         }
 
-        return text.nameext_from_url(data["url"], data)
+        text.nameext_from_url(href, data)
+        if type.lower() == "pic":
+            data["type"] = "picture"
+        else:
+            data["type"] = "gif"
+            if self.gifs and data["extension"] == "webp":
+                data["extension"] = "gif"
+                data["_fallback"] = (href,)
+                data["url"] = href[:-4] + "gif"
+
+        return data
 
     def _parse_pin_video(self, response):
         extr = text.extract_from(response.text)
@@ -147,6 +161,7 @@ class SexcomExtractor(Extractor):
 
         data = {
             "_ytdl_manifest": "hls",
+            "_ytdl_manifest_headers": {"Referer": response.url},
             "extension": "mp4",
             "type": "video",
             "title": text.unescape(extr("<title>", " | Sex.com<")),

test/results/sexcom.py

@@ -13,6 +13,7 @@ __tests__ = (
     "#comment" : "picture (legacy URL)",
     "#category": ("", "sexcom", "pin"),
     "#class"   : sexcom.SexcomPinExtractor,
+    "#skip"    : "legacy",
     "#urls"        : "https://imagex1.sx.cdn.live/images/pinporn/2014/08/26/7637609.jpg",
     "#sha1_content": "8cd419c6790ef7348bd398c364ab10f956e438dc",
 
@@ -49,9 +50,9 @@ __tests__ = (
     "extension": "jpg",
     "filename" : "7637609",
     "pin_id"   : 612398,
-    "tags"     : ["#Hot"],
+    "tags"     : ["Hot"],
-    "title"    : "",
+    "title"    : "Sexy Ecchi Girls 166",
-    "type"     : "gif",
+    "type"     : "picture",
 },
 
 {
@@ -70,9 +71,10 @@ __tests__ = (
     "title"    : "Ecchi",
     "type"     : "gif",
     "url"      : "https://imagex1.sx.cdn.live/images/pinporn/2017/12/07/18760842.gif",
+    "_fallback": ("https://imagex1.sx.cdn.live/images/pinporn/2017/12/07/18760842.webp",),
     "tags"     : [
-        "#Big Tits",
+        "Big Tits",
-        "#Hentai",
+        "Hentai",
     ],
 },
 
@@ -81,28 +83,30 @@ __tests__ = (
     "#comment" : "gif",
     "#category": ("", "sexcom", "pin"),
     "#class"   : sexcom.SexcomPinExtractor,
-    "#urls"        : "https://imagex1.sx.cdn.live/images/pinporn/2017/12/07/18760842.gif",
+    "#options" : {"gifs": False},
-    "#sha1_content": "176cc63fa05182cb0438c648230c0f324a5965fe",
+    "#urls"        : "https://imagex1.sx.cdn.live/images/pinporn/2017/12/07/18760842.webp",
+    "#sha1_content": "d5d58fbb92f87be49a37d29d82687c9efa7f796f",
 
     "date"     : "dt:2017-12-07 00:00:00",
     "date_url" : "dt:2017-12-07 00:00:00",
-    "extension": "gif",
+    "extension": "webp",
     "filename" : "18760842",
     "pin_id"   : 209061,
     "title"    : "Ecchi",
     "type"     : "gif",
-    "url"      : "https://imagex1.sx.cdn.live/images/pinporn/2017/12/07/18760842.gif",
+    "url"      : "https://imagex1.sx.cdn.live/images/pinporn/2017/12/07/18760842.webp",
     "tags"     : [
-        "#Big Tits",
+        "Big Tits",
-        "#Hentai",
+        "Hentai",
     ],
 },
 
 {
     "#url"     : "https://www.sex.com/pin/55748341/",
-    "#comment" : "video legacy URL",
+    "#comment" : "video (legacy URL)",
     "#category": ("", "sexcom", "pin"),
     "#class"   : sexcom.SexcomPinExtractor,
+    "#skip"    : "gone",
     "#urls"        : "https://video1.sx.cdn.live/videos/pinporn/2018/02/10/776229_hd.mp4",
     "#sha1_content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff",
 
@@ -137,27 +141,27 @@ __tests__ = (
     "type"     : "video",
     "url"      : "ytdl:https://videos.sex.com/680933/video.m3u8",
     "tags"     : [
-        "#Babe",
+        "Babe",
-        "#Beach",
+        "Beach",
-        "#Big Boobs",
+        "Big Boobs",
-        "#Bikini",
+        "Bikini",
-        "#Brunette",
+        "Brunette",
-        "#Brunette Babe",
+        "Brunette Babe",
-        "#Girlfriend",
+        "Girlfriend",
-        "#Natural Tits",
+        "Natural Tits",
-        "#Nude",
+        "Nude",
-        "#Pool",
+        "Pool",
-        "#Poolside",
+        "Poolside",
-        "#Public Sex",
+        "Public Sex",
-        "#Russian",
+        "Russian",
-        "#Shower",
+        "Shower",
-        "#Small Boobs",
+        "Small Boobs",
-        "#Swimming",
+        "Swimming",
-        "#Swimming Pool",
+        "Swimming Pool",
-        "#Tight Pussy",
+        "Tight Pussy",
-        "#Underwater",
+        "Underwater",
-        "#With",
+        "With",
-        "#Young",
+        "Young",
     ],
 },