[girlsreleased] add support (#6200)

* [girlsreleased] add support
* Lint
* Change "galleries" to "sets"
    As it is on the site
* Add pagination
    Add tests
* Fix tests
* Remove leftover print()
* Don't remove first set
* Yield pages
* Add filename metadata
* [girlsreleased] Refactor
* Return models as array
* Add filename numbering
    Add date metadata
* Add URL metadata
* Spawn set extractor the right way
* Adjust model/site regex
* update
    - restructure some code
    - remove constructors
    - use f-strings
* expand tests
* update docs/supportedsites

docs/supportedsites.md

@@ -325,6 +325,12 @@ Consider all listed sites to potentially be NSFW.
     <td>Posts, Search Results</td>
     <td>Supported</td>
 </tr>
+<tr>
+    <td>Girlsreleased</td>
+    <td>https://girlsreleased.com/</td>
+    <td>Models, Sets, Sites</td>
+    <td></td>
+</tr>
 <tr>
     <td>Gofile</td>
     <td>https://gofile.io/</td>

gallery_dl/extractor/__init__.py

@@ -63,6 +63,7 @@ modules = [
     "gelbooru",
     "gelbooru_v01",
     "gelbooru_v02",
+    "girlsreleased",
     "girlswithmuscle",
     "gofile",
     "hatenablog",

gallery_dl/extractor/girlsreleased.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://girlsreleased.com/"""
+
+from .common import Extractor, Message
+from .. import text
+import itertools
+
+BASE_PATTERN = r"(?:https?://)?(?:www\.)?girlsreleased\.com"
+
+
+class GirlsreleasedExtractor(Extractor):
+    """Base class for girlsreleased extractors"""
+    category = "girlsreleased"
+    root = "https://girlsreleased.com"
+    request_interval = 0.5
+    request_interval_min = 0.2
+
+    def items(self):
+        data = {"_extractor": GirlsreleasedSetExtractor}
+        base = f"{self.root}/set/"
+        for set in self._pagination():
+            yield Message.Queue, f"{base}{set[0]}", data
+
+    def _pagination(self):
+        base = f"{self.root}/api/0.1/sets/{self._path}/{self.groups[0]}/page/"
+        for pnum in itertools.count():
+            sets = self.request_json(f"{base}{pnum}")["sets"]
+            if not sets:
+                return
+
+            yield from sets[1:] if pnum else sets
+            if len(sets) < 80:
+                return
+
+
+class GirlsreleasedSetExtractor(GirlsreleasedExtractor):
+    """Extractor for girlsreleased galleries"""
+    subcategory = "set"
+    pattern = BASE_PATTERN + r"/set/(\d+)"
+    example = "https://girlsreleased.com/set/12345"
+
+    def items(self):
+        url = f"{self.root}/api/0.1/set/{self.groups[0]}"
+        json = self.request_json(url)["set"]
+        data = {
+            "title": json["name"] or json["id"],
+            "id": json["id"],
+            "site": json["site"],
+            "model": [model for _, model in json["models"]],
+            "date": text.parse_timestamp(json["date"]),
+            "count": len(json["images"]),
+            "url": "https://girlsreleased.com/set/" + json["id"],
+        }
+        yield Message.Directory, data
+        for data["num"], image in enumerate(json["images"], 1):
+            text.nameext_from_url(image[5], data)
+            yield Message.Queue, image[3], data
+
+
+class GirlsreleasedModelExtractor(GirlsreleasedExtractor):
+    """Extractor for girlsreleased models"""
+    subcategory = _path = "model"
+    pattern = BASE_PATTERN + r"/model/(\d+(?:/.+)?)"
+    example = "https://girlsreleased.com/model/12345/MODEL"
+
+
+class GirlsreleasedSiteExtractor(GirlsreleasedExtractor):
+    """Extractor for girlsreleased sites"""
+    subcategory = _path = "site"
+    pattern = BASE_PATTERN + r"/site/([^/?#]+(?:/model/\d+/?.*)?)"
+    example = "https://girlsreleased.com/site/SITE"

test/results/girlsreleased.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import girlsreleased
+
+
+__tests__ = (
+{
+    "#url"     : "https://girlsreleased.com/set/32332",
+    "#category": ("", "girlsreleased", "set"),
+    "#class"   : girlsreleased.GirlsreleasedSetExtractor,
+    "#pattern" : r"https://imx\.to/i/\w+",
+    "#count"   : 122,
+
+    "id"        : "32332",
+    "title"     : "Monadiko",
+    "model"     : ["Mia Sollis"],
+    "site"      : "sexart.com",
+    "date"      : "dt:2014-07-27 07:57:19",
+},
+
+{
+    "#url"     : "https://girlsreleased.com/set/124943",
+    "#category": ("", "girlsreleased", "set"),
+    "#class"   : girlsreleased.GirlsreleasedSetExtractor,
+    "#pattern" : r"https://imx\.to/i/\w+",
+    "#count"   : 79,
+
+    "id"        : "124943",
+    "title"     : "124943",
+    "model"     : ["Iveta"],
+    "site"      : "errotica-archives.com",
+    "date"      : "dt:2022-02-21 14:08:32",
+},
+
+{
+    "#url"     : "https://girlsreleased.com/model/11545/Ariana%20Regent",
+    "#category": ("", "girlsreleased", "model"),
+    "#class"   : girlsreleased.GirlsreleasedModelExtractor,
+    "#results" : (
+        "https://girlsreleased.com/set/142691",
+        "https://girlsreleased.com/set/142690",
+    ),
+},
+
+{
+    "#url"     : "https://girlsreleased.com/site/amourangels.com",
+    "#category": ("", "girlsreleased", "site"),
+    "#class"   : girlsreleased.GirlsreleasedSiteExtractor,
+},
+
+{
+    "#url"     : "https://girlsreleased.com/site/femjoy.com/model/854/Gabi",
+    "#category": ("", "girlsreleased", "site"),
+    "#class"   : girlsreleased.GirlsreleasedSiteExtractor,
+    "#pattern" : girlsreleased.GirlsreleasedSetExtractor.pattern,
+    "#count"   : 17,
+},
+
+)