[sankaku] implement support for new 'tags' categories (#7333 #7553)

docs/configuration.rst

@@ -3863,7 +3863,7 @@ Description
 
 
 extractor.pixeldrain.recursive
---------------------------
+------------------------------
 Type
     ``bool``
 Default
@@ -4317,6 +4317,39 @@ Description
     Refresh download URLs before they expire.
 
 
+extractor.sankaku.tags
+----------------------
+Type
+    * ``bool``
+    * ``string``
+Default
+    ``false``
+Description
+    | Group ``tags`` by type and
+      provide them as ``tags_TYPE`` and ``tag_string_TYPE`` metadata fields,
+    | for example ``tags_artist`` and ``tags_character``.
+
+    ``true``
+        Enable general ``tags`` categories
+
+        Requires:
+
+        * 1 additional API request per 100 tags per post
+
+    ``"extended"``
+        Group ``tags`` by the new, extended tag category system
+        used on ``chan.sankakucomplex.com``
+
+        Requires:
+
+        * 1 additional HTTP request per post
+        * logged-in `cookies <extractor.*.cookies_>`__
+          to fetch full ``tags`` category data
+
+    ``false``
+        Disable ``tags`` categories
+
+
 extractor.sankakucomplex.embeds
 -------------------------------
 Type

gallery_dl/extractor/sankaku.py

@@ -47,6 +47,8 @@ class SankakuExtractor(BooruExtractor):
 
     def _init(self):
         self.api = SankakuAPI(self)
+        if self.config("tags") == "extended":
+            self._tags = self._tags_extended
 
     def _file_url(self, post):
         url = post["file_url"]
@@ -85,6 +87,24 @@ class SankakuExtractor(BooruExtractor):
             post["tags_" + name] = values
             post["tag_string_" + name] = " ".join(values)
 
+    def _tags_extended(self, post, page):
+        try:
+            url = "https://chan.sankakucomplex.com/posts/" + post["id"]
+            page = self.request(url).text
+        except Exception as exc:
+            return self.log.warning(
+                "%s: Failed to extract extended tag categories (%s: %s)",
+                post["id"], exc.__class__.__name__, exc)
+
+        tags = collections.defaultdict(list)
+        tag_sidebar = text.extr(page, '<ul id="tag-sidebar"', "</ul>")
+        pattern = re.compile(r"tag-type-([^\"' ]+).*?\?tags=([^\"'&]+)")
+        for tag_type, tag_name in pattern.findall(tag_sidebar):
+            tags[tag_type].append(text.unescape(text.unquote(tag_name)))
+        for type, values in tags.items():
+            post["tags_" + type] = values
+            post["tag_string_" + type] = " ".join(values)
+
     def _notes(self, post, page):
         if post.get("has_notes"):
             post["notes"] = self.api.notes(post["id"])

test/results/sankaku.py

@@ -179,6 +179,48 @@ __tests__ = (
     "tags_general": list,
 },
 
+{
+    "#url"     : "https://sankaku.app/posts/y0abGlDOr2o",
+    "#comment" : "new tag categories (#7333)",
+    "#category": ("booru", "sankaku", "post"),
+    "#class"   : sankaku.SankakuPostExtractor,
+    "#options" : {"tags": "extended"},
+
+    "id": "y0abGlDOr2o",
+    "tags_anatomy": [
+        "brown_eyes",
+        "male",
+        "upper_body",
+    ],
+    "tags_artist": [
+        "bonocho",
+    ],
+    "tags_character": [
+        "batman",
+        "letty_whiterock",
+        "bruce_wayne",
+        "the_joker",
+        "heath_ledger",
+    ],
+    "tags_copyright": [
+        "batman_(series)",
+        "the_dark_knight",
+    ],
+    "tags_fashion": [
+        "black_bodysuit",
+        "bodysuit",
+        "clothing",
+        "collared_shirt",
+        "facepaint",
+        "pink_shirt",
+        "shirt",
+        "wing_collar",
+    ],
+    "tags_studio": [
+        "dc_comics",
+    ],
+},
+
 {
     "#url"     : "https://sankaku.app/posts/9PMwlDWjXaB",
     "#comment" : ">100 tags",
@@ -249,8 +291,8 @@ __tests__ = (
         "pink_bow",
         "pink_dress",
         "frilled_skirt",
-        "frilled_sleeves",
         "high_heel_boots",
+        "frilled_sleeves",
         "white_sleeves",
         "red_hairband",
         "center_frills",
@@ -283,6 +325,7 @@ __tests__ = (
         "glass",
         "doll",
         "character_doll",
+        "broken_glass",
         "pink_gemstone",
         "bodily_fluids",
         "tears",
@@ -292,7 +335,6 @@ __tests__ = (
         "2girls",
         "stairs",
         "bow_choker",
-        "broken_glass",
         "button_eyes",
         "chest_jewel",
         "mahou_shoujo_madoka_magica_(anime)",