[realbooru] fix 'tags' for video posts (#8455)

* fix realbooru tags for video posts
* fix lines too long
* combine 'tags' and 'tags_<category>' extraction
    - extract tag categories independent of 'tags' option
* add 'video' & 'animated GIF' tests

---------

Co-authored-by: Mike Fährmann <mike_faehrmann@web.de>

gallery_dl/extractor/realbooru.py

@@ -28,18 +28,30 @@ class RealbooruExtractor(booru.BooruExtractor):
         extr('class="container"', '>')
 
         post = {
-            "_html"     : page,
             "id"        : post_id,
             "rating"    : "e" if rating == "adult" else (rating or "?")[0],
-            "tags"      : text.unescape(extr(' alt="', '"')),
             "file_url"  : extr('src="', '"'),
             "created_at": extr(">Posted at ", " by "),
             "uploader"  : extr(">", "<"),
             "score"     : extr('">', "<"),
+            "tags"      : extr('<br />', "</div>"),
             "title"     : extr('id="title" style="width: 100%;" value="', '"'),
             "source"    : extr('d="source" style="width: 100%;" value="', '"'),
         }
 
+        tags_container = post["tags"]
+        tags = []
+        tags_categories = collections.defaultdict(list)
+        pattern = text.re(r'<a class="(?:tag-type-)?([^"]+).*?;tags=([^"&]+)')
+        for tag_type, tag_name in pattern.findall(tags_container):
+            tag = text.unescape(text.unquote(tag_name))
+            tags.append(tag)
+            tags_categories[tag_type].append(tag)
+        for key, value in tags_categories.items():
+            post[f"tags_{key}"] = ", ".join(value)
+        tags.sort()
+
+        post["tags"] = ", ".join(tags)
         post["md5"] = post["file_url"].rpartition("/")[2].partition(".")[0]
         return post
 
@@ -66,16 +78,6 @@ class RealbooruExtractor(booru.BooruExtractor):
                 return
             params["pid"] += self.per_page
 
-    def _tags(self, post, _):
-        page = post["_html"]
-        tag_container = text.extr(page, 'id="tagLink"', '</div>')
-        tags = collections.defaultdict(list)
-        pattern = text.re(r'<a class="(?:tag-type-)?([^"]+).*?;tags=([^"&]+)')
-        for tag_type, tag_name in pattern.findall(tag_container):
-            tags[tag_type].append(text.unescape(text.unquote(tag_name)))
-        for key, value in tags.items():
-            post["tags_" + key] = " ".join(value)
-
 
 class RealbooruTagExtractor(RealbooruExtractor):
     subcategory = "tag"

test/results/realbooru.py

@@ -38,7 +38,6 @@ __tests__ = (
     "#comment" : "regular post",
     "#category": ("booru", "realbooru", "post"),
     "#class"   : realbooru.RealbooruPostExtractor,
-    "#options"     : {"tags": True},
     "#results"     : "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg",
     "#sha1_content": "f6213e6f25c3cb9e3cfefa6d4b3a78e44b9dea5b",
 
@@ -51,9 +50,10 @@ __tests__ = (
     "rating"        : "e",
     "score"         : r"re:\d+",
     "source"        : "https://www.instagram.com/p/CwAO1UyJBnw",
-    "tags"          : "1girl asian bikini black_hair breasts cleavage female female_only floral_print instagram japanese kurita_emi large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo",
+    "tags"          : "1girl, asian, bikini, black_hair, breasts, cleavage, female, female_only, floral_print, instagram, japanese, kurita_emi, large_breasts, looking_at_viewer, navel, sauna, short_hair, side-tie_bikini, sitting, solo",
     "tags_copyright": "instagram",
-    "tags_general"  : "1girl asian bikini black_hair breasts cleavage female female_only floral_print japanese large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo",
+    "tags_general"  : "1girl, asian, bikini, black_hair, breasts, cleavage, female, female_only, floral_print, large_breasts, looking_at_viewer, navel, sauna, short_hair, side-tie_bikini, sitting, solo",
+    "tags_metadata" : "japanese",
     "tags_model"    : "kurita_emi",
 },
 
@@ -65,4 +65,55 @@ __tests__ = (
     "#sha1_content": "4a7424810f5f846c161b5d3b7c8b0a85a03368c8",
 },
 
+{
+    "#url"     : "https://realbooru.com/index.php?page=post&s=view&id=825911",
+    "#comment" : "animated GIF",
+    "#category": ("booru", "realbooru", "post"),
+    "#class"   : realbooru.RealbooruPostExtractor,
+    "#results" : "https://realbooru.com//images/a3/9b/a39bfe5ea07f132db486e118b598c66c.gif",
+
+    "created_at"   : "Feb, 19 2023",
+    "date"         : "dt:2023-02-19 00:00:00",
+    "extension"    : "gif",
+    "file_url"     : "https://realbooru.com//images/a3/9b/a39bfe5ea07f132db486e118b598c66c.gif",
+    "filename"     : "a39bfe5ea07f132db486e118b598c66c",
+    "id"           : "825911",
+    "md5"          : "a39bfe5ea07f132db486e118b598c66c",
+    "rating"       : "e",
+    "score"        : r"re:\d+",
+    "source"       : "",
+    "title"        : "",
+    "uploader"     : "freakyguro",
+    "tags"         : "1girl, amateur, animated, animated_gif, asian, ass, ass_shake, biting_lip, breasts, brown_hair, butt, cute, fringe, girlfriend, japanese, japanese_(nationality), kitchen, looking_at_viewer, looking_back, medium_breasts, obokozu, panties, petite, seductive_eyes, seductive_smile, solo, underwear",
+    "tags_general" : "1girl, asian, ass, ass_shake, biting_lip, breasts, brown_hair, butt, cute, fringe, girlfriend, japanese_(nationality), kitchen, looking_at_viewer, looking_back, medium_breasts, panties, petite, seductive_eyes, seductive_smile, solo, underwear",
+    "tags_metadata": "amateur, animated, animated_gif, japanese",
+    "tags_model"   : "obokozu",
+},
+
+{
+    "#url"     : "https://realbooru.com/index.php?page=post&s=view&id=751237",
+    "#comment" : "video",
+    "#category": ("booru", "realbooru", "post"),
+    "#class"   : realbooru.RealbooruPostExtractor,
+    "#results" : "https://video-cdn.realbooru.com//images/3c/34/3c343385126c90a7fdbccc58d342a511.mp4",
+
+    "created_at"    : "Dec, 07 2020",
+    "date"          : "dt:2020-12-07 00:00:00",
+    "extension"     : "mp4",
+    "file_url"      : "https://video-cdn.realbooru.com//images/3c/34/3c343385126c90a7fdbccc58d342a511.mp4",
+    "filename"      : "3c343385126c90a7fdbccc58d342a511",
+    "id"            : "751237",
+    "md5"           : "3c343385126c90a7fdbccc58d342a511",
+    "rating"        : "e",
+    "score"         : r"re:\d+",
+    "source"        : "",
+    "title"         : "",
+    "uploader"      : "Rock 7800",
+    "tags"          : "1girl, animated, anri_okita, asian, breasts, brown_hair, brunette, dress, dress_pull, female, female_only, female_solo, huge_breasts, human_only, japanese, lipstick, makeup, model, mostly_clothed, nadine-j, nipples, no_bra, no_sound, topless, undressing, webm",
+    "tags_copyright": "nadine-j",
+    "tags_general"  : "1girl, asian, breasts, brown_hair, brunette, dress, dress_pull, female, female_only, female_solo, huge_breasts, human_only, lipstick, makeup, mostly_clothed, nipples, no_bra, topless, undressing",
+    "tags_metadata" : "animated, japanese, model, no_sound, webm",
+    "tags_model"    : "anri_okita",
+},
+
 )