[realbooru] fix extraction (#6543)
- extract data from HTML pages since API is no longer usable - move code into its own separate 'realbooru' module
This commit is contained in:
Mike Fährmann
@@ -769,6 +769,12 @@ Consider all listed sites to potentially be NSFW.
|
||||
<td>Comic Issues, Comics</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Realbooru</td>
|
||||
<td>https://realbooru.com/</td>
|
||||
<td>Favorites, Pools, Posts, Tag Searches</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Reddit</td>
|
||||
<td>https://www.reddit.com/</td>
|
||||
@@ -1213,12 +1219,6 @@ Consider all listed sites to potentially be NSFW.
|
||||
<tr>
|
||||
<td colspan="4"><strong>Gelbooru Beta 0.2</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Realbooru</td>
|
||||
<td>https://realbooru.com/</td>
|
||||
<td>Favorites, Pools, Posts, Tag Searches</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Rule 34</td>
|
||||
<td>https://rule34.xxx/</td>
|
||||
|
||||
@@ -140,6 +140,7 @@ modules = [
|
||||
"postmill",
|
||||
"reactor",
|
||||
"readcomiconline",
|
||||
"realbooru",
|
||||
"reddit",
|
||||
"redgifs",
|
||||
"rule34us",
|
||||
|
||||
@@ -24,10 +24,6 @@ class GelbooruV02Extractor(booru.BooruExtractor):
|
||||
self.user_id = self.config("user-id")
|
||||
self.root_api = self.config_instance("root-api") or self.root
|
||||
|
||||
if self.category == "realbooru":
|
||||
self.items = self._items_realbooru
|
||||
self._tags = self._tags_realbooru
|
||||
|
||||
def _api_request(self, params):
|
||||
url = self.root_api + "/index.php?page=dapi&s=post&q=index"
|
||||
return ElementTree.fromstring(self.request(url, params=params).text)
|
||||
@@ -137,59 +133,8 @@ class GelbooruV02Extractor(booru.BooruExtractor):
|
||||
"body" : text.unescape(text.remove_html(extr(">", "</div>"))),
|
||||
})
|
||||
|
||||
def _file_url_realbooru(self, post):
|
||||
url = post["file_url"]
|
||||
md5 = post["md5"]
|
||||
if md5 not in post["preview_url"] or url.count("/") == 5:
|
||||
url = "{}/images/{}/{}/{}.{}".format(
|
||||
self.root, md5[0:2], md5[2:4], md5, url.rpartition(".")[2])
|
||||
return url
|
||||
|
||||
def _items_realbooru(self):
|
||||
from .common import Message
|
||||
data = self.metadata()
|
||||
|
||||
for post in self.posts():
|
||||
try:
|
||||
html = self._html(post)
|
||||
fallback = post["file_url"]
|
||||
url = post["file_url"] = text.rextract(
|
||||
html, 'href="', '"', html.index(">Original<"))[0]
|
||||
except Exception:
|
||||
self.log.debug("Unable to fetch download URL for post %s "
|
||||
"(md5: %s)", post.get("id"), post.get("md5"))
|
||||
continue
|
||||
|
||||
text.nameext_from_url(url, post)
|
||||
post.update(data)
|
||||
self._prepare(post)
|
||||
self._tags(post, html)
|
||||
|
||||
path = url.rpartition("/")[0]
|
||||
post["_fallback"] = (
|
||||
"{}/{}.{}".format(path, post["md5"], post["extension"]),
|
||||
fallback,
|
||||
)
|
||||
|
||||
yield Message.Directory, post
|
||||
yield Message.Url, url, post
|
||||
|
||||
def _tags_realbooru(self, post, page):
|
||||
tag_container = text.extr(page, 'id="tagLink"', '</div>')
|
||||
tags = collections.defaultdict(list)
|
||||
pattern = re.compile(
|
||||
r'<a class="(?:tag-type-)?([^"]+).*?;tags=([^"&]+)')
|
||||
for tag_type, tag_name in pattern.findall(tag_container):
|
||||
tags[tag_type].append(text.unescape(text.unquote(tag_name)))
|
||||
for key, value in tags.items():
|
||||
post["tags_" + key] = " ".join(value)
|
||||
|
||||
|
||||
BASE_PATTERN = GelbooruV02Extractor.update({
|
||||
"realbooru": {
|
||||
"root": "https://realbooru.com",
|
||||
"pattern": r"realbooru\.com",
|
||||
},
|
||||
"rule34": {
|
||||
"root": "https://rule34.xxx",
|
||||
"root-api": "https://api.rule34.xxx",
|
||||
|
||||
157
gallery_dl/extractor/realbooru.py
Normal file
157
gallery_dl/extractor/realbooru.py
Normal file
@@ -0,0 +1,157 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2024 Mike Fährmann
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
"""Extractors for https://realbooru.com/"""
|
||||
|
||||
from . import booru
|
||||
from .. import text, util
|
||||
import collections
|
||||
import re
|
||||
|
||||
BASE_PATTERN = r"(?:https?://)?realbooru\.com"
|
||||
|
||||
|
||||
class RealbooruExtractor(booru.BooruExtractor):
|
||||
basecategory = "booru"
|
||||
category = "realbooru"
|
||||
root = "https://realbooru.com"
|
||||
|
||||
def _parse_post(self, post_id):
|
||||
url = "{}/index.php?page=post&s=view&id={}".format(
|
||||
self.root, post_id)
|
||||
page = self.request(url).text
|
||||
extr = text.extract_from(page)
|
||||
rating = extr('name="rating" content="', '"')
|
||||
extr('class="container"', '>')
|
||||
|
||||
post = {
|
||||
"_html" : page,
|
||||
"id" : post_id,
|
||||
"rating" : "e" if rating == "adult" else (rating or "?")[0],
|
||||
"tags" : text.unescape(extr(' alt="', '"')),
|
||||
"file_url" : extr('src="', '"'),
|
||||
"created_at": extr(">Posted at ", " by "),
|
||||
"uploader" : extr(">", "<"),
|
||||
"score" : extr('">', "<"),
|
||||
"title" : extr('id="title" style="width: 100%;" value="', '"'),
|
||||
"source" : extr('d="source" style="width: 100%;" value="', '"'),
|
||||
}
|
||||
|
||||
post["md5"] = post["file_url"].rpartition("/")[2].partition(".")[0]
|
||||
return post
|
||||
|
||||
def skip(self, num):
|
||||
self.page_start += num
|
||||
return num
|
||||
|
||||
def _prepare(self, post):
|
||||
post["date"] = text.parse_datetime(post["created_at"], "%b, %d %Y")
|
||||
|
||||
def _pagination(self, params, begin, end):
|
||||
url = self.root + "/index.php"
|
||||
params["pid"] = self.page_start
|
||||
|
||||
while True:
|
||||
page = self.request(url, params=params).text
|
||||
|
||||
cnt = 0
|
||||
for post_id in text.extract_iter(page, begin, end):
|
||||
cnt += 1
|
||||
yield self._parse_post(post_id)
|
||||
|
||||
if cnt < self.per_page:
|
||||
return
|
||||
params["pid"] += self.per_page
|
||||
|
||||
def _tags(self, post, _):
|
||||
page = post["_html"]
|
||||
tag_container = text.extr(page, 'id="tagLink"', '</div>')
|
||||
tags = collections.defaultdict(list)
|
||||
pattern = re.compile(
|
||||
r'<a class="(?:tag-type-)?([^"]+).*?;tags=([^"&]+)')
|
||||
for tag_type, tag_name in pattern.findall(tag_container):
|
||||
tags[tag_type].append(text.unescape(text.unquote(tag_name)))
|
||||
for key, value in tags.items():
|
||||
post["tags_" + key] = " ".join(value)
|
||||
|
||||
|
||||
class RealbooruTagExtractor(RealbooruExtractor):
|
||||
subcategory = "tag"
|
||||
directory_fmt = ("{category}", "{search_tags}")
|
||||
archive_fmt = "t_{search_tags}_{id}"
|
||||
per_page = 42
|
||||
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]*)"
|
||||
example = "https://realbooru.com/index.php?page=post&s=list&tags=TAG"
|
||||
|
||||
def metadata(self):
|
||||
self.tags = text.unquote(self.groups[0].replace("+", " "))
|
||||
return {"search_tags": self.tags}
|
||||
|
||||
def posts(self):
|
||||
return self._pagination({
|
||||
"page": "post",
|
||||
"s" : "list",
|
||||
"tags": self.tags,
|
||||
}, '<a id="p', '"')
|
||||
|
||||
|
||||
class RealbooruFavoriteExtractor(RealbooruExtractor):
|
||||
subcategory = "favorite"
|
||||
directory_fmt = ("{category}", "favorites", "{favorite_id}")
|
||||
archive_fmt = "f_{favorite_id}_{id}"
|
||||
per_page = 50
|
||||
pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)"
|
||||
example = "https://realbooru.com/index.php?page=favorites&s=view&id=12345"
|
||||
|
||||
def metadata(self):
|
||||
return {"favorite_id": text.parse_int(self.groups[0])}
|
||||
|
||||
def posts(self):
|
||||
return self._pagination({
|
||||
"page": "favorites",
|
||||
"s" : "view",
|
||||
"id" : self.groups[0],
|
||||
}, '" id="p', '"')
|
||||
|
||||
|
||||
class RealbooruPoolExtractor(RealbooruExtractor):
|
||||
subcategory = "pool"
|
||||
directory_fmt = ("{category}", "pool", "{pool} {pool_name}")
|
||||
archive_fmt = "p_{pool}_{id}"
|
||||
pattern = BASE_PATTERN + r"/index\.php\?page=pool&s=show&id=(\d+)"
|
||||
example = "https://realbooru.com/index.php?page=pool&s=show&id=12345"
|
||||
|
||||
def metadata(self):
|
||||
pool_id = self.groups[0]
|
||||
url = "{}/index.php?page=pool&s=show&id={}".format(self.root, pool_id)
|
||||
page = self.request(url).text
|
||||
|
||||
name, pos = text.extract(page, "<h4>Pool: ", "</h4>")
|
||||
self.post_ids = text.extract_iter(
|
||||
page, 'class="thumb" id="p', '"', pos)
|
||||
|
||||
return {
|
||||
"pool": text.parse_int(pool_id),
|
||||
"pool_name": text.unescape(name),
|
||||
}
|
||||
|
||||
def posts(self):
|
||||
return map(
|
||||
self._parse_post,
|
||||
util.advance(self.post_ids, self.page_start)
|
||||
)
|
||||
|
||||
|
||||
class RealbooruPostExtractor(RealbooruExtractor):
|
||||
subcategory = "post"
|
||||
archive_fmt = "{id}"
|
||||
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)"
|
||||
example = "https://realbooru.com/index.php?page=post&s=view&id=12345"
|
||||
|
||||
def posts(self):
|
||||
return (self._parse_post(self.groups[0]),)
|
||||
@@ -4,81 +4,64 @@
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
from gallery_dl.extractor import gelbooru_v02
|
||||
from gallery_dl.extractor import realbooru
|
||||
|
||||
|
||||
__tests__ = (
|
||||
{
|
||||
"#url" : "https://realbooru.com/index.php?page=post&s=list&tags=wine",
|
||||
"#category": ("gelbooru_v02", "realbooru", "tag"),
|
||||
"#class" : gelbooru_v02.GelbooruV02TagExtractor,
|
||||
"#category": ("booru", "realbooru", "tag"),
|
||||
"#class" : realbooru.RealbooruTagExtractor,
|
||||
"#count" : ">= 64",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://realbooru.com/index.php?page=pool&s=show&id=1",
|
||||
"#category": ("gelbooru_v02", "realbooru", "pool"),
|
||||
"#class" : gelbooru_v02.GelbooruV02PoolExtractor,
|
||||
"#category": ("booru", "realbooru", "pool"),
|
||||
"#class" : realbooru.RealbooruPoolExtractor,
|
||||
"#urls" : (
|
||||
"https://realbooru.com//images/bf/d6/bfd682f338691e5254de796040fcba21.webm",
|
||||
"https://realbooru.com//images/cb/7d/cb7d921673ba99f688031ac554777695.webm",
|
||||
"https://realbooru.com//images/9e/14/9e140edc1cb2e4cc734ba5bdc4870955.webm",
|
||||
"https://realbooru.com//images/bf/d6/bfd682f338691e5254de796040fcba21.mp4",
|
||||
"https://realbooru.com//images/cb/7d/cb7d921673ba99f688031ac554777695.mp4",
|
||||
"https://realbooru.com//images/9e/14/9e140edc1cb2e4cc734ba5bdc4870955.mp4",
|
||||
),
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://realbooru.com/index.php?page=favorites&s=view&id=274",
|
||||
"#category": ("gelbooru_v02", "realbooru", "favorite"),
|
||||
"#class" : gelbooru_v02.GelbooruV02FavoriteExtractor,
|
||||
"#urls" : (
|
||||
"https://realbooru.com//images/20/3e/0c2c4d8c978355c053602dc963eb13136c1614c1.jpeg",
|
||||
),
|
||||
"#category": ("booru", "realbooru", "favorite"),
|
||||
"#class" : realbooru.RealbooruFavoriteExtractor,
|
||||
"#urls" : "https://realbooru.com//images/20/3e/203eefb39f54de049e30ff788a022ac7.jpeg",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://realbooru.com/index.php?page=post&s=view&id=862054",
|
||||
"#comment" : "regular post",
|
||||
"#category": ("gelbooru_v02", "realbooru", "post"),
|
||||
"#class" : gelbooru_v02.GelbooruV02PostExtractor,
|
||||
"#category": ("booru", "realbooru", "post"),
|
||||
"#class" : realbooru.RealbooruPostExtractor,
|
||||
"#options" : {"tags": True},
|
||||
"#urls" : "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg",
|
||||
"#sha1_content": "f6213e6f25c3cb9e3cfefa6d4b3a78e44b9dea5b",
|
||||
|
||||
"change" : "1705562002",
|
||||
"created_at" : "Thu Jan 18 01:12:50 -0600 2024",
|
||||
"creator_id" : "32011",
|
||||
"date" : "dt:2024-01-18 07:12:50",
|
||||
"created_at" : "Jan, 18 2024",
|
||||
"date" : "dt:2024-01-18 00:00:00",
|
||||
"file_url" : "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg",
|
||||
"filename" : "8a345820da989637c21ac013d522bf69",
|
||||
"has_children" : "false",
|
||||
"has_comments" : "false",
|
||||
"has_notes" : "false",
|
||||
"height" : "1800",
|
||||
"id" : "862054",
|
||||
"md5" : "8a345820da989637c21ac013d522bf69",
|
||||
"parent_id" : "",
|
||||
"preview_height": "150",
|
||||
"preview_url" : "https://realbooru.com/thumbnails/8a/34/thumbnail_8a345820da989637c21ac013d522bf69.jpg",
|
||||
"preview_width" : "120",
|
||||
"rating" : "e",
|
||||
"sample_height" : "1063",
|
||||
"sample_url" : "https://realbooru.com/samples/8a/34/sample_8a345820da989637c21ac013d522bf69.jpg",
|
||||
"sample_width" : "850",
|
||||
"score" : "",
|
||||
"score" : r"re:\d+",
|
||||
"source" : "https://www.instagram.com/p/CwAO1UyJBnw",
|
||||
"status" : "active",
|
||||
"tags" : "1girl asian bikini black_hair breasts cleavage female female_only floral_print instagram japanese kurita_emi large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo",
|
||||
"tags_copyright": "instagram",
|
||||
"tags_general" : "1girl asian bikini black_hair breasts cleavage female female_only floral_print japanese large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo",
|
||||
"tags_model" : "kurita_emi",
|
||||
"width" : "1440",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://realbooru.com/index.php?page=post&s=view&id=568145",
|
||||
"#comment" : "older post",
|
||||
"#category": ("gelbooru_v02", "realbooru", "post"),
|
||||
"#class" : gelbooru_v02.GelbooruV02PostExtractor,
|
||||
"#category": ("booru", "realbooru", "post"),
|
||||
"#class" : realbooru.RealbooruPostExtractor,
|
||||
"#sha1_content": "4a7424810f5f846c161b5d3b7c8b0a85a03368c8",
|
||||
},
|
||||
|
||||
|
||||
Reference in New Issue
Block a user