github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[shimme2] move 'giantessbooru' back into shimmie module (#4373)

Browse Source 
Do the same thing as for 'realbooru' and override 'posts()'
insteadd of using a separate module.
This commit is contained in:
Mike Fährmann
2023-08-18 15:25:28 +02:00
parent 6482f9453b
commit 8dceea3384
3 changed files with 81 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
gallery_dl/extractor/__init__.py
View File

@@ -50,7 +50,6 @@ modules = [
"gelbooru_v01",
"gelbooru_v02",
"gfycat",
"giantessbooru",
"gofile",
"hbrowse",
"hentai2read",

124
gallery_dl/extractor/giantessbooru.py
View File

@@ -1,124 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://giantessbooru.com/"""
from . import shimmie2
from .. import text
BASE_PATTERN = r"(?:https?://)?giantessbooru\.com/(?:index\.php\?q=/?)?"
class GiantessbooruExtractor(shimmie2.Shimmie2Extractor):
"""Base class for giantessbooru extractors"""
category = "giantessbooru"
root = "https://giantessbooru.com"
def _init(self):
self.cookies.set("agreed", "true", domain="giantessbooru.com")
class GiantessbooruTagExtractor(GiantessbooruExtractor):
"""Extractor for giantessbooru posts by tag search"""
subcategory = "tag"
directory_fmt = ("{category}", "{search_tags}")
pattern = BASE_PATTERN + r"post/list/([^/?#]+)(?:/(\d+))?"
test = (
("https://giantessbooru.com/index.php?q=/post/list/drawing/1", {
"pattern": r"https://giantessbooru\.com/index\.php"
r"\?q=/image/\d+\.jpg",
"range": "1-100",
"count": 100,
}),
("https://giantessbooru.com/post/list/drawing/1"),
)
def __init__(self, match):
GiantessbooruExtractor.__init__(self, match)
self.tags, self.page = match.groups()
def metadata(self):
return {"search_tags": self.tags}
def posts(self):
pnum = text.parse_int(self.page, 1)
file_url_fmt = (self.root + "/index.php?q=/image/{}.jpg").format
while True:
url = "{}/index.php?q=/post/list/{}/{}".format(
self.root, self.tags, pnum)
extr = text.extract_from(self.request(url).text)
while True:
pid = extr('href="./index.php?q=/post/view/', '&')
if not pid:
break
tags, dimensions, size = extr('title="', '"').split(" // ")
width, _, height = dimensions.partition("x")
yield {
"file_url": file_url_fmt(pid),
"id": pid,
"md5": "",
"tags": tags,
"width": width,
"height": height,
"size": text.parse_bytes(size[:-1]),
}
pnum += 1
if not extr('/{}">{}<'.format(pnum, pnum), ">"):
return
class GiantessbooruPostExtractor(GiantessbooruExtractor):
"""Extractor for single giantessbooru posts"""
subcategory = "post"
pattern = BASE_PATTERN + r"post/view/(\d+)"
test = (
("https://giantessbooru.com/index.php?q=/post/view/41", {
"pattern": r"https://giantessbooru\.com/index\.php"
r"\?q=/image/41\.jpg",
"content": "79115ed309d1f4e82e7bead6948760e889139c91",
"keyword": {
"extension": "jpg",
"file_url": "https://giantessbooru.com/index.php"
"?q=/image/41.jpg",
"filename": "41",
"height": 0,
"id": 41,
"md5": "",
"size": 0,
"tags": "anime bare_midriff color drawing gentle giantess "
"karbo looking_at_tinies negeyari outdoors smiling "
"snake_girl white_hair",
"width": 1387,
},
}),
("https://giantessbooru.com/post/view/41"),
)
def __init__(self, match):
GiantessbooruExtractor.__init__(self, match)
self.post_id = match.group(1)
def posts(self):
url = "{}/index.php?q=/post/view/{}".format(
self.root, self.post_id)
extr = text.extract_from(self.request(url).text)
return ({
"id" : self.post_id,
"tags" : extr(": ", "<").partition(" - ")[0].rstrip(")"),
"md5" : "",
"file_url": self.root + extr('id="main_image" src=".', '"'),
"width" : extr("orig_width =", ";"),
"height" : 0,
"size" : 0,
},)

89
gallery_dl/extractor/shimmie2.py
View File

@@ -33,7 +33,8 @@ class Shimmie2Extractor(BaseExtractor):
if file_url:
self.file_url_fmt = file_url
self._pid_needle = instance.get("needle")
if self.category == "giantessbooru":
self.posts = self._posts_giantessbooru
def items(self):
data = self.metadata()
@@ -74,6 +75,11 @@ INSTANCES = {
"pattern": r"loudbooru\.com",
"cookies": {"ui-tnc-agreed": "true"},
},
"giantessbooru": {
"root": "https://giantessbooru.com",
"pattern": r"giantessbooru\.com",
"cookies": {"agreed": "true"},
},
"tentaclerape": {
"root": "https://tentaclerape.net",
"pattern": r"tentaclerape\.net",
@@ -104,6 +110,13 @@ class Shimmie2TagExtractor(Shimmie2Extractor):
"range": "1-100",
"count": 100,
}),
("https://giantessbooru.com/index.php?q=/post/list/drawing/1", {
"pattern": r"https://giantessbooru\.com/index\.php"
r"\?q=/image/\d+\.jpg",
"range": "1-100",
"count": 100,
}),
("https://giantessbooru.com/post/list/drawing/1"),
("https://tentaclerape.net/post/list/comic/1", {
"pattern": r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+",
"range": "1-100",
@@ -131,12 +144,6 @@ class Shimmie2TagExtractor(Shimmie2Extractor):
pnum = text.parse_int(self.page, 1)
file_url_fmt = self.file_url_fmt.format
if self._pid_needle:
pid_begin, pid_end = self._pid_needle
else:
pid_begin = "href='/post/view/"
pid_end = "?"
init = True
mime = ""
@@ -156,7 +163,7 @@ class Shimmie2TagExtractor(Shimmie2Extractor):
if has_pid:
pid = extr("data-post-id='", "'")
else:
pid = extr(pid_begin, pid_end)
pid = extr("href='/post/view/", "?")
if not pid:
break
@@ -182,6 +189,37 @@ class Shimmie2TagExtractor(Shimmie2Extractor):
if not extr("/{}'>{}<".format(pnum, pnum), ">"):
return
def _posts_giantessbooru(self):
pnum = text.parse_int(self.page, 1)
file_url_fmt = (self.root + "/index.php?q=/image/{}.jpg").format
while True:
url = "{}/index.php?q=/post/list/{}/{}".format(
self.root, self.tags, pnum)
extr = text.extract_from(self.request(url).text)
while True:
pid = extr('href="./index.php?q=/post/view/', '&')
if not pid:
break
tags, dimensions, size = extr('title="', '"').split(" // ")
width, _, height = dimensions.partition("x")
yield {
"file_url": file_url_fmt(pid),
"id": pid,
"md5": "",
"tags": tags,
"width": width,
"height": height,
"size": text.parse_bytes(size[:-1]),
}
pnum += 1
if not extr('/{}">{}<'.format(pnum, pnum), ">"):
return
class Shimmie2PostExtractor(Shimmie2Extractor):
"""Extractor for single shimmie2 posts"""
@@ -232,6 +270,26 @@ class Shimmie2PostExtractor(Shimmie2Extractor):
"width": 1078,
},
}),
("https://giantessbooru.com/index.php?q=/post/view/41", {
"pattern": r"https://giantessbooru\.com/index\.php"
r"\?q=/image/41\.jpg",
"content": "79115ed309d1f4e82e7bead6948760e889139c91",
"keyword": {
"extension": "jpg",
"file_url": "https://giantessbooru.com/index.php"
"?q=/image/41.jpg",
"filename": "41",
"height": 0,
"id": 41,
"md5": "",
"size": 0,
"tags": "anime bare_midriff color drawing gentle giantess "
"karbo looking_at_tinies negeyari outdoors smiling "
"snake_girl white_hair",
"width": 1387,
},
}),
("https://giantessbooru.com/post/view/41"),
("https://tentaclerape.net/post/view/10", {
"pattern": r"https://tentaclerape\.net/\./index\.php"
r"\?q=/image/10\.jpg",
@@ -302,3 +360,18 @@ class Shimmie2PostExtractor(Shimmie2Extractor):
post["md5"] = text.extr(post["file_url"], "/_images/", "/")
return (post,)
def _posts_giantessbooru(self):
url = "{}/index.php?q=/post/view/{}".format(
self.root, self.post_id)
extr = text.extract_from(self.request(url).text)
return ({
"id" : self.post_id,
"tags" : extr(": ", "<").partition(" - ")[0].rstrip(")"),
"md5" : "",
"file_url": self.root + extr('id="main_image" src=".', '"'),
"width" : extr("orig_width =", ";"),
"height" : 0,
"size" : 0,
},)