github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[e621] derive from Danbooru extractors (#651)

Browse Source 
- use extractor implementations from 'danbooru'
- use "page": "b[ID]" to paginate over results instead of
  "tags": "id:<[ID]", avoiding infinite loops with certain
  post orders
- bump User-Agent version
This commit is contained in:
Mike Fährmann
2020-03-22 20:59:17 +01:00
parent 9b39e1cd7e
commit ad2efa8509
2 changed files with 18 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
gallery_dl/extractor/danbooru.py
View File

@@ -30,7 +30,7 @@ class DanbooruExtractor(SharedConfigMixin, Extractor):
per_page = 200
def __init__(self, match):
Extractor.__init__(self, match)
super().__init__(match)
self.root = "https://{}.donmai.us".format(match.group(1))
self.ugoira = self.config("ugoira", True)
self.params = {}
@@ -83,6 +83,8 @@ class DanbooruExtractor(SharedConfigMixin, Extractor):
while True:
posts = self.request(url, params=params).json()
if "posts" in posts:
posts = posts["posts"]
yield from posts
if len(posts) < self.per_page:
@@ -114,7 +116,7 @@ class DanbooruTagExtractor(DanbooruExtractor):
)
def __init__(self, match):
DanbooruExtractor.__init__(self, match)
super().__init__(match)
self.params["tags"] = text.unquote(match.group(2).replace("+", " "))
def metadata(self):
@@ -132,7 +134,7 @@ class DanbooruPoolExtractor(DanbooruExtractor):
})
def __init__(self, match):
DanbooruExtractor.__init__(self, match)
super().__init__(match)
self.pool_id = match.group(2)
self.params["tags"] = "pool:" + self.pool_id
@@ -160,12 +162,13 @@ class DanbooruPostExtractor(DanbooruExtractor):
)
def __init__(self, match):
DanbooruExtractor.__init__(self, match)
super().__init__(match)
self.post_id = match.group(2)
def posts(self):
url = "{}/posts/{}.json".format(self.root, self.post_id)
return (self.request(url).json(),)
post = self.request(url).json()
return (post["post"] if "post" in post else post,)
class DanbooruPopularExtractor(DanbooruExtractor):
@@ -184,7 +187,7 @@ class DanbooruPopularExtractor(DanbooruExtractor):
)
def __init__(self, match):
DanbooruExtractor.__init__(self, match)
super().__init__(match)
self.params.update(text.parse_query(match.group(2)))
def metadata(self):

104
gallery_dl/extractor/e621.py
View File

@@ -8,18 +8,16 @@
"""Extractors for https://e621.net/"""
from .common import Extractor, Message, SharedConfigMixin
from .. import text
import datetime
from .common import Extractor, Message
from . import danbooru
import time
BASE_PATTERN = r"(?:https?://)?e(621|926)\.net"
class E621Extractor(SharedConfigMixin, Extractor):
class E621Extractor(danbooru.DanbooruExtractor):
"""Base class for e621 extractors"""
basecategory = "booru"
category = "e621"
filename_fmt = "{category}_{id}_{file[md5]}.{extension}"
page_limit = 750
@@ -28,14 +26,8 @@ class E621Extractor(SharedConfigMixin, Extractor):
_last_request = 0
def __init__(self, match):
Extractor.__init__(self, match)
super().__init__(match)
self.root = "https://e{}.net".format(match.group(1))
self.params = {}
username, api_key = self._get_auth_info()
if username:
self.log.debug("Using HTTP Basic Auth for user '%s'", username)
self.session.auth = (username, api_key)
def request(self, url, **kwargs):
diff = time.time() - E621Extractor._last_request
@@ -43,7 +35,7 @@ class E621Extractor(SharedConfigMixin, Extractor):
delay = 1.0 - diff
self.log.debug("Sleeping for %s seconds", delay)
time.sleep(delay)
kwargs["headers"] = {"User-Agent": "gallery-dl/1.13.0 (by mikf)"}
kwargs["headers"] = {"User-Agent": "gallery-dl/1.14.0 (by mikf)"}
response = Extractor.request(self, url, **kwargs)
E621Extractor._last_request = time.time()
return response
@@ -64,31 +56,9 @@ class E621Extractor(SharedConfigMixin, Extractor):
yield Message.Directory, post
yield Message.Url, file["url"], post
def metadata(self):
return {}
def posts(self):
return self._pagination(self.root + "/posts.json")
def _pagination(self, url):
params = self.params.copy()
params["limit"] = self.per_page
tags = params.get("tags", "")
while True:
posts = self.request(url, params=params).json()["posts"]
yield from posts
if len(posts) < self.per_page:
return
params["tags"] = "id:<{} {}".format(posts[-1]["id"], tags)
class E621TagExtractor(E621Extractor):
class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor):
"""Extractor for e621 posts from tag searches"""
subcategory = "tag"
directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}"
pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)"
test = (
("https://e621.net/posts?tags=anry", {
@@ -100,19 +70,9 @@ class E621TagExtractor(E621Extractor):
("https://e621.net/post?tags=anry"),
)
def __init__(self, match):
E621Extractor.__init__(self, match)
self.params["tags"] = text.unquote(match.group(2).replace("+", " "))
def metadata(self):
return {"search_tags": self.params["tags"]}
class E621PoolExtractor(E621Extractor):
class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor):
"""Extractor for e621 pools"""
subcategory = "pool"
directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}")
archive_fmt = "p_{pool[id]}_{id}"
pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)"
test = (
("https://e621.net/pools/73", {
@@ -122,23 +82,9 @@ class E621PoolExtractor(E621Extractor):
("https://e621.net/pool/show/73"),
)
def __init__(self, match):
E621Extractor.__init__(self, match)
self.pool_id = match.group(2)
self.params["tags"] = "pool:" + self.pool_id
def metadata(self):
url = "{}/pools/{}.json".format(self.root, self.pool_id)
pool = self.request(url).json()
pool["name"] = pool["name"].replace("_", " ")
del pool["post_ids"]
return {"pool": pool}
class E621PostExtractor(E621Extractor):
class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor):
"""Extractor for single e621 posts"""
subcategory = "post"
archive_fmt = "{id}"
pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)"
test = (
("https://e621.net/posts/535", {
@@ -148,20 +94,9 @@ class E621PostExtractor(E621Extractor):
("https://e621.net/post/show/535"),
)
def __init__(self, match):
E621Extractor.__init__(self, match)
self.post_id = match.group(2)
def posts(self):
url = "{}/posts/{}.json".format(self.root, self.post_id)
return (self.request(url).json()["post"],)
class E621PopularExtractor(E621Extractor):
class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor):
"""Extractor for popular images from e621"""
subcategory = "popular"
directory_fmt = ("{category}", "popular", "{scale}", "{date}")
archive_fmt = "P_{scale[0]}_{date}_{id}"
pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?"
test = (
("https://e621.net/explore/posts/popular"),
@@ -171,24 +106,3 @@ class E621PopularExtractor(E621Extractor):
"count": ">= 70",
})
)
def __init__(self, match):
E621Extractor.__init__(self, match)
self.params.update(text.parse_query(match.group(2)))
def metadata(self):
scale = self.params.get("scale", "day")
date = self.params.get("date") or datetime.date.today().isoformat()
date = date[:10]
if scale == "week":
date = datetime.date.fromisoformat(date)
date = (date - datetime.timedelta(days=date.weekday())).isoformat()
elif scale == "month":
date = date[:-3]
return {"date": date, "scale": scale}
def posts(self):
url = self.root + "/explore/posts/popular.json"
return self._pagination(url)