github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update 'match.lastindex' usage

Browse Source
This commit is contained in:
Mike Fährmann
2025-06-18 16:58:25 +02:00
parent 41191bb60a
commit b0580aba86
16 changed files with 126 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
gallery_dl/extractor/blogger.py
View File

@@ -102,12 +102,8 @@ class BloggerPostExtractor(BloggerExtractor):
pattern = BASE_PATTERN + r"(/\d\d\d\d/\d\d/[^/?#]+\.html)"
example = "https://BLOG.blogspot.com/1970/01/TITLE.html"
def __init__(self, match):
BloggerExtractor.__init__(self, match)
self.path = match[match.lastindex]
def posts(self, blog):
return (self.api.post_by_path(blog["id"], self.path),)
return (self.api.post_by_path(blog["id"], self.groups[-1]),)
class BloggerBlogExtractor(BloggerExtractor):
@@ -126,16 +122,13 @@ class BloggerSearchExtractor(BloggerExtractor):
pattern = BASE_PATTERN + r"/search/?\?q=([^&#]+)"
example = "https://BLOG.blogspot.com/search?q=QUERY"
def __init__(self, match):
BloggerExtractor.__init__(self, match)
self.query = text.unquote(match[match.lastindex])
def metadata(self):
self.query = query = text.unquote(self.groups[-1])
return {"query": query}
def posts(self, blog):
return self.api.blog_search(blog["id"], self.query)
def metadata(self):
return {"query": self.query}
class BloggerLabelExtractor(BloggerExtractor):
"""Extractor for Blogger posts by label"""
@@ -143,21 +136,18 @@ class BloggerLabelExtractor(BloggerExtractor):
pattern = BASE_PATTERN + r"/search/label/([^/?#]+)"
example = "https://BLOG.blogspot.com/search/label/LABEL"
def __init__(self, match):
BloggerExtractor.__init__(self, match)
self.label = text.unquote(match[match.lastindex])
def metadata(self):
self.label = label = text.unquote(self.groups[-1])
return {"label": label}
def posts(self, blog):
return self.api.blog_posts(blog["id"], self.label)
def metadata(self):
return {"label": self.label}
class BloggerAPI():
"""Minimal interface for the Blogger v3 API
"""Minimal interface for the Blogger API v3
Ref: https://developers.google.com/blogger
https://developers.google.com/blogger
"""
API_KEY = "AIzaSyCN9ax34oMMyM07g_M-5pjeDp_312eITK8"
@@ -166,27 +156,27 @@ class BloggerAPI():
self.api_key = extractor.config("api-key") or self.API_KEY
def blog_by_url(self, url):
return self._call("blogs/byurl", {"url": url}, "blog")
return self._call("/blogs/byurl", {"url": url}, "blog")
def blog_posts(self, blog_id, label=None):
endpoint = "blogs/{}/posts".format(blog_id)
endpoint = f"/blogs/{blog_id}/posts"
params = {"labels": label}
return self._pagination(endpoint, params)
def blog_search(self, blog_id, query):
endpoint = "blogs/{}/posts/search".format(blog_id)
endpoint = f"/blogs/{blog_id}/posts/search"
params = {"q": query}
return self._pagination(endpoint, params)
def post_by_path(self, blog_id, path):
endpoint = "blogs/{}/posts/bypath".format(blog_id)
endpoint = f"/blogs/{blog_id}/posts/bypath"
return self._call(endpoint, {"path": path}, "post")
def _call(self, endpoint, params, notfound=None):
url = "https://www.googleapis.com/blogger/v3/" + endpoint
url = "https://www.googleapis.com/blogger/v3" + endpoint
params["key"] = self.api_key
return self.extractor.request(
url, params=params, notfound=notfound).json()
return self.extractor.request_json(
url, params=params, notfound=notfound)
def _pagination(self, endpoint, params):
while True:

31
gallery_dl/extractor/foolfuuka.py
View File

@@ -184,13 +184,12 @@ class FoolfuukaBoardExtractor(FoolfuukaExtractor):
self.page = self.groups[-1]
def items(self):
index_base = "{}/_/api/chan/index/?board={}&page=".format(
self.root, self.board)
thread_base = "{}/{}/thread/".format(self.root, self.board)
index_base = f"{self.root}/_/api/chan/index/?board={self.board}&page="
thread_base = f"{self.root}/{self.board}/thread/"
page = self.page
for pnum in itertools.count(text.parse_int(page, 1)):
with self.request(index_base + format(pnum)) as response:
with self.request(index_base + str(pnum)) as response:
try:
threads = response.json()
except ValueError:
@@ -270,27 +269,17 @@ class FoolfuukaGalleryExtractor(FoolfuukaExtractor):
pattern = BASE_PATTERN + r"/([^/?#]+)/gallery(?:/(\d+))?"
example = "https://archived.moe/a/gallery"
def __init__(self, match):
FoolfuukaExtractor.__init__(self, match)
board = match[match.lastindex]
if board.isdecimal():
self.board = match[match.lastindex-1]
self.pages = (board,)
else:
self.board = board
self.pages = map(format, itertools.count(1))
def metadata(self):
return {"board": self.board}
self.board = board = self.groups[-2]
return {"board": board}
def posts(self):
base = "{}/_/api/chan/gallery/?board={}&page=".format(
self.root, self.board)
pnum = self.groups[-1]
pages = itertools.count(1) if pnum is None else (pnum,)
base = f"{self.root}/_/api/chan/gallery/?board={self.board}&page="
for page in self.pages:
with self.request(base + page) as response:
posts = response.json()
for pnum in pages:
posts = self.request_json(f"{base}{pnum}")
if not posts:
return
yield from posts

2
gallery_dl/extractor/foolslide.py
View File

@@ -18,7 +18,7 @@ class FoolslideExtractor(BaseExtractor):
def __init__(self, match):
BaseExtractor.__init__(self, match)
self.gallery_url = self.root + match[match.lastindex]
self.gallery_url = self.root + self.groups[-1]
def request(self, url):
return BaseExtractor.request(

27
gallery_dl/extractor/gelbooru_v01.py
View File

@@ -92,16 +92,12 @@ class GelbooruV01TagExtractor(GelbooruV01Extractor):
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)"
example = "https://allgirl.booru.org/index.php?page=post&s=list&tags=TAG"
def __init__(self, match):
GelbooruV01Extractor.__init__(self, match)
self.tags = match[match.lastindex]
def metadata(self):
return {"search_tags": text.unquote(self.tags.replace("+", " "))}
self.tags = tags = self.groups[-1]
return {"search_tags": text.unquote(tags.replace("+", " "))}
def posts(self):
url = "{}/index.php?page=post&s=list&tags={}&pid=".format(
self.root, self.tags)
url = f"{self.root}/index.php?page=post&s=list&tags={self.tags}&pid="
return self._pagination(url, 'class="thumb"><a id="p', '"')
@@ -113,16 +109,13 @@ class GelbooruV01FavoriteExtractor(GelbooruV01Extractor):
pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)"
example = "https://allgirl.booru.org/index.php?page=favorites&s=view&id=1"
def __init__(self, match):
GelbooruV01Extractor.__init__(self, match)
self.favorite_id = match[match.lastindex]
def metadata(self):
return {"favorite_id": text.parse_int(self.favorite_id)}
self.favorite_id = fav_id = self.groups[-1]
return {"favorite_id": text.parse_int(fav_id)}
def posts(self):
url = "{}/index.php?page=favorites&s=view&id={}&pid=".format(
self.root, self.favorite_id)
url = (f"{self.root}/index.php"
f"?page=favorites&s=view&id={self.favorite_id}&pid=")
return self._pagination(url, "posts[", "]")
@@ -132,9 +125,5 @@ class GelbooruV01PostExtractor(GelbooruV01Extractor):
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)"
example = "https://allgirl.booru.org/index.php?page=post&s=view&id=12345"
def __init__(self, match):
GelbooruV01Extractor.__init__(self, match)
self.post_id = match[match.lastindex]
def posts(self):
return (self._parse_post(self.post_id),)
return (self._parse_post(self.groups[-1]),)

30
gallery_dl/extractor/gelbooru_v02.py
View File

@@ -95,8 +95,8 @@ class GelbooruV02Extractor(booru.BooruExtractor):
post["created_at"], "%a %b %d %H:%M:%S %z %Y")
def _html(self, post):
return self.request("{}/index.php?page=post&s=view&id={}".format(
self.root, post["id"])).text
url = f"{self.root}/index.php?page=post&s=view&id={post['id']}"
return self.request(url).text
def _tags(self, post, page):
tag_container = (text.extr(page, '<ul id="tag-', '</ul>') or
@@ -161,13 +161,9 @@ class GelbooruV02TagExtractor(GelbooruV02Extractor):
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]*)"
example = "https://safebooru.org/index.php?page=post&s=list&tags=TAG"
def __init__(self, match):
GelbooruV02Extractor.__init__(self, match)
tags = match[match.lastindex]
self.tags = text.unquote(tags.replace("+", " "))
def metadata(self):
return {"search_tags": self.tags}
self.tags = tags = text.unquote(self.groups[-1].replace("+", " "))
return {"search_tags": tags}
def posts(self):
if self.tags == "all":
@@ -184,7 +180,7 @@ class GelbooruV02PoolExtractor(GelbooruV02Extractor):
def __init__(self, match):
GelbooruV02Extractor.__init__(self, match)
self.pool_id = match[match.lastindex]
self.pool_id = self.groups[-1]
if self.category == "rule34":
self.posts = self._posts_pages
@@ -197,8 +193,7 @@ class GelbooruV02PoolExtractor(GelbooruV02Extractor):
return num
def metadata(self):
url = "{}/index.php?page=pool&s=show&id={}".format(
self.root, self.pool_id)
url = f"{self.root}/index.php?page=pool&s=show&id={self.pool_id}"
page = self.request(url).text
name, pos = text.extract(page, "<h4>Pool: ", "</h4>")
@@ -234,12 +229,9 @@ class GelbooruV02FavoriteExtractor(GelbooruV02Extractor):
pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)"
example = "https://safebooru.org/index.php?page=favorites&s=view&id=12345"
def __init__(self, match):
GelbooruV02Extractor.__init__(self, match)
self.favorite_id = match[match.lastindex]
def metadata(self):
return {"favorite_id": text.parse_int(self.favorite_id)}
self.favorite_id = fav_id = self.groups[-1]
return {"favorite_id": text.parse_int(fav_id)}
def posts(self):
return self._pagination_html({
@@ -255,9 +247,5 @@ class GelbooruV02PostExtractor(GelbooruV02Extractor):
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)"
example = "https://safebooru.org/index.php?page=post&s=view&id=12345"
def __init__(self, match):
GelbooruV02Extractor.__init__(self, match)
self.post_id = match[match.lastindex]
def posts(self):
return self._pagination({"id": self.post_id})
return self._pagination({"id": self.groups[-1]})

25
gallery_dl/extractor/jschan.py
View File

@@ -33,16 +33,9 @@ class JschanThreadExtractor(JschanExtractor):
pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)\.html"
example = "https://94chan.org/a/thread/12345.html"
def __init__(self, match):
JschanExtractor.__init__(self, match)
index = match.lastindex
self.board = match[index-1]
self.thread = match[index]
def items(self):
url = "{}/{}/thread/{}.json".format(
self.root, self.board, self.thread)
thread = self.request(url).json()
url = f"{self.root}/{self.groups[-2]}/thread/{self.groups[-1]}.json"
thread = self.request_json(url)
thread["threadId"] = thread["postId"]
posts = thread.pop("replies", ())
@@ -53,7 +46,7 @@ class JschanThreadExtractor(JschanExtractor):
thread.update(post)
thread["count"] = len(files)
for num, file in enumerate(files):
url = self.root + "/file/" + file["filename"]
url = f"{self.root}/file/{file['filename']}"
file.update(thread)
file["num"] = num
file["siteFilename"] = file["filename"]
@@ -68,14 +61,10 @@ class JschanBoardExtractor(JschanExtractor):
r"(?:/index\.html|/catalog\.html|/\d+\.html|/?$)")
example = "https://94chan.org/a/"
def __init__(self, match):
JschanExtractor.__init__(self, match)
self.board = match[match.lastindex]
def items(self):
url = "{}/{}/catalog.json".format(self.root, self.board)
for thread in self.request(url).json():
url = "{}/{}/thread/{}.html".format(
self.root, self.board, thread["postId"])
board = self.groups[-1]
url = f"{self.root}/{board}/catalog.json"
for thread in self.request_json(url):
url = f"{self.root}/{board}/thread/{thread['postId']}.html"
thread["_extractor"] = JschanThreadExtractor
yield Message.Queue, url, thread

25
gallery_dl/extractor/lynxchan.py
View File

@@ -42,22 +42,15 @@ class LynxchanThreadExtractor(LynxchanExtractor):
pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)"
example = "https://endchan.org/a/res/12345.html"
def __init__(self, match):
LynxchanExtractor.__init__(self, match)
index = match.lastindex
self.board = match[index-1]
self.thread = match[index]
def items(self):
url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread)
thread = self.request(url).json()
url = f"{self.root}/{self.groups[-2]}/res/{self.groups[-1]}.json"
thread = self.request_json(url)
thread["postId"] = thread["threadId"]
posts = thread.pop("posts", ())
yield Message.Directory, thread
for post in itertools.chain((thread,), posts):
files = post.pop("files", ())
if files:
if files := post.pop("files", ()):
thread.update(post)
for num, file in enumerate(files):
file.update(thread)
@@ -73,14 +66,10 @@ class LynxchanBoardExtractor(LynxchanExtractor):
pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)"
example = "https://endchan.org/a/"
def __init__(self, match):
LynxchanExtractor.__init__(self, match)
self.board = match[match.lastindex]
def items(self):
url = "{}/{}/catalog.json".format(self.root, self.board)
for thread in self.request(url).json():
url = "{}/{}/res/{}.html".format(
self.root, self.board, thread["threadId"])
board = self.groups[-1]
url = f"{self.root}/{board}/catalog.json"
for thread in self.request_json(url):
url = f"{self.root}/{board}/res/{thread['threadId']}.html"
thread["_extractor"] = LynxchanThreadExtractor
yield Message.Queue, url, thread

2
gallery_dl/extractor/mastodon.py
View File

@@ -22,7 +22,7 @@ class MastodonExtractor(BaseExtractor):
def __init__(self, match):
BaseExtractor.__init__(self, match)
self.item = match[match.lastindex]
self.item = self.groups[-1]
def _init(self):
self.instance = self.root.partition("://")[2]

13
gallery_dl/extractor/misskey.py
View File

@@ -20,7 +20,7 @@ class MisskeyExtractor(BaseExtractor):
def __init__(self, match):
BaseExtractor.__init__(self, match)
self.item = match[match.lastindex]
self.item = self.groups[-1]
def _init(self):
self.api = MisskeyAPI(self)
@@ -110,7 +110,7 @@ class MisskeyUserExtractor(Dispatch, MisskeyExtractor):
example = "https://misskey.io/@USER"
def items(self):
base = "{}/@{}/".format(self.root, self.item)
base = f"{self.root}/@{self.item}/"
return self._dispatch_extractors((
(MisskeyInfoExtractor , base + "info"),
(MisskeyAvatarExtractor , base + "avatar"),
@@ -174,10 +174,9 @@ class MisskeyFollowingExtractor(MisskeyExtractor):
user_id = self.api.user_id_by_username(self.item)
for user in self.api.users_following(user_id):
user = user["followee"]
url = self.root + "/@" + user["username"]
host = user["host"]
if host is not None:
url += "@" + host
url = f"{self.root}/@{user['username']}"
if (host := user["host"]) is not None:
url = f"{url}@{host}"
user["_extractor"] = MisskeyUserExtractor
yield Message.Queue, url, user
@@ -248,7 +247,7 @@ class MisskeyAPI():
return self._pagination(endpoint, data)
def _call(self, endpoint, data):
url = self.root + "/api" + endpoint
url = f"{self.root}/api{endpoint}"
return self.extractor.request_json(url, method="POST", json=data)
def _pagination(self, endpoint, data):

35
gallery_dl/extractor/moebooru.py
View File

@@ -24,8 +24,8 @@ class MoebooruExtractor(BooruExtractor):
post["date"] = text.parse_timestamp(post["created_at"])
def _html(self, post):
return self.request("{}/post/show/{}".format(
self.root, post["id"])).text
url = f"{self.root}/post/show/{post['id']}"
return self.request(url).text
def _tags(self, post, page):
tag_container = text.extr(page, '<ul id="tag-', '</ul>')
@@ -61,7 +61,7 @@ class MoebooruExtractor(BooruExtractor):
params["limit"] = self.per_page
while True:
posts = self.request(url, params=params).json()
posts = self.request_json(url, params=params)
yield from posts
if len(posts) < self.per_page:
@@ -98,15 +98,14 @@ class MoebooruTagExtractor(MoebooruExtractor):
def __init__(self, match):
MoebooruExtractor.__init__(self, match)
tags = match[match.lastindex]
self.tags = text.unquote(tags.replace("+", " "))
self.tags = text.unquote(self.groups[-1].replace("+", " "))
def metadata(self):
return {"search_tags": self.tags}
def posts(self):
params = {"tags": self.tags}
return self._pagination(self.root + "/post.json", params)
return self._pagination(f"{self.root}/post.json", params)
class MoebooruPoolExtractor(MoebooruExtractor):
@@ -118,12 +117,12 @@ class MoebooruPoolExtractor(MoebooruExtractor):
def __init__(self, match):
MoebooruExtractor.__init__(self, match)
self.pool_id = match[match.lastindex]
self.pool_id = self.groups[-1]
def metadata(self):
if self.config("metadata"):
url = "{}/pool/show/{}.json".format(self.root, self.pool_id)
pool = self.request(url).json()
url = f"{self.root}/pool/show/{self.pool_id}.json"
pool = self.request_json(url)
pool["name"] = pool["name"].replace("_", " ")
pool.pop("posts", None)
return {"pool": pool}
@@ -131,7 +130,7 @@ class MoebooruPoolExtractor(MoebooruExtractor):
def posts(self):
params = {"tags": "pool:" + self.pool_id}
return self._pagination(self.root + "/post.json", params)
return self._pagination(f"{self.root}/post.json", params)
class MoebooruPostExtractor(MoebooruExtractor):
@@ -140,13 +139,9 @@ class MoebooruPostExtractor(MoebooruExtractor):
pattern = BASE_PATTERN + r"/post/show/(\d+)"
example = "https://yande.re/post/show/12345"
def __init__(self, match):
MoebooruExtractor.__init__(self, match)
self.post_id = match[match.lastindex]
def posts(self):
params = {"tags": "id:" + self.post_id}
return self.request(self.root + "/post.json", params=params).json()
params = {"tags": "id:" + self.groups[-1]}
return self.request_json(f"{self.root}/post.json", params=params)
class MoebooruPopularExtractor(MoebooruExtractor):
@@ -159,8 +154,8 @@ class MoebooruPopularExtractor(MoebooruExtractor):
def __init__(self, match):
MoebooruExtractor.__init__(self, match)
self.scale = match[match.lastindex-1]
self.query = match[match.lastindex]
self.scale = self.groups[-2]
self.query = self.groups[-1]
def metadata(self):
self.params = params = text.parse_query(self.query)
@@ -186,5 +181,5 @@ class MoebooruPopularExtractor(MoebooruExtractor):
return {"date": date, "scale": scale}
def posts(self):
url = "{}/post/popular_{}.json".format(self.root, self.scale)
return self.request(url, params=self.params).json()
url = f"{self.root}/post/popular_{self.scale}.json"
return self.request_json(url, params=self.params)

8
gallery_dl/extractor/nijie.py
View File

@@ -23,7 +23,7 @@ class NijieExtractor(AsynchronousMixin, BaseExtractor):
def __init__(self, match):
BaseExtractor.__init__(self, match)
self.user_id = text.parse_int(match[match.lastindex])
self.user_id = text.parse_int(self.groups[-1])
def initialize(self):
self.cookies_domain = "." + self.root.rpartition("/")[2]
@@ -294,9 +294,5 @@ class NijieImageExtractor(NijieExtractor):
pattern = BASE_PATTERN + r"/view(?:_popup)?\.php\?id=(\d+)"
example = "https://nijie.info/view.php?id=12345"
def __init__(self, match):
NijieExtractor.__init__(self, match)
self.image_id = match[match.lastindex]
def image_ids(self):
return (self.image_id,)
return (self.groups[-1],)

5
gallery_dl/extractor/nitter.py
View File

@@ -24,9 +24,8 @@ class NitterExtractor(BaseExtractor):
self.cookies_domain = self.root.partition("://")[2]
BaseExtractor.__init__(self, match)
lastindex = match.lastindex
self.user = match[lastindex]
self.user_id = match[lastindex + 1]
self.user = self.groups[-2]
self.user_id = self.groups[-1]
self.user_obj = None
def items(self):

22
gallery_dl/extractor/philomena.py
View File

@@ -65,12 +65,8 @@ class PhilomenaPostExtractor(PhilomenaExtractor):
pattern = BASE_PATTERN + r"/(?:images/)?(\d+)"
example = "https://derpibooru.org/images/12345"
def __init__(self, match):
PhilomenaExtractor.__init__(self, match)
self.image_id = match[match.lastindex]
def posts(self):
return (self.api.image(self.image_id),)
return (self.api.image(self.groups[-1]),)
class PhilomenaSearchExtractor(PhilomenaExtractor):
@@ -82,9 +78,9 @@ class PhilomenaSearchExtractor(PhilomenaExtractor):
def __init__(self, match):
PhilomenaExtractor.__init__(self, match)
groups = match.groups()
if groups[-1]:
q = groups[-1].replace("+", " ")
if q := self.groups[-1]:
q = q.replace("+", " ")
for old, new in (
("-colon-" , ":"),
("-dash-" , "-"),
@@ -97,7 +93,7 @@ class PhilomenaSearchExtractor(PhilomenaExtractor):
q = q.replace(old, new)
self.params = {"q": text.unquote(text.unquote(q))}
else:
self.params = text.parse_query(groups[-2])
self.params = text.parse_query(self.groups[-2])
def metadata(self):
return {"search_tags": self.params.get("q", "")}
@@ -114,18 +110,14 @@ class PhilomenaGalleryExtractor(PhilomenaExtractor):
pattern = BASE_PATTERN + r"/galleries/(\d+)"
example = "https://derpibooru.org/galleries/12345"
def __init__(self, match):
PhilomenaExtractor.__init__(self, match)
self.gallery_id = match[match.lastindex]
def metadata(self):
try:
return {"gallery": self.api.gallery(self.gallery_id)}
return {"gallery": self.api.gallery(self.groups[-1])}
except IndexError:
raise exception.NotFoundError("gallery")
def posts(self):
gallery_id = "gallery_id:" + self.gallery_id
gallery_id = f"gallery_id:{self.groups[-1]}"
params = {"sd": "desc", "sf": gallery_id, "q": gallery_id}
return self.api.search(params)

8
gallery_dl/extractor/reactor.py
View File

@@ -176,7 +176,7 @@ class ReactorTagExtractor(ReactorExtractor):
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.tag = match[match.lastindex]
self.tag = self.groups[-1]
def metadata(self):
return {"search_tags": text.unescape(self.tag).replace("+", " ")}
@@ -192,7 +192,7 @@ class ReactorSearchExtractor(ReactorExtractor):
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.tag = match[match.lastindex]
self.tag = self.groups[-1]
def metadata(self):
return {"search_tags": text.unescape(self.tag).replace("+", " ")}
@@ -207,7 +207,7 @@ class ReactorUserExtractor(ReactorExtractor):
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.user = match[match.lastindex]
self.user = self.groups[-1]
def metadata(self):
return {"user": text.unescape(self.user).replace("+", " ")}
@@ -221,7 +221,7 @@ class ReactorPostExtractor(ReactorExtractor):
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.post_id = match[match.lastindex]
self.post_id = self.groups[-1]
def items(self):
post = self.request(self.root + self.path).text

14
gallery_dl/extractor/shopify.py
View File

@@ -18,10 +18,6 @@ class ShopifyExtractor(BaseExtractor):
filename_fmt = "{product[title]}_{num:>02}_{id}.{extension}"
archive_fmt = "{id}"
def __init__(self, match):
BaseExtractor.__init__(self, match)
self.item_url = self.root + match[match.lastindex]
def items(self):
data = self.metadata()
yield Message.Directory, data
@@ -98,14 +94,15 @@ class ShopifyCollectionExtractor(ShopifyExtractor):
example = "https://www.fashionnova.com/collections/TITLE"
def metadata(self):
return self.request(self.item_url + ".json").json()
url = f"{self.root}{self.groups[-1]}.json"
return self.request_json(url)
def products(self):
url = self.item_url + "/products.json"
url = f"{self.root}{self.groups[-1]}/products.json"
params = {"page": 1}
while True:
data = self.request(url, params=params).json()["products"]
data = self.request_json(url, params=params)["products"]
if not data:
return
yield from data
@@ -120,6 +117,7 @@ class ShopifyProductExtractor(ShopifyExtractor):
example = "https://www.fashionnova.com/collections/TITLE/products/NAME"
def products(self):
product = self.request(self.item_url + ".json").json()["product"]
url = f"{self.root}{self.groups[-1]}.json"
product = self.request_json(url)["product"]
del product["image"]
return (product,)

56
gallery_dl/extractor/vichan.py
View File

@@ -38,21 +38,18 @@ class VichanThreadExtractor(VichanExtractor):
pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)"
example = "https://8kun.top/a/res/12345.html"
def __init__(self, match):
VichanExtractor.__init__(self, match)
index = match.lastindex
self.board = match[index-1]
self.thread = match[index]
def items(self):
url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread)
posts = self.request(url).json()["posts"]
board = self.groups[-2]
thread = self.groups[-1]
url = f"{self.root}/{board}/res/{thread}.json"
posts = self.request_json(url)["posts"]
title = posts[0].get("sub") or text.remove_html(posts[0]["com"])
process = (self._process_8kun if self.category == "8kun" else
self._process)
data = {
"board" : self.board,
"thread": self.thread,
"board" : board,
"thread": thread,
"title" : text.unescape(title)[:50],
"num" : 0,
}
@@ -68,24 +65,25 @@ class VichanThreadExtractor(VichanExtractor):
def _process(self, post, data):
post.update(data)
post["extension"] = post["ext"][1:]
post["url"] = "{}/{}/src/{}{}".format(
self.root, post["board"], post["tim"], post["ext"])
return Message.Url, post["url"], post
ext = post["ext"]
post["extension"] = ext[1:]
post["url"] = url = \
f"{self.root}/{post['board']}/src/{post['tim']}{ext}"
return Message.Url, url, post
def _process_8kun(self, post, data):
post.update(data)
post["extension"] = post["ext"][1:]
ext = post["ext"]
tim = post["tim"]
if len(tim) > 16:
post["url"] = "https://media.128ducks.com/file_store/{}{}".format(
tim, post["ext"])
else:
post["url"] = "https://media.128ducks.com/{}/src/{}{}".format(
post["board"], tim, post["ext"])
return Message.Url, post["url"], post
if len(tim) > 16:
url = f"https://media.128ducks.com/file_store/{tim}{ext}"
else:
url = f"https://media.128ducks.com/{post['board']}/src/{tim}{ext}"
post["url"] = url
post["extension"] = ext[1:]
return Message.Url, url, post
class VichanBoardExtractor(VichanExtractor):
@@ -94,18 +92,14 @@ class VichanBoardExtractor(VichanExtractor):
pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)"
example = "https://8kun.top/a/"
def __init__(self, match):
VichanExtractor.__init__(self, match)
self.board = match[match.lastindex]
def items(self):
url = "{}/{}/threads.json".format(self.root, self.board)
threads = self.request(url).json()
board = self.groups[-1]
url = f"{self.root}/{board}/threads.json"
threads = self.request_json(url)
for page in threads:
for thread in page["threads"]:
url = "{}/{}/res/{}.html".format(
self.root, self.board, thread["no"])
url = f"{self.root}/{board}/res/{thread['no']}.html"
thread["page"] = page["page"]
thread["_extractor"] = VichanThreadExtractor
yield Message.Queue, url, thread