[kemono] prevent 'revisions' API requests when possible
posts from '/v1/{service}/user/{creator_id}/post/{post_id}' already
include their revisions and don't need an additional API request
This commit is contained in:
@@ -82,6 +82,8 @@ class KemonoExtractor(Extractor):
|
|||||||
posts = self._revisions(posts)
|
posts = self._revisions(posts)
|
||||||
|
|
||||||
for post in posts:
|
for post in posts:
|
||||||
|
if "post" in post:
|
||||||
|
post = post["post"]
|
||||||
headers["Referer"] = (f"{self.root}/{post['service']}/user/"
|
headers["Referer"] = (f"{self.root}/{post['service']}/user/"
|
||||||
f"{post['user']}/post/{post['id']}")
|
f"{post['user']}/post/{post['id']}")
|
||||||
post["_http_headers"] = headers
|
post["_http_headers"] = headers
|
||||||
@@ -250,11 +252,23 @@ class KemonoExtractor(Extractor):
|
|||||||
return itertools.chain.from_iterable(
|
return itertools.chain.from_iterable(
|
||||||
self._revisions_post(post) for post in posts)
|
self._revisions_post(post) for post in posts)
|
||||||
|
|
||||||
|
def _revisions_get(self, post):
|
||||||
|
if (props := post.get("props")) and "revisions" in props:
|
||||||
|
return [
|
||||||
|
rev[1]
|
||||||
|
for rev in props["revisions"]
|
||||||
|
if "revision_id" in rev[1]
|
||||||
|
]
|
||||||
|
return self.api.creator_post_revisions(
|
||||||
|
post["service"], post["user"], post["id"])
|
||||||
|
|
||||||
def _revisions_post(self, post):
|
def _revisions_post(self, post):
|
||||||
|
revs = self._revisions_get(post)
|
||||||
|
|
||||||
|
if "post" in post:
|
||||||
|
post = post["post"]
|
||||||
post["revision_id"] = 0
|
post["revision_id"] = 0
|
||||||
|
|
||||||
revs = self.api.creator_post_revisions(
|
|
||||||
post["service"], post["user"], post["id"])
|
|
||||||
if not revs:
|
if not revs:
|
||||||
post["revision_hash"] = self._revision_hash(post)
|
post["revision_hash"] = self._revision_hash(post)
|
||||||
post["revision_index"] = 1
|
post["revision_index"] = 1
|
||||||
@@ -285,8 +299,8 @@ class KemonoExtractor(Extractor):
|
|||||||
|
|
||||||
return revs
|
return revs
|
||||||
|
|
||||||
def _revisions_all(self, service, creator_id, post_id):
|
def _revisions_all(self, post):
|
||||||
revs = self.api.creator_post_revisions(service, creator_id, post_id)
|
revs = self._revisions_get(post)
|
||||||
|
|
||||||
cnt = idx = len(revs)
|
cnt = idx = len(revs)
|
||||||
for rev in revs:
|
for rev in revs:
|
||||||
@@ -373,11 +387,11 @@ class KemonoPostExtractor(KemonoExtractor):
|
|||||||
_, _, service, creator_id, post_id, revision, revision_id = self.groups
|
_, _, service, creator_id, post_id, revision, revision_id = self.groups
|
||||||
post = self.api.creator_post(service, creator_id, post_id)
|
post = self.api.creator_post(service, creator_id, post_id)
|
||||||
if not revision:
|
if not revision:
|
||||||
return (post["post"],)
|
return (post,)
|
||||||
|
|
||||||
self.revisions = False
|
self.revisions = False
|
||||||
|
|
||||||
revs = self._revisions_all(service, creator_id, post_id)
|
revs = self._revisions_all(post)
|
||||||
if not revision_id:
|
if not revision_id:
|
||||||
return revs
|
return revs
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,6 @@
|
|||||||
# published by the Free Software Foundation.
|
# published by the Free Software Foundation.
|
||||||
|
|
||||||
from gallery_dl.extractor import kemono
|
from gallery_dl.extractor import kemono
|
||||||
from gallery_dl import util, exception
|
|
||||||
|
|
||||||
|
|
||||||
__tests__ = (
|
__tests__ = (
|
||||||
@@ -357,6 +356,8 @@ __tests__ = (
|
|||||||
"revision_index": {1, 2, 3},
|
"revision_index": {1, 2, 3},
|
||||||
"revision_count": 3,
|
"revision_count": 3,
|
||||||
"revision_hash" : {
|
"revision_hash" : {
|
||||||
|
"eb2fa4385af730509a42f8f0424bd0b9a0e4bc21",
|
||||||
|
"a44ad7fa57ebc2473e861c1d7f11de721c809549",
|
||||||
"e0e93281495e151b11636c156e52bfe9234c2a40",
|
"e0e93281495e151b11636c156e52bfe9234c2a40",
|
||||||
"bc5713195e14799da40c525381216c5a1a340b0f",
|
"bc5713195e14799da40c525381216c5a1a340b0f",
|
||||||
"9872bfb536a47cc69d95d2f195cd5c825808f089",
|
"9872bfb536a47cc69d95d2f195cd5c825808f089",
|
||||||
@@ -388,7 +389,7 @@ __tests__ = (
|
|||||||
"#comment" : "revisions (#4498)",
|
"#comment" : "revisions (#4498)",
|
||||||
"#category": ("", "kemono", "patreon"),
|
"#category": ("", "kemono", "patreon"),
|
||||||
"#class" : kemono.KemonoPostExtractor,
|
"#class" : kemono.KemonoPostExtractor,
|
||||||
"#exception": exception.NotFoundError,
|
"#exception": "NotFoundError",
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -537,7 +538,6 @@ __tests__ = (
|
|||||||
"#comment" : "user profile data unavailable (#8382)",
|
"#comment" : "user profile data unavailable (#8382)",
|
||||||
"#category": ("", "kemono", "patreon"),
|
"#category": ("", "kemono", "patreon"),
|
||||||
"#class" : kemono.KemonoPostExtractor,
|
"#class" : kemono.KemonoPostExtractor,
|
||||||
"#log" : "patreon/34792417/137409895: 'Creator not found'",
|
|
||||||
"#results" : (
|
"#results" : (
|
||||||
"https://kemono.cr/data/a9/87/a9874d7e1229396b0b2706fd7fa9949eac924e86256d84d077c10ecbace8bd17.bin",
|
"https://kemono.cr/data/a9/87/a9874d7e1229396b0b2706fd7fa9949eac924e86256d84d077c10ecbace8bd17.bin",
|
||||||
"https://kemono.cr/data/a2/eb/a2eba02204086c789d59bc7112510aebf0428455ad1664153bfbb92eb8aa5643.jpg",
|
"https://kemono.cr/data/a2/eb/a2eba02204086c789d59bc7112510aebf0428455ad1664153bfbb92eb8aa5643.jpg",
|
||||||
@@ -545,8 +545,8 @@ __tests__ = (
|
|||||||
|
|
||||||
"title" : "Capella - Re:zero (20P)",
|
"title" : "Capella - Re:zero (20P)",
|
||||||
"user" : "34792417",
|
"user" : "34792417",
|
||||||
"user_profile": util.NONE,
|
"user_profile": dict,
|
||||||
"username" : util.NONE,
|
"username" : "Varas",
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -785,7 +785,7 @@ __tests__ = (
|
|||||||
"#category": ("", "kemono", "discord-server"),
|
"#category": ("", "kemono", "discord-server"),
|
||||||
"#class" : kemono.KemonoDiscordServerExtractor,
|
"#class" : kemono.KemonoDiscordServerExtractor,
|
||||||
"#pattern" : kemono.KemonoDiscordExtractor.pattern,
|
"#pattern" : kemono.KemonoDiscordExtractor.pattern,
|
||||||
"#count" : 26,
|
"#count" : 27,
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -871,23 +871,7 @@ __tests__ = (
|
|||||||
"#category": ("", "kemono", "artists"),
|
"#category": ("", "kemono", "artists"),
|
||||||
"#class" : kemono.KemonoArtistsExtractor,
|
"#class" : kemono.KemonoArtistsExtractor,
|
||||||
"#pattern" : kemono.KemonoUserExtractor.pattern,
|
"#pattern" : kemono.KemonoUserExtractor.pattern,
|
||||||
"#results" : (
|
"#count" : range(15, 20),
|
||||||
"https://kemono.cr/patreon/user/91205314",
|
|
||||||
"https://kemono.cr/patreon/user/51528107",
|
|
||||||
"https://kemono.cr/fanbox/user/12812028",
|
|
||||||
"https://kemono.cr/patreon/user/35237747",
|
|
||||||
"https://kemono.cr/patreon/user/8296916",
|
|
||||||
"https://kemono.cr/patreon/user/155095324",
|
|
||||||
"https://kemono.cr/patreon/user/75988930",
|
|
||||||
"https://kemono.cr/patreon/user/93703989",
|
|
||||||
"https://kemono.cr/patreon/user/100292687",
|
|
||||||
"https://kemono.cr/patreon/user/138609443",
|
|
||||||
"https://kemono.cr/patreon/user/61646879",
|
|
||||||
"https://kemono.cr/patreon/user/110669843",
|
|
||||||
"https://kemono.cr/patreon/user/44343773",
|
|
||||||
"https://kemono.cr/patreon/user/77920059",
|
|
||||||
"https://kemono.cr/patreon/user/102386631",
|
|
||||||
),
|
|
||||||
|
|
||||||
"favorited": int,
|
"favorited": int,
|
||||||
"id" : str,
|
"id" : str,
|
||||||
|
|||||||
Reference in New Issue
Block a user