[vsco] fix user profile extraction … again
Given the pattern from last time, collections will also change in due time and use cursor-based pagination.
This commit is contained in:
@@ -66,10 +66,20 @@ class VscoExtractor(Extractor):
|
|||||||
|
|
||||||
while True:
|
while True:
|
||||||
data = self.request(url, params=params, headers=headers).json()
|
data = self.request(url, params=params, headers=headers).json()
|
||||||
if not data.get(key):
|
medias = data.get(key)
|
||||||
|
if not medias:
|
||||||
return
|
return
|
||||||
yield from data[key]
|
|
||||||
params["page"] += 1
|
if "cursor" in params:
|
||||||
|
for media in medias:
|
||||||
|
yield media[media["type"]]
|
||||||
|
cursor = data.get("next_cursor")
|
||||||
|
if not cursor:
|
||||||
|
return
|
||||||
|
params["cursor"] = cursor
|
||||||
|
else:
|
||||||
|
yield from medias
|
||||||
|
params["page"] += 1
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _transform_media(media):
|
def _transform_media(media):
|
||||||
@@ -102,13 +112,19 @@ class VscoUserExtractor(VscoExtractor):
|
|||||||
|
|
||||||
tkn = data["users"]["currentUser"]["tkn"]
|
tkn = data["users"]["currentUser"]["tkn"]
|
||||||
sid = str(data["sites"]["siteByUsername"][self.user]["site"]["id"])
|
sid = str(data["sites"]["siteByUsername"][self.user]["site"]["id"])
|
||||||
|
site = data["medias"]["bySiteId"][sid]
|
||||||
|
|
||||||
|
url = "{}/api/3.0/medias/profile".format(self.root)
|
||||||
|
params = {
|
||||||
|
"site_id" : sid,
|
||||||
|
"limit" : "14",
|
||||||
|
"show_only": "0",
|
||||||
|
"cursor" : site["nextCursor"],
|
||||||
|
}
|
||||||
|
|
||||||
url = "{}/api/2.0/medias".format(self.root)
|
|
||||||
params = {"page": 2, "size": "30", "site_id": sid}
|
|
||||||
return self._pagination(url, params, tkn, "media", (
|
return self._pagination(url, params, tkn, "media", (
|
||||||
data["medias"]["byId"][mid]["media"]
|
data["medias"]["byId"][media[media["type"]]]["media"]
|
||||||
for mid in data
|
for media in site["medias"]
|
||||||
["medias"]["bySiteId"][sid]["byPage"]["1"]["mediaIds"]
|
|
||||||
))
|
))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user