github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[vsco] add collection & image extractor + video support (#331)

Browse Source
This commit is contained in:
Mike Fährmann
2019-07-25 17:51:35 +02:00
parent 547ea71463
commit 279db2c5b2
2 changed files with 104 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/supportedsites.rst
View File

@@ -107,7 +107,7 @@ The /b/ Archive https://thebarchive.com/ Threads
Tsumino https://www.tsumino.com/ Galleries, Search Results Optional
Tumblr https://www.tumblr.com/ Images from Users, Likes, Posts, Tag-Searches Optional (OAuth)
Twitter https://twitter.com/ Media Timelines, Timelines, Tweets Optional
VSCO https://vsco.co/ Images from Users
VSCO https://vsco.co/ Images from Users, Collections, individual Images
Wallhaven https://wallhaven.cc/ individual Images, Search Results
Warosu https://warosu.org/ Threads
Weibo https://www.weibo.com/ Images from Users, Images from Statuses

142
gallery_dl/extractor/vsco.py
View File

@@ -17,8 +17,10 @@ BASE_PATTERN = r"(?:https?://)?(?:www\.)?vsco\.co/([^/]+)"
class VscoExtractor(Extractor):
"""Base class for vsco extractors"""
category = "vsco"
root = "https://vsco.co"
directory_fmt = ("{category}", "{user}")
filename_fmt = "{id}.{extension}"
archive_fmt = "{id}"
@@ -30,7 +32,7 @@ class VscoExtractor(Extractor):
yield Message.Version, 1
yield Message.Directory, {"user": self.user}
for img in self.images():
url = "https://" + img["responsive_url"]
url = "https://" + (img.get("video_url") or img["responsive_url"])
data = text.nameext_from_url(url, {
"id" : img["_id"],
"user" : self.user,
@@ -38,15 +40,21 @@ class VscoExtractor(Extractor):
"meta" : img.get("image_meta") or {},
"tags" : [tag["text"] for tag in img.get("tags") or ()],
"date" : text.parse_timestamp(img["upload_date"] // 1000),
"video" : img["is_video"],
"width" : img["width"],
"height": img["height"],
"description": img["description"],
})
yield Message.Url, url, data
def _pagination(self, url, params, token, extra):
yield from extra
def images(self):
"""Return an iterable with all relevant image objects"""
def _extract_preload_state(self, url):
page = self.request(url, notfound=self.subcategory).text
return json.loads(text.extract(page, "__PRELOADED_STATE__ = ", "<")[0])
def _pagination(self, url, params, token, key, extra):
headers = {
"Referer" : "{}/{}".format(self.root, self.user),
"Authorization" : "Bearer " + token,
@@ -54,58 +62,114 @@ class VscoExtractor(Extractor):
"X-Client-Build" : "1",
}
yield from map(self._transform_media, extra)
while True:
data = self.request(url, headers=headers, params=params).json()
if not data.get("media"):
data = self.request(url, params=params, headers=headers).json()
if not data.get(key):
return
yield from data["media"]
yield from data[key]
params["page"] += 1
@staticmethod
def _transform_media(media):
media["_id"] = media["id"]
media["is_video"] = media["isVideo"]
media["grid_name"] = media["gridName"]
media["upload_date"] = media["uploadDate"]
media["responsive_url"] = media["responsiveUrl"]
media["video_url"] = media.get("videoUrl")
media["image_meta"] = media.get("imageMeta")
return media
class VscoUserExtractor(VscoExtractor):
"""Extractor for images from a user on vsco.co"""
subcategory = "user"
directory_fmt = ("{category}", "{user}")
pattern = BASE_PATTERN + r"/images/"
test = ("https://vsco.co/missuri/images/1", {
"range": "1-80",
"count": 80,
"pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+",
"keyword": {
"id" : str,
"user" : "missuri",
"grid" : "anybodyseenmylife",
"meta" : dict,
"tags" : list,
"date" : "type:datetime",
"width" : int,
"height": int,
"description": str,
},
})
def images(self):
url = "{}/{}/images/1".format(self.root, self.user)
page = self.request(url, notfound="user").text
data = json.loads(text.extract(page, "__PRELOADED_STATE__ = ", "<")[0])
site_id = str(data["sites"]["siteByUsername"][self.user]["site"]["id"])
token = data["users"]["currentUser"]["tkn"]
data = self._extract_preload_state(url)
url = "https://vsco.co/api/2.0/medias"
params = {
"site_id": site_id,
"page" : 2,
"size" : "30",
}
tkn = data["users"]["currentUser"]["tkn"]
sid = str(data["sites"]["siteByUsername"][self.user]["site"]["id"])
extra = []
medias = data["medias"]["byId"]
for mid in data["medias"]["bySiteId"][site_id]["medias"]["1"]:
media = medias[mid]["media"]
media["_id"] = media["id"]
media["grid_name"] = media["gridName"]
media["image_meta"] = media["imageMeta"]
media["upload_date"] = media["uploadDate"]
media["responsive_url"] = media["responsiveUrl"]
extra.append(media)
url = "{}/api/2.0/medias".format(self.root)
params = {"page": 2, "size": "30", "site_id": sid}
return self._pagination(url, params, tkn, "media", (
data["medias"]["byId"][mid]["media"]
for mid in data["medias"]["bySiteId"][sid]["medias"]["1"]
))
return self._pagination(url, params, token, extra)
class VscoCollectionExtractor(VscoExtractor):
"""Extractor for images from a collection on vsco.co"""
subcategory = "collection"
directory_fmt = ("{category}", "{user}", "collection")
pattern = BASE_PATTERN + r"/collection/"
test = ("https://vsco.co/vsco/collection/1", {
"range": "1-80",
"count": 80,
"pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+",
})
def images(self):
url = "{}/{}/collection/1".format(self.root, self.user)
data = self._extract_preload_state(url)
tkn = data["users"]["currentUser"]["tkn"]
cid = (data["sites"]["siteByUsername"][self.user]
["site"]["siteCollectionId"])
url = "{}/api/2.0/collections/{}/medias".format(self.root, cid)
params = {"page": 2, "size": "20"}
return self._pagination(url, params, tkn, "medias", (
data["medias"]["byId"][mid]["media"]
for mid in data
["collections"]["byCollectionId"][cid]["collection"]["1"]
))
class VscoImageExtractor(VscoExtractor):
"""Extractor for individual images on vsco.co"""
subcategory = "image"
pattern = BASE_PATTERN + r"/media/([0-9a-fA-F]+)"
test = (
("https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", {
"url": "faa214d10f859f374ad91da3f7547d2439f5af08",
"content": "1394d070828d82078035f19a92f404557b56b83f",
"keyword": {
"id" : "5d34b93ef632433030707ce2",
"user" : "erenyildiz",
"grid" : "erenyildiz",
"meta" : dict,
"tags" : list,
"date" : "type:datetime",
"video" : False,
"width" : 1537,
"height": 1537,
"description": "re:Ni seviyorum. #vsco #vscox #vscochallenges",
},
}),
("https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", {
"url": "08e7eef3301756ce81206c0b47c1e9373756a74a",
"content": "e739f058d726ee42c51c180a505747972a7dfa47",
"keyword": {"video" : True},
}),
)
def __init__(self, match):
VscoExtractor.__init__(self, match)
self.media_id = match.group(2)
def images(self):
url = "{}/{}/media/{}".format(self.root, self.user, self.media_id)
data = self._extract_preload_state(url)
media = data["medias"]["byId"].popitem()[1]["media"]
return (self._transform_media(media),)