github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[weibo] fixes and improvements

Browse Source 
- ignore unavailable videos (fixes #427)
- handle empty 'geo' fields
- consistent metadata fields for images and videos
This commit is contained in:
Mike Fährmann
2019-09-26 14:57:35 +02:00
parent 38d97f3da6
commit d1ea08c67d
1 changed files with 25 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
gallery_dl/extractor/weibo.py
View File

@@ -25,18 +25,14 @@ class WeiboExtractor(Extractor):
self.retweets = self.config("retweets", True) self.retweets = self.config("retweets", True)
def items(self): def items(self):
first = True yield Message.Version, 1
for status in self.statuses(): for status in self.statuses():
yield Message.Directory, status
obj = status obj = status
num = 1 num = 1
if first:
yield Message.Version, 1
yield Message.Directory, status
first = False
while True: while True:
if "pics" in obj: if "pics" in obj:
@@ -44,11 +40,13 @@ class WeiboExtractor(Extractor):
pid = image["pid"] pid = image["pid"]
if "large" in image: if "large" in image:
image = image["large"] image = image["large"]
geo = image.get("geo") or {}
data = text.nameext_from_url(image["url"], { data = text.nameext_from_url(image["url"], {
"num": num, "num" : num,
"pid": pid, "pid" : pid,
"width": text.parse_int(image["geo"]["width"]), "url" : image["url"],
"height": text.parse_int(image["geo"]["height"]), "width" : text.parse_int(geo.get("width")),
"height": text.parse_int(geo.get("height")),
"status": status, "status": status,
}) })
yield Message.Url, image["url"], data yield Message.Url, image["url"], data
@@ -56,15 +54,18 @@ class WeiboExtractor(Extractor):
if "page_info" in obj and "media_info" in obj["page_info"]: if "page_info" in obj and "media_info" in obj["page_info"]:
info = obj["page_info"]["media_info"] info = obj["page_info"]["media_info"]
url = info.get("stream_url_hd") or info["stream_url"] url = info.get("stream_url_hd") or info.get("stream_url")
data = text.nameext_from_url(url, {
"num": num, if url and not info.get("goto"):
"url": url, data = text.nameext_from_url(url, {
"width": 0, "num" : num,
"height": 0, "pid" : 0,
"status": status, "url" : url,
}) "width" : 0,
yield Message.Url, url, data "height": 0,
"status": status,
})
yield Message.Url, url, data
if self.retweets and "retweeted_status" in obj: if self.retweets and "retweeted_status" in obj:
obj = obj["retweeted_status"] obj = obj["retweeted_status"]
@@ -104,7 +105,7 @@ class WeiboUserExtractor(WeiboExtractor):
if "mblog" in card: if "mblog" in card:
yield card["mblog"] yield card["mblog"]
if len(data["data"]["cards"]) < 5: if not data["data"]["cards"]:
return return
params["page"] += 1 params["page"] += 1
@@ -121,6 +122,10 @@ class WeiboStatusExtractor(WeiboExtractor):
("https://m.weibo.cn/detail/4339748116375525", { ("https://m.weibo.cn/detail/4339748116375525", {
"pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_hd", "pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_hd",
}), }),
# unavailable video (#427)
("https://m.weibo.cn/status/4268682979207023", {
"count": 0,
}),
("https://m.weibo.cn/status/4339748116375525"), ("https://m.weibo.cn/status/4339748116375525"),
("https://m.weibo.cn/5746766133/4339748116375525"), ("https://m.weibo.cn/5746766133/4339748116375525"),
) )