github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[weibo] improve extractor logic (#829)

Browse Source
This commit is contained in:
Mike Fährmann
2020-06-17 23:25:21 +02:00
parent 37d71f6e09
commit 7158bdd7c7
1 changed files with 44 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
gallery_dl/extractor/weibo.py
View File

@@ -10,6 +10,7 @@
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, exception from .. import text, exception
import itertools
import json import json
@@ -30,55 +31,53 @@ class WeiboExtractor(Extractor):
for status in self.statuses(): for status in self.statuses():
yield Message.Directory, status files = self._files_from_status(status)
obj = status if self.retweets and "retweeted_status" in status:
obj["date"] = text.parse_datetime( files = itertools.chain(
obj["created_at"], "%a %b %d %H:%M:%S %z %Y") files,
num = 1 self._files_from_status(status["retweeted_status"]),
)
while True: for num, file in enumerate(files, 1):
if num == 1:
status["date"] = text.parse_datetime(
status["created_at"], "%a %b %d %H:%M:%S %z %Y")
yield Message.Directory, status
file["status"] = status
file["num"] = num
yield Message.Url, file["url"], file
if "pics" in obj: def _files_from_status(self, status):
for image in obj["pics"]: images = status.pop("pics", ())
pid = image["pid"] page_info = status.pop("page_info", ())
if "large" in image:
image = image["large"]
geo = image.get("geo") or {}
data = text.nameext_from_url(image["url"], {
"num" : num,
"pid" : pid,
"url" : image["url"],
"width" : text.parse_int(geo.get("width")),
"height": text.parse_int(geo.get("height")),
"status": status,
})
yield Message.Url, image["url"], data
num += 1
if self.videos and "media_info" in obj.get("page_info", ()): for image in images:
info = obj["page_info"]["media_info"] pid = image["pid"]
url = info.get("stream_url_hd") or info.get("stream_url") if "large" in image:
image = image["large"]
geo = image.get("geo") or {}
yield text.nameext_from_url(image["url"], {
"url" : image["url"],
"pid" : pid,
"width" : text.parse_int(geo.get("width")),
"height": text.parse_int(geo.get("height")),
})
if url: if self.videos and "media_info" in page_info:
data = text.nameext_from_url(url, { info = page_info["media_info"]
"num" : num, url = info.get("stream_url_hd") or info.get("stream_url")
"pid" : 0, if url:
"url" : url, data = text.nameext_from_url(url, {
"width" : 0, "url" : url,
"height": 0, "pid" : 0,
"status": status, "width" : 0,
}) "height": 0,
if data["extension"] == "m3u8": })
url = "ytdl:" + url if data["extension"] == "m3u8":
data["extension"] = "mp4" data["extension"] = "mp4"
data["_ytdl_extra"] = {"protocol": "m3u8_native"} data["url"] = "ytdl:" + url
yield Message.Url, url, data data["_ytdl_extra"] = {"protocol": "m3u8_native"}
num += 1 yield data
if self.retweets and "retweeted_status" in obj:
obj = obj["retweeted_status"]
else:
break
def statuses(self): def statuses(self):
"""Returns an iterable containing all relevant 'status' objects""" """Returns an iterable containing all relevant 'status' objects"""