github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[weibo] improve extractor logic (#829)

Browse Source
This commit is contained in:
Mike Fährmann
2020-06-17 23:25:21 +02:00
parent 37d71f6e09
commit 7158bdd7c7
1 changed files with 44 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
gallery_dl/extractor/weibo.py
View File

@@ -10,6 +10,7 @@
from .common import Extractor, Message
from .. import text, exception
import itertools
import json
@@ -30,55 +31,53 @@ class WeiboExtractor(Extractor):
for status in self.statuses():
yield Message.Directory, status
obj = status
obj["date"] = text.parse_datetime(
obj["created_at"], "%a %b %d %H:%M:%S %z %Y")
num = 1
files = self._files_from_status(status)
if self.retweets and "retweeted_status" in status:
files = itertools.chain(
files,
self._files_from_status(status["retweeted_status"]),
)
while True:
for num, file in enumerate(files, 1):
if num == 1:
status["date"] = text.parse_datetime(
status["created_at"], "%a %b %d %H:%M:%S %z %Y")
yield Message.Directory, status
file["status"] = status
file["num"] = num
yield Message.Url, file["url"], file
if "pics" in obj:
for image in obj["pics"]:
pid = image["pid"]
if "large" in image:
image = image["large"]
geo = image.get("geo") or {}
data = text.nameext_from_url(image["url"], {
"num" : num,
"pid" : pid,
"url" : image["url"],
"width" : text.parse_int(geo.get("width")),
"height": text.parse_int(geo.get("height")),
"status": status,
})
yield Message.Url, image["url"], data
num += 1
def _files_from_status(self, status):
images = status.pop("pics", ())
page_info = status.pop("page_info", ())
if self.videos and "media_info" in obj.get("page_info", ()):
info = obj["page_info"]["media_info"]
url = info.get("stream_url_hd") or info.get("stream_url")
for image in images:
pid = image["pid"]
if "large" in image:
image = image["large"]
geo = image.get("geo") or {}
yield text.nameext_from_url(image["url"], {
"url" : image["url"],
"pid" : pid,
"width" : text.parse_int(geo.get("width")),
"height": text.parse_int(geo.get("height")),
})
if url:
data = text.nameext_from_url(url, {
"num" : num,
"pid" : 0,
"url" : url,
"width" : 0,
"height": 0,
"status": status,
})
if data["extension"] == "m3u8":
url = "ytdl:" + url
data["extension"] = "mp4"
data["_ytdl_extra"] = {"protocol": "m3u8_native"}
yield Message.Url, url, data
num += 1
if self.retweets and "retweeted_status" in obj:
obj = obj["retweeted_status"]
else:
break
if self.videos and "media_info" in page_info:
info = page_info["media_info"]
url = info.get("stream_url_hd") or info.get("stream_url")
if url:
data = text.nameext_from_url(url, {
"url" : url,
"pid" : 0,
"width" : 0,
"height": 0,
})
if data["extension"] == "m3u8":
data["extension"] = "mp4"
data["url"] = "ytdl:" + url
data["_ytdl_extra"] = {"protocol": "m3u8_native"}
yield data
def statuses(self):
"""Returns an iterable containing all relevant 'status' objects"""