diff --git a/gallery_dl/extractor/bcy.py b/gallery_dl/extractor/bcy.py index f1fcfc62..c3049a49 100644 --- a/gallery_dl/extractor/bcy.py +++ b/gallery_dl/extractor/bcy.py @@ -31,96 +31,103 @@ class BcyExtractor(Extractor): iroot = "https://img-bcy-qn.pstatp.com" noop = self.config("noop") - for post_id in self.posts(): - post = self._parse_post(post_id) - if not post: + for post in self.posts(): + if not post["image_list"]: continue - yield Message.Directory, post - for post["num"], image in enumerate(post["_multi"], 1): - post["id"] = image["mid"] - post["width"] = image["w"] - post["height"] = image["h"] + multi = None + tags = post.get("post_tags") or () + data = { + "user": { + "id" : post["uid"], + "name" : post["uname"], + "avatar" : sub(iroot, post["avatar"].partition("~")[0]), + }, + "post": { + "id" : text.parse_int(post["item_id"]), + "tags" : [t["tag_name"] for t in tags], + "date" : text.parse_timestamp(post["ctime"]), + "parody" : post["work"], + "content": post["plain"], + "likes" : post["like_count"], + "shares" : post["share_count"], + "replies": post["reply_count"], + }, + } + + yield Message.Directory, data + for data["num"], image in enumerate(post["image_list"], 1): + data["id"] = image["mid"] + data["width"] = image["w"] + data["height"] = image["h"] url = image["path"].partition("~")[0] - text.nameext_from_url(url, post) + text.nameext_from_url(url, data) - if post["extension"]: + if data["extension"]: if not url.startswith(iroot): url = sub(iroot, url) - post["filter"] = "" - yield Message.Url, url, post + data["filter"] = "" + yield Message.Url, url, data else: - post["filter"] = "watermark" - yield Message.Url, image["origin"], post + if not multi: + if len(post["multi"]) < len(post["image_list"]): + multi = self._data_from_post(post["item_id"]) + multi = multi["post_data"]["multi"] + else: + multi = post["multi"] + image = multi[data["num"] - 1] + + if image["origin"]: + data["filter"] = "watermark" + yield Message.Url, image["origin"], data if noop: - post["extension"] = "" - post["filter"] = "noop" - yield Message.Url, image["original_path"], post + data["extension"] = "" + data["filter"] = "noop" + yield Message.Url, image["original_path"], data - def _parse_post(self, post_id): + def posts(self): + """Returns an iterable with all relevant 'post' objects""" + + def _data_from_post(self, post_id): url = "{}/item/detail/{}".format(self.root, post_id) - response = self.request(url) - if response.status_code >= 400: - return None - - data = json.loads( - text.extract(response.text, 'JSON.parse("', '");')[0] + page = self.request(url).text + return json.loads( + text.extract(page, 'JSON.parse("', '");')[0] .replace('\\\\u002F', '/') .replace('\\"', '"') )["detail"] - post = data["post_data"] - if not post["multi"]: - return None - user = data["detail_user"] - - return { - "user": { - "id" : user["uid"], - "name" : user["uname"], - "avatar" : user["avatar"], - }, - "post": { - "id" : text.parse_int(post["item_id"]), - "tags" : [t["tag_name"] for t in post["post_tags"]], - "date" : text.parse_timestamp(post["ctime"]), - "parody" : text.parse_unicode_escapes(post["work"]), - "content": post["plain"], - "likes" : post["like_count"], - "shares" : post["share_count"], - "replies": post["reply_count"], - }, - "_multi": post["multi"], - } - class BcyUserExtractor(BcyExtractor): """Extractor for user timelines""" subcategory = "user" pattern = r"(?:https?://)?bcy\.net/u/(\d+)" - test = ("https://bcy.net/u/1933712", { - "pattern": r"https://img-bcy-qn.pstatp.com/\w+/\d+/post/\w+/\w+.jpg", - "count": ">= 25", - }) + test = ( + ("https://bcy.net/u/1933712", { + "pattern": r"https://img-bcy-qn.pstatp.com/\w+/\d+/post/\w+/.+jpg", + "count": ">= 25", + }), + ("https://bcy.net/u/109282764041", { + "pattern": r"https://p\d-bcy.byteimg.com/img/banciyuan/[0-9a-f]+" + r"~tplv-banciyuan-logo-v3:.+\.image", + "range": "1-25", + "count": 25, + }), + ) def posts(self): url = self.root + "/apiv3/user/selfPosts" - params = { - "uid": self.item_id, - "since": None, - # "_signature": None, - } + params = {"uid": self.item_id, "since": None} while True: data = self.request(url, params=params).json() item = None for item in data["data"]["items"]: - if item["item_detail"]["multi"]: - yield item["item_detail"]["item_id"] + yield item["item_detail"] if not item: return @@ -131,33 +138,51 @@ class BcyPostExtractor(BcyExtractor): """Extractor for individual posts""" subcategory = "post" pattern = r"(?:https?://)?bcy\.net/item/detail/(\d+)" - test = ("https://bcy.net/item/detail/6355835481002893070", { - "url": "301202375e61fd6e0e2e35de6c3ac9f74885dec3", - "count": 1, - "keyword": { - "user": { - "id" : 1933712, - "name" : "wukloo", - "avatar" : str, + test = ( + ("https://bcy.net/item/detail/6355835481002893070", { + "url": "301202375e61fd6e0e2e35de6c3ac9f74885dec3", + "count": 1, + "keyword": { + "user": { + "id" : 1933712, + "name" : "wukloo", + "avatar" : "re:https://img-bcy-qn.pstatp.com/Public/", + }, + "post": { + "id" : 6355835481002893070, + "tags" : list, + "date" : "dt:2016-11-22 08:47:46", + "parody" : "东方PROJECT", + "content": "re:根据微博的建议稍微做了点修改", + "likes" : int, + "shares" : int, + "replies": int, + }, + "id": 8330182, + "num": 1, + "width" : 3000, + "height": 1687, + "filename": "712e0780b09011e696f973c3d1568337", + "extension": "jpg", }, - "post": { - "id" : 6355835481002893070, - "tags" : list, - "date" : "dt:2016-11-22 08:47:46", - "parody" : "东方PROJECT", - "content": "re:根据微博的建议稍微做了点修改", - "likes" : int, - "shares" : int, - "replies": int, - }, - "id": 8330182, - "num": 1, - "width" : 3000, - "height": 1687, - "filename": "712e0780b09011e696f973c3d1568337", - "extension": "jpg", - }, - }) + }), + # only watermarked images available + ("https://bcy.net/item/detail/6780546160802143236", { + "pattern": r"https://p\d-bcy.byteimg.com/img/banciyuan/[0-9a-f]+" + r"~tplv-banciyuan-logo-v3:.+\.image", + "count": 8, + "keyword": {"filter": "watermark"} + }), + # only visible to logged in users + ("https://bcy.net/item/detail/6747523535150783495", { + "count": 0, + }), + ) def posts(self): - return (self.item_id,) + data = self._data_from_post(self.item_id) + post = data["post_data"] + post["image_list"] = post["multi"] + post["plain"] = text.parse_unicode_escapes(post["plain"]) + post.update(data["detail_user"]) + return (post,)