[patreon] update code to 3.8
This commit is contained in:
@@ -35,8 +35,7 @@ class PatreonExtractor(Extractor):
|
|||||||
self.session.headers["User-Agent"] = \
|
self.session.headers["User-Agent"] = \
|
||||||
"Patreon/7.6.28 (Android; Android 11; Scale/2.10)"
|
"Patreon/7.6.28 (Android; Android 11; Scale/2.10)"
|
||||||
|
|
||||||
format_images = self.config("format-images")
|
if format_images := self.config("format-images"):
|
||||||
if format_images:
|
|
||||||
self._images_fmt = format_images
|
self._images_fmt = format_images
|
||||||
self._images_url = self._images_url_fmt
|
self._images_url = self._images_url_fmt
|
||||||
|
|
||||||
@@ -73,11 +72,9 @@ class PatreonExtractor(Extractor):
|
|||||||
self.log.debug("skipping %s (%s %s)", url, fhash, kind)
|
self.log.debug("skipping %s (%s %s)", url, fhash, kind)
|
||||||
|
|
||||||
def _postfile(self, post):
|
def _postfile(self, post):
|
||||||
postfile = post.get("post_file")
|
if postfile := post.get("post_file"):
|
||||||
if postfile:
|
|
||||||
url = postfile["url"]
|
url = postfile["url"]
|
||||||
name = postfile.get("name")
|
if not (name := postfile.get("name")):
|
||||||
if not name:
|
|
||||||
if url.startswith("https://stream.mux.com/"):
|
if url.startswith("https://stream.mux.com/"):
|
||||||
name = url
|
name = url
|
||||||
else:
|
else:
|
||||||
@@ -86,11 +83,11 @@ class PatreonExtractor(Extractor):
|
|||||||
return ()
|
return ()
|
||||||
|
|
||||||
def _images(self, post):
|
def _images(self, post):
|
||||||
for image in post.get("images") or ():
|
if images := post.get("images"):
|
||||||
url = self._images_url(image)
|
for image in images:
|
||||||
if url:
|
if url := self._images_url(image):
|
||||||
name = image.get("file_name") or self._filename(url) or url
|
name = image.get("file_name") or self._filename(url) or url
|
||||||
yield "image", url, name
|
yield "image", url, name
|
||||||
|
|
||||||
def _images_url(self, image):
|
def _images_url(self, image):
|
||||||
return image.get("download_url")
|
return image.get("download_url")
|
||||||
@@ -102,32 +99,26 @@ class PatreonExtractor(Extractor):
|
|||||||
return image.get("download_url")
|
return image.get("download_url")
|
||||||
|
|
||||||
def _image_large(self, post):
|
def _image_large(self, post):
|
||||||
image = post.get("image")
|
if image := post.get("image"):
|
||||||
if image:
|
if url := image.get("large_url"):
|
||||||
url = image.get("large_url")
|
|
||||||
if url:
|
|
||||||
name = image.get("file_name") or self._filename(url) or url
|
name = image.get("file_name") or self._filename(url) or url
|
||||||
return (("image_large", url, name),)
|
return (("image_large", url, name),)
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
def _attachments(self, post):
|
def _attachments(self, post):
|
||||||
for attachment in post.get("attachments") or ():
|
for attachment in post.get("attachments") or ():
|
||||||
url = self.request_location(attachment["url"], fatal=False)
|
if url := self.request_location(attachment["url"], fatal=False):
|
||||||
if url:
|
|
||||||
yield "attachment", url, attachment["name"]
|
yield "attachment", url, attachment["name"]
|
||||||
|
|
||||||
for attachment in post.get("attachments_media") or ():
|
for attachment in post.get("attachments_media") or ():
|
||||||
url = attachment.get("download_url")
|
if url := attachment.get("download_url"):
|
||||||
if url:
|
|
||||||
yield "attachment", url, attachment["file_name"]
|
yield "attachment", url, attachment["file_name"]
|
||||||
|
|
||||||
def _content(self, post):
|
def _content(self, post):
|
||||||
content = post.get("content")
|
if content := post.get("content"):
|
||||||
if content:
|
|
||||||
for img in text.extract_iter(
|
for img in text.extract_iter(
|
||||||
content, '<img data-media-id="', '>'):
|
content, '<img data-media-id="', '>'):
|
||||||
url = text.extr(img, 'src="', '"')
|
if url := text.extr(img, 'src="', '"'):
|
||||||
if url:
|
|
||||||
yield "content", url, self._filename(url) or url
|
yield "content", url, self._filename(url) or url
|
||||||
|
|
||||||
def posts(self):
|
def posts(self):
|
||||||
@@ -234,7 +225,7 @@ class PatreonExtractor(Extractor):
|
|||||||
|
|
||||||
def _build_url(self, endpoint, query):
|
def _build_url(self, endpoint, query):
|
||||||
return (
|
return (
|
||||||
"https://www.patreon.com/api/" + endpoint +
|
f"https://www.patreon.com/api/{endpoint}"
|
||||||
|
|
||||||
"?include=campaign,access_rules,attachments,attachments_media,"
|
"?include=campaign,access_rules,attachments,attachments_media,"
|
||||||
"audio,images,media,native_video_insights,poll.choices,"
|
"audio,images,media,native_video_insights,poll.choices,"
|
||||||
@@ -264,7 +255,7 @@ class PatreonExtractor(Extractor):
|
|||||||
"&fields[media]=id,image_urls,download_url,metadata,file_name"
|
"&fields[media]=id,image_urls,download_url,metadata,file_name"
|
||||||
"&fields[native_video_insights]=average_view_duration,"
|
"&fields[native_video_insights]=average_view_duration,"
|
||||||
"average_view_pct,has_preview,id,last_updated_at,num_views,"
|
"average_view_pct,has_preview,id,last_updated_at,num_views,"
|
||||||
"preview_views,video_duration" + query +
|
f"preview_views,video_duration{query}"
|
||||||
|
|
||||||
"&json-api-version=1.0"
|
"&json-api-version=1.0"
|
||||||
)
|
)
|
||||||
@@ -311,7 +302,7 @@ class PatreonExtractor(Extractor):
|
|||||||
data = text.extr(page, "window.patreon = {", "};\n")
|
data = text.extr(page, "window.patreon = {", "};\n")
|
||||||
if data:
|
if data:
|
||||||
try:
|
try:
|
||||||
return util.json_loads("{" + data + "}")["bootstrap"]
|
return util.json_loads(f"{{{data}}}")["bootstrap"]
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -338,10 +329,10 @@ class PatreonCreatorExtractor(PatreonExtractor):
|
|||||||
self.log.debug("campaign_id: %s", campaign_id)
|
self.log.debug("campaign_id: %s", campaign_id)
|
||||||
|
|
||||||
url = self._build_url("posts", (
|
url = self._build_url("posts", (
|
||||||
"&filter[campaign_id]=" + campaign_id +
|
f"&filter[campaign_id]={campaign_id}"
|
||||||
"&filter[contains_exclusive_posts]=true"
|
"&filter[contains_exclusive_posts]=true"
|
||||||
"&filter[is_draft]=false" + filters +
|
"&filter[is_draft]=false"
|
||||||
"&sort=" + query.get("sort", "-published_at")
|
f"{filters}&sort={query.get('sort', '-published_at')}"
|
||||||
))
|
))
|
||||||
return self._pagination(url)
|
return self._pagination(url)
|
||||||
|
|
||||||
@@ -353,11 +344,10 @@ class PatreonCreatorExtractor(PatreonExtractor):
|
|||||||
if campaign_id:
|
if campaign_id:
|
||||||
return campaign_id
|
return campaign_id
|
||||||
|
|
||||||
user_id = query.get("u")
|
if user_id := query.get("u"):
|
||||||
if user_id:
|
url = f"{self.root}/user?u={user_id}"
|
||||||
url = "{}/user?u={}".format(self.root, user_id)
|
|
||||||
else:
|
else:
|
||||||
url = "{}/{}".format(self.root, creator)
|
url = f"{self.root}/{creator}"
|
||||||
page = self.request(url, notfound="creator").text
|
page = self.request(url, notfound="creator").text
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -373,7 +363,7 @@ class PatreonCreatorExtractor(PatreonExtractor):
|
|||||||
|
|
||||||
def _get_filters(self, query):
|
def _get_filters(self, query):
|
||||||
return "".join(
|
return "".join(
|
||||||
"&filter[{}={}".format(key[8:], text.escape(value))
|
f"&filter[{key[8:]}={text.escape(value)}"
|
||||||
for key, value in query.items()
|
for key, value in query.items()
|
||||||
if key.startswith("filters[")
|
if key.startswith("filters[")
|
||||||
)
|
)
|
||||||
@@ -401,7 +391,7 @@ class PatreonPostExtractor(PatreonExtractor):
|
|||||||
example = "https://www.patreon.com/posts/TITLE-12345"
|
example = "https://www.patreon.com/posts/TITLE-12345"
|
||||||
|
|
||||||
def posts(self):
|
def posts(self):
|
||||||
url = "{}/posts/{}".format(self.root, self.groups[0])
|
url = f"{self.root}/posts/{self.groups[0]}"
|
||||||
page = self.request(url, notfound="post").text
|
page = self.request(url, notfound="post").text
|
||||||
bootstrap = self._extract_bootstrap(page)
|
bootstrap = self._extract_bootstrap(page)
|
||||||
|
|
||||||
|
|||||||
@@ -119,7 +119,7 @@ __tests__ = (
|
|||||||
"#url" : "https://www.patreon.com/posts/free-mari-8s-113049301",
|
"#url" : "https://www.patreon.com/posts/free-mari-8s-113049301",
|
||||||
"#comment" : "'This page has been removed' - postfile + attachments_media (#6241)",
|
"#comment" : "'This page has been removed' - postfile + attachments_media (#6241)",
|
||||||
"#class" : patreon.PatreonPostExtractor,
|
"#class" : patreon.PatreonPostExtractor,
|
||||||
"#count" : 0,
|
"#exception": exception.NotFoundError,
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -138,7 +138,7 @@ __tests__ = (
|
|||||||
"tags": ["AWMedia"],
|
"tags": ["AWMedia"],
|
||||||
"campaign": {
|
"campaign": {
|
||||||
"avatar_photo_image_urls": dict,
|
"avatar_photo_image_urls": dict,
|
||||||
"avatar_photo_url": "https://c10.patreonusercontent.com/4/patreon-media/p/campaign/350434/cadc16f03fa1460f9185505b0a858c1b/eyJ3IjoyMDB9/1.png?token-time=2145916800&token-hash=yBXVH1-UXYOUow9qRey-I6eJe8PcuRQDDKhw730g5jc%3D",
|
"avatar_photo_url": "https://c10.patreonusercontent.com/4/patreon-media/p/campaign/350434/cadc16f03fa1460f9185505b0a858c1b/eyJ3Ijo2MjB9/1.png?token-hash=tpUv_bM0-mEuUSizstb00UrVA-btPS5RyGSCWRx24oc%3D",
|
||||||
"creation_name": "creating Art Photography/Videography",
|
"creation_name": "creating Art Photography/Videography",
|
||||||
"currency": "USD",
|
"currency": "USD",
|
||||||
"current_user_can_be_free_member": True,
|
"current_user_can_be_free_member": True,
|
||||||
@@ -164,7 +164,10 @@ __tests__ = (
|
|||||||
"#class" : patreon.PatreonPostExtractor,
|
"#class" : patreon.PatreonPostExtractor,
|
||||||
"#options" : {"format-images": "thumbnail"},
|
"#options" : {"format-images": "thumbnail"},
|
||||||
"#pattern" : r"https://c10.patreonusercontent.com/4/patreon-media/p/post/12497641/3d99f5f5b635428ca237fedf0f223f1a/eyJoIjozNjAsInciOjM2MH0%3D/1\.JPG\?.+",
|
"#pattern" : r"https://c10.patreonusercontent.com/4/patreon-media/p/post/12497641/3d99f5f5b635428ca237fedf0f223f1a/eyJoIjozNjAsInciOjM2MH0%3D/1\.JPG\?.+",
|
||||||
"#sha1_content": "190e249295eeca1a8ffbcf1aece788b4f69bbb64",
|
"#sha1_content": (
|
||||||
|
"2967d7567d55debdfa59cfd27cd5edf89d9c3503",
|
||||||
|
"190e249295eeca1a8ffbcf1aece788b4f69bbb64",
|
||||||
|
)
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user