[tumblr] small improvements
- don't transform inline GIF URLs - set 'type' parameter for API calls if there is only one post type selected
This commit is contained in:
@@ -15,6 +15,8 @@ import re
|
|||||||
|
|
||||||
|
|
||||||
def _original_image(url):
|
def _original_image(url):
|
||||||
|
if url.endswith(".gif") and "_inline_" in url:
|
||||||
|
return url
|
||||||
return re.sub(
|
return re.sub(
|
||||||
(r"https?://\d+\.media\.tumblr\.com"
|
(r"https?://\d+\.media\.tumblr\.com"
|
||||||
r"/([0-9a-f]+)/tumblr_([^/?&#.]+)_\d+\.([0-9a-z]+)"),
|
r"/([0-9a-f]+)/tumblr_([^/?&#.]+)_\d+\.([0-9a-z]+)"),
|
||||||
@@ -45,24 +47,14 @@ class TumblrExtractor(Extractor):
|
|||||||
self.user = match.group(1)
|
self.user = match.group(1)
|
||||||
self.api = TumblrAPI(self)
|
self.api = TumblrAPI(self)
|
||||||
|
|
||||||
|
self.types = self._setup_posttypes()
|
||||||
self.inline = self.config("inline", False)
|
self.inline = self.config("inline", False)
|
||||||
self.external = self.config("external", False)
|
self.external = self.config("external", False)
|
||||||
|
|
||||||
types = self.config("posts", ("photo",))
|
if len(self.types) == 1:
|
||||||
if types == "all":
|
self.api.params["type"] = next(iter(self.types))
|
||||||
self.types = POST_TYPES
|
elif not self.types:
|
||||||
elif types:
|
self.log.warning("no valid post types selected")
|
||||||
if isinstance(types, str):
|
|
||||||
types = types.split(",")
|
|
||||||
self.types = frozenset(types)
|
|
||||||
|
|
||||||
invalid = self.types - POST_TYPES
|
|
||||||
if invalid:
|
|
||||||
self.log.warning('invalid post types: "%s"',
|
|
||||||
'", "'.join(sorted(invalid)))
|
|
||||||
else:
|
|
||||||
self.types = frozenset()
|
|
||||||
self.log.warning("no post types selected")
|
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
blog = self.api.info(self.user)
|
blog = self.api.info(self.user)
|
||||||
@@ -111,6 +103,27 @@ class TumblrExtractor(Extractor):
|
|||||||
def posts(self):
|
def posts(self):
|
||||||
"""Return an iterable containing all relevant posts"""
|
"""Return an iterable containing all relevant posts"""
|
||||||
|
|
||||||
|
def _setup_posttypes(self):
|
||||||
|
types = self.config("posts", ("photo",))
|
||||||
|
|
||||||
|
if types == "all":
|
||||||
|
return POST_TYPES
|
||||||
|
|
||||||
|
elif not types:
|
||||||
|
return frozenset()
|
||||||
|
|
||||||
|
else:
|
||||||
|
if isinstance(types, str):
|
||||||
|
types = types.split(",")
|
||||||
|
types = frozenset(types)
|
||||||
|
|
||||||
|
invalid = types - POST_TYPES
|
||||||
|
if invalid:
|
||||||
|
types = types & POST_TYPES
|
||||||
|
self.log.warning('invalid post types: "%s"',
|
||||||
|
'", "'.join(sorted(invalid)))
|
||||||
|
return types
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _prepare(url, post):
|
def _prepare(url, post):
|
||||||
post["offset"] += 1
|
post["offset"] += 1
|
||||||
@@ -152,11 +165,14 @@ class TumblrPostExtractor(TumblrExtractor):
|
|||||||
def __init__(self, match):
|
def __init__(self, match):
|
||||||
TumblrExtractor.__init__(self, match)
|
TumblrExtractor.__init__(self, match)
|
||||||
self.post_id = match.group(2)
|
self.post_id = match.group(2)
|
||||||
self.types = POST_TYPES
|
|
||||||
|
|
||||||
def posts(self):
|
def posts(self):
|
||||||
return self.api.posts(self.user, {"id": self.post_id})
|
return self.api.posts(self.user, {"id": self.post_id})
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _setup_posttypes():
|
||||||
|
return POST_TYPES
|
||||||
|
|
||||||
|
|
||||||
class TumblrTagExtractor(TumblrExtractor):
|
class TumblrTagExtractor(TumblrExtractor):
|
||||||
"""Extractor for images from a tumblr-user by tag"""
|
"""Extractor for images from a tumblr-user by tag"""
|
||||||
|
|||||||
@@ -83,6 +83,7 @@ skip = [
|
|||||||
"archivedmoe", "archiveofsins", "thebarchive",
|
"archivedmoe", "archiveofsins", "thebarchive",
|
||||||
# temporary issues
|
# temporary issues
|
||||||
"mangazuki",
|
"mangazuki",
|
||||||
|
"yeet",
|
||||||
]
|
]
|
||||||
# enable selective testing for direct calls
|
# enable selective testing for direct calls
|
||||||
if __name__ == '__main__' and len(sys.argv) > 1:
|
if __name__ == '__main__' and len(sys.argv) > 1:
|
||||||
|
|||||||
Reference in New Issue
Block a user