github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[smugmug] code cleanup

Browse Source 
- combine User and Node extractors
- (re)move miscellaneous helper functions
- rename "Owner" to "User"
This commit is contained in:
Mike Fährmann
2018-05-03 14:12:10 +02:00
parent 42ed7667b8
commit 3ce5296313
1 changed files with 101 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
gallery_dl/extractor/smugmug.py
View File

@@ -19,28 +19,43 @@ BASE_PATTERN = (
class SmugmugExtractor(Extractor):
"""Base class for smugmug extractors"""
category = "smugmug"
filename_fmt = "{category}_{Owner[NickName]}_{Image[ImageKey]}.{extension}"
filename_fmt = ("{category}_{User[NickName]}_"
"{Image[UploadKey]}_{Image[ImageKey]}.{extension}")
def __init__(self):
Extractor.__init__(self)
self.api = SmugmugAPI(self)
self.domain = None
self.user = None
def _resolve_user(self):
if not self.user:
self.user = self.api.site_user(self.domain)["NickName"]
@staticmethod
def _apply_largest(image, delete=True):
largest = image["Uris"]["LargestImage"]
if delete:
del image["Uris"]
for key in ("Url", "Width", "Height", "MD5", "Size", "Watermarked"):
if key in largest:
image[key] = largest[key]
return image["Url"]
class SmugmugAlbumExtractor(SmugmugExtractor):
"""Extractor for smugmug albums"""
subcategory = "album"
directory_fmt = ["{category}", "{Owner[NickName]}", "{Album[Name]}"]
directory_fmt = ["{category}", "{User[NickName]}", "{Album[Name]}"]
archive_fmt = "a_{Album[AlbumKey]}_{Image[ImageKey]}"
pattern = [r"smugmug:album:([^:]+)$"]
test = [("smugmug:album:xgkb4C", {
"url": "eb6133445064115ad83d32cbc6472520a2d24d53",
"content": "864f6953cb04121290407a579611bc5087d117ee",
})]
test = [
("smugmug:album:MN6kHH", {
"count": 0,
}),
("smugmug:album:6Ffcgk", {
"count": 1,
"pattern": ".*/i-L4CxBdg/0/33e0b290/X3/i-L4CxBdg-X3.jpg",
}),
("smugmug:album:drn76C", {
"count": 2,
"content": "864f6953cb04121290407a579611bc5087d117ee",
}),
]
def __init__(self, match):
SmugmugExtractor.__init__(self)
@@ -48,29 +63,30 @@ class SmugmugAlbumExtractor(SmugmugExtractor):
def items(self):
album = self.api.album(self.album_id, "User")
owner = album["Uris"]["User"]
user = album["Uris"]["User"]
del user["Uris"]
del album["Uris"]
del owner["Uris"]
data = {"Album": album, "Owner": owner}
data = {"Album": album, "User": user}
yield Message.Version, 1
yield Message.Directory, data
for image in self.api.album_images(self.album_id, "LargestImage"):
url = _apply_largest(image)
url = self._apply_largest(image)
data["Image"] = image
yield Message.Url, url, text.nameext_from_url(url, data)
class SmugmugImageExtractor(SmugmugExtractor):
"""Extractor for individual smugmug images"""
subcategory = "image"
directory_fmt = ["{category}", "{Owner[NickName]}"]
directory_fmt = ["{category}", "{User[NickName]}"]
archive_fmt = "{Image[ImageKey]}"
pattern = [BASE_PATTERN + r"(?:/[^/?&#]+)+/i-([^/?&#]+)"]
test = [("https://mikf.smugmug.com/Test/n-xnNH3s/i-L4CxBdg", {
"url": "905bfdef52ce1a731a4eae17e9ac348511e17ae4",
"keyword": "3fd6db2ab3d12a6d3cfc49ee57adc91fdd295a6c",
"keyword": "df63d36cfaeb128fda86802942d9a4271d3feafc",
"content": "626fe50d25fe49beeda15e116938db36e163c01f",
})]
@@ -80,12 +96,11 @@ class SmugmugImageExtractor(SmugmugExtractor):
def items(self):
image = self.api.image(self.image_id, "LargestImage,ImageOwner")
owner = image["Uris"]["ImageOwner"]
user = image["Uris"]["ImageOwner"]
url = self._apply_largest(image)
url = _apply_largest(image)
del owner["Uris"]
data = {"Image": image, "Owner": owner}
del user["Uris"]
data = {"Image": image, "User": user}
text.nameext_from_url(url, data)
yield Message.Version, 1
@@ -93,39 +108,10 @@ class SmugmugImageExtractor(SmugmugExtractor):
yield Message.Url, url, data
class SmugmugUserExtractor(SmugmugExtractor):
subcategory = "user"
pattern = [BASE_PATTERN + "(?:/browse)?/?$"]
test = [
("https://mikf.smugmug.com/", {
"pattern": "smugmug:album:xgkb4C$",
}),
("https://mikf.smugmug.com/browse", None),
("smugmug:https://www.creativedogportraits.com/", {
"pattern": "smugmug:album:txWXzs$",
}),
("smugmug:www.creativedogportraits.com/", None),
("smugmug:www.creativedogportraits.com/browse", None),
]
def __init__(self, match):
SmugmugExtractor.__init__(self)
self.domain = match.group(1)
self.user = match.group(2)
def items(self):
self._resolve_user()
yield Message.Version, 1
for album in self.api.user_albums(self.user):
uri = "smugmug:album:" + album["AlbumKey"]
yield Message.Queue, uri, album
class SmugmugNodeExtractor(SmugmugExtractor):
subcategory = "node"
pattern = [BASE_PATTERN +
r"((?:/[^/?&#a-z][^/?&#]*)+)"
r"(?:/n-([^/?&#]+))?/?$"]
class SmugmugPathExtractor(SmugmugExtractor):
"""Extractor for smugmug albums from URL paths and users"""
subcategory = "path"
pattern = [BASE_PATTERN + r"((?:/[^/?&#a-mo-z][^/?&#]*)*)/?$"]
test = [
("https://mikf.smugmug.com/Test/", {
"pattern": "smugmug:album:xgkb4C$",
@@ -133,30 +119,51 @@ class SmugmugNodeExtractor(SmugmugExtractor):
("https://mikf.smugmug.com/Test/n-xnNH3s", {
"pattern": "smugmug:album:xgkb4C$",
}),
("smugmug:https://www.creativedogportraits.com/PortfolioGallery/", {
("https://mikf.smugmug.com/", {
"count": 4,
"pattern": "smugmug:album:(xgkb4C|MN6kHH|6Ffcgk|drn76C)$",
}),
("smugmug:www.creativedogportraits.com/PortfolioGallery/", {
"pattern": "smugmug:album:txWXzs$",
}),
("smugmug:www.creativedogportraits.com/", {
"pattern": "smugmug:album:txWXzs$",
}),
("smugmug:https://www.creativedogportraits.com/", None),
]
def __init__(self, match):
SmugmugExtractor.__init__(self)
self.domain, self.user, self.path, self.node_id = match.groups()
self.domain, self.user, self.path = match.groups()
def items(self):
yield Message.Version, 1
if self.node_id:
node = self.api.node(self.node_id)
else:
self._resolve_user()
if not self.user:
self.user = self.api.site_user(self.domain)["NickName"]
if self.path:
data = self.api.user_urlpathlookup(self.user, self.path)
node = data["Uris"]["Node"]
nodes = (node,) if node["Type"] == "Album" else self.album_nodes(node)
for node in nodes:
yield Message.Queue, "smugmug:album:" + _get(node, "Album"), node
if node["Type"] == "Album":
nodes = (node,)
elif node["Type"] == "Folder":
nodes = self.album_nodes(node)
else:
nodes = ()
for node in nodes:
album_id = node["Uris"]["Album"].rpartition("/")[2]
yield Message.Queue, "smugmug:album:" + album_id, node
else:
for album in self.api.user_albums(self.user):
uri = "smugmug:album:" + album["AlbumKey"]
yield Message.Queue, uri, album
def album_nodes(self, root):
"""Yield all descendant album nodes of 'root'"""
for node in self.api.node_children(root["NodeID"]):
if node["Type"] == "Album":
yield node
@@ -213,7 +220,7 @@ class SmugmugAPI():
return self._pagination("user/" + username + "!albums", expands)
def site_user(self, domain):
return _unwrap(self._call("!siteuser", domain=domain)["Response"])
return self._call("!siteuser", domain=domain)["Response"]["User"]
def user_urlpathlookup(self, username, path):
endpoint = "user/" + username + "!urlpathlookup"
@@ -242,76 +249,50 @@ class SmugmugAPI():
raise exception.StopExtraction()
def _expansion(self, endpoint, expands, params=None):
if expands:
endpoint += "?_expand=" + expands
return _apply_expansions(self._call(endpoint, params), expands)
endpoint = self._extend(endpoint, expands)
result = self._apply_expansions(self._call(endpoint, params), expands)
if not result:
raise exception.NotFoundError()
return result[0]
def _pagination(self, endpoint, expands=None):
if expands:
endpoint += "?_expand=" + expands
endpoint = self._extend(endpoint, expands)
params = {"start": 1, "count": 100}
while True:
data = self._call(endpoint, params)
yield from _apply_expansions_iter(data, expands)
yield from self._apply_expansions(data, expands)
if "NextPage" not in data["Response"]["Pages"]:
return
params["start"] += params["count"]
@staticmethod
def _extend(endpoint, expands):
if expands:
endpoint += "?_expand=" + expands
return endpoint
def _apply_largest(image, delete=True):
largest = image["Uris"]["LargestImage"]
if delete:
del image["Uris"]
for key in ("Url", "Width", "Height", "MD5", "Size", "Watermarked"):
if key in largest:
image[key] = largest[key]
return image["Url"]
@staticmethod
def _apply_expansions(data, expands):
def unwrap(response):
locator = response["Locator"]
return response[locator] if locator in response else []
def _get(obj, key):
return obj["Uris"][key].rpartition("/")[2]
objs = unwrap(data["Response"])
if not isinstance(objs, list):
objs = (objs,)
if "Expansions" in data:
expansions = data["Expansions"]
expands = expands.split(",")
def _apply_expansions(data, expands):
obj = _unwrap(data["Response"])
for obj in objs:
uris = obj["Uris"]
if "Expansions" in data:
expansions = data["Expansions"]
uris = obj["Uris"]
for name in expands:
uri = uris[name]
uris[name] = unwrap(expansions[uri])
for name in expands.split(","):
uri = uris[name]
uris[name] = _unwrap(expansions[uri])
return obj
def _apply_expansions_iter(data, expands):
objs = _unwrap_iter(data["Response"])
if "Expansions" in data:
expansions = data["Expansions"]
expands = expands.split(",")
for obj in objs:
uris = obj["Uris"]
for name in expands:
uri = uris[name]
uris[name] = _unwrap(expansions[uri])
return objs
def _unwrap(response):
locator = response["Locator"]
return response[locator] if locator in response else []
def _unwrap_iter(response):
obj = _unwrap(response)
if isinstance(obj, list):
return obj
return (obj,)
return objs