From 3ce52963131e7fdcc7e47f18ec7c94fcb54766a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 3 May 2018 14:12:10 +0200 Subject: [PATCH] [smugmug] code cleanup - combine User and Node extractors - (re)move miscellaneous helper functions - rename "Owner" to "User" --- gallery_dl/extractor/smugmug.py | 221 +++++++++++++++----------------- 1 file changed, 101 insertions(+), 120 deletions(-) diff --git a/gallery_dl/extractor/smugmug.py b/gallery_dl/extractor/smugmug.py index 6f8d1be0..3306bc58 100644 --- a/gallery_dl/extractor/smugmug.py +++ b/gallery_dl/extractor/smugmug.py @@ -19,28 +19,43 @@ BASE_PATTERN = ( class SmugmugExtractor(Extractor): """Base class for smugmug extractors""" category = "smugmug" - filename_fmt = "{category}_{Owner[NickName]}_{Image[ImageKey]}.{extension}" + filename_fmt = ("{category}_{User[NickName]}_" + "{Image[UploadKey]}_{Image[ImageKey]}.{extension}") def __init__(self): Extractor.__init__(self) self.api = SmugmugAPI(self) - self.domain = None - self.user = None - def _resolve_user(self): - if not self.user: - self.user = self.api.site_user(self.domain)["NickName"] + @staticmethod + def _apply_largest(image, delete=True): + largest = image["Uris"]["LargestImage"] + if delete: + del image["Uris"] + for key in ("Url", "Width", "Height", "MD5", "Size", "Watermarked"): + if key in largest: + image[key] = largest[key] + return image["Url"] class SmugmugAlbumExtractor(SmugmugExtractor): + """Extractor for smugmug albums""" subcategory = "album" - directory_fmt = ["{category}", "{Owner[NickName]}", "{Album[Name]}"] + directory_fmt = ["{category}", "{User[NickName]}", "{Album[Name]}"] archive_fmt = "a_{Album[AlbumKey]}_{Image[ImageKey]}" pattern = [r"smugmug:album:([^:]+)$"] - test = [("smugmug:album:xgkb4C", { - "url": "eb6133445064115ad83d32cbc6472520a2d24d53", - "content": "864f6953cb04121290407a579611bc5087d117ee", - })] + test = [ + ("smugmug:album:MN6kHH", { + "count": 0, + }), + ("smugmug:album:6Ffcgk", { + "count": 1, + "pattern": ".*/i-L4CxBdg/0/33e0b290/X3/i-L4CxBdg-X3.jpg", + }), + ("smugmug:album:drn76C", { + "count": 2, + "content": "864f6953cb04121290407a579611bc5087d117ee", + }), + ] def __init__(self, match): SmugmugExtractor.__init__(self) @@ -48,29 +63,30 @@ class SmugmugAlbumExtractor(SmugmugExtractor): def items(self): album = self.api.album(self.album_id, "User") - owner = album["Uris"]["User"] + user = album["Uris"]["User"] + del user["Uris"] del album["Uris"] - del owner["Uris"] - data = {"Album": album, "Owner": owner} + data = {"Album": album, "User": user} yield Message.Version, 1 yield Message.Directory, data for image in self.api.album_images(self.album_id, "LargestImage"): - url = _apply_largest(image) + url = self._apply_largest(image) data["Image"] = image yield Message.Url, url, text.nameext_from_url(url, data) class SmugmugImageExtractor(SmugmugExtractor): + """Extractor for individual smugmug images""" subcategory = "image" - directory_fmt = ["{category}", "{Owner[NickName]}"] + directory_fmt = ["{category}", "{User[NickName]}"] archive_fmt = "{Image[ImageKey]}" pattern = [BASE_PATTERN + r"(?:/[^/?&#]+)+/i-([^/?&#]+)"] test = [("https://mikf.smugmug.com/Test/n-xnNH3s/i-L4CxBdg", { "url": "905bfdef52ce1a731a4eae17e9ac348511e17ae4", - "keyword": "3fd6db2ab3d12a6d3cfc49ee57adc91fdd295a6c", + "keyword": "df63d36cfaeb128fda86802942d9a4271d3feafc", "content": "626fe50d25fe49beeda15e116938db36e163c01f", })] @@ -80,12 +96,11 @@ class SmugmugImageExtractor(SmugmugExtractor): def items(self): image = self.api.image(self.image_id, "LargestImage,ImageOwner") - owner = image["Uris"]["ImageOwner"] + user = image["Uris"]["ImageOwner"] + url = self._apply_largest(image) - url = _apply_largest(image) - - del owner["Uris"] - data = {"Image": image, "Owner": owner} + del user["Uris"] + data = {"Image": image, "User": user} text.nameext_from_url(url, data) yield Message.Version, 1 @@ -93,39 +108,10 @@ class SmugmugImageExtractor(SmugmugExtractor): yield Message.Url, url, data -class SmugmugUserExtractor(SmugmugExtractor): - subcategory = "user" - pattern = [BASE_PATTERN + "(?:/browse)?/?$"] - test = [ - ("https://mikf.smugmug.com/", { - "pattern": "smugmug:album:xgkb4C$", - }), - ("https://mikf.smugmug.com/browse", None), - ("smugmug:https://www.creativedogportraits.com/", { - "pattern": "smugmug:album:txWXzs$", - }), - ("smugmug:www.creativedogportraits.com/", None), - ("smugmug:www.creativedogportraits.com/browse", None), - ] - - def __init__(self, match): - SmugmugExtractor.__init__(self) - self.domain = match.group(1) - self.user = match.group(2) - - def items(self): - self._resolve_user() - yield Message.Version, 1 - for album in self.api.user_albums(self.user): - uri = "smugmug:album:" + album["AlbumKey"] - yield Message.Queue, uri, album - - -class SmugmugNodeExtractor(SmugmugExtractor): - subcategory = "node" - pattern = [BASE_PATTERN + - r"((?:/[^/?&#a-z][^/?&#]*)+)" - r"(?:/n-([^/?&#]+))?/?$"] +class SmugmugPathExtractor(SmugmugExtractor): + """Extractor for smugmug albums from URL paths and users""" + subcategory = "path" + pattern = [BASE_PATTERN + r"((?:/[^/?&#a-mo-z][^/?&#]*)*)/?$"] test = [ ("https://mikf.smugmug.com/Test/", { "pattern": "smugmug:album:xgkb4C$", @@ -133,30 +119,51 @@ class SmugmugNodeExtractor(SmugmugExtractor): ("https://mikf.smugmug.com/Test/n-xnNH3s", { "pattern": "smugmug:album:xgkb4C$", }), - ("smugmug:https://www.creativedogportraits.com/PortfolioGallery/", { + ("https://mikf.smugmug.com/", { + "count": 4, + "pattern": "smugmug:album:(xgkb4C|MN6kHH|6Ffcgk|drn76C)$", + }), + ("smugmug:www.creativedogportraits.com/PortfolioGallery/", { "pattern": "smugmug:album:txWXzs$", }), + ("smugmug:www.creativedogportraits.com/", { + "pattern": "smugmug:album:txWXzs$", + }), + ("smugmug:https://www.creativedogportraits.com/", None), ] def __init__(self, match): SmugmugExtractor.__init__(self) - self.domain, self.user, self.path, self.node_id = match.groups() + self.domain, self.user, self.path = match.groups() def items(self): yield Message.Version, 1 - if self.node_id: - node = self.api.node(self.node_id) - else: - self._resolve_user() + if not self.user: + self.user = self.api.site_user(self.domain)["NickName"] + + if self.path: data = self.api.user_urlpathlookup(self.user, self.path) node = data["Uris"]["Node"] - nodes = (node,) if node["Type"] == "Album" else self.album_nodes(node) - for node in nodes: - yield Message.Queue, "smugmug:album:" + _get(node, "Album"), node + if node["Type"] == "Album": + nodes = (node,) + elif node["Type"] == "Folder": + nodes = self.album_nodes(node) + else: + nodes = () + + for node in nodes: + album_id = node["Uris"]["Album"].rpartition("/")[2] + yield Message.Queue, "smugmug:album:" + album_id, node + + else: + for album in self.api.user_albums(self.user): + uri = "smugmug:album:" + album["AlbumKey"] + yield Message.Queue, uri, album def album_nodes(self, root): + """Yield all descendant album nodes of 'root'""" for node in self.api.node_children(root["NodeID"]): if node["Type"] == "Album": yield node @@ -213,7 +220,7 @@ class SmugmugAPI(): return self._pagination("user/" + username + "!albums", expands) def site_user(self, domain): - return _unwrap(self._call("!siteuser", domain=domain)["Response"]) + return self._call("!siteuser", domain=domain)["Response"]["User"] def user_urlpathlookup(self, username, path): endpoint = "user/" + username + "!urlpathlookup" @@ -242,76 +249,50 @@ class SmugmugAPI(): raise exception.StopExtraction() def _expansion(self, endpoint, expands, params=None): - if expands: - endpoint += "?_expand=" + expands - return _apply_expansions(self._call(endpoint, params), expands) + endpoint = self._extend(endpoint, expands) + result = self._apply_expansions(self._call(endpoint, params), expands) + if not result: + raise exception.NotFoundError() + return result[0] def _pagination(self, endpoint, expands=None): - if expands: - endpoint += "?_expand=" + expands + endpoint = self._extend(endpoint, expands) params = {"start": 1, "count": 100} while True: data = self._call(endpoint, params) - yield from _apply_expansions_iter(data, expands) + yield from self._apply_expansions(data, expands) if "NextPage" not in data["Response"]["Pages"]: return params["start"] += params["count"] + @staticmethod + def _extend(endpoint, expands): + if expands: + endpoint += "?_expand=" + expands + return endpoint -def _apply_largest(image, delete=True): - largest = image["Uris"]["LargestImage"] - if delete: - del image["Uris"] - for key in ("Url", "Width", "Height", "MD5", "Size", "Watermarked"): - if key in largest: - image[key] = largest[key] - return image["Url"] + @staticmethod + def _apply_expansions(data, expands): + def unwrap(response): + locator = response["Locator"] + return response[locator] if locator in response else [] -def _get(obj, key): - return obj["Uris"][key].rpartition("/")[2] + objs = unwrap(data["Response"]) + if not isinstance(objs, list): + objs = (objs,) + if "Expansions" in data: + expansions = data["Expansions"] + expands = expands.split(",") -def _apply_expansions(data, expands): - obj = _unwrap(data["Response"]) + for obj in objs: + uris = obj["Uris"] - if "Expansions" in data: - expansions = data["Expansions"] - uris = obj["Uris"] + for name in expands: + uri = uris[name] + uris[name] = unwrap(expansions[uri]) - for name in expands.split(","): - uri = uris[name] - uris[name] = _unwrap(expansions[uri]) - - return obj - - -def _apply_expansions_iter(data, expands): - objs = _unwrap_iter(data["Response"]) - - if "Expansions" in data: - expansions = data["Expansions"] - expands = expands.split(",") - - for obj in objs: - uris = obj["Uris"] - - for name in expands: - uri = uris[name] - uris[name] = _unwrap(expansions[uri]) - - return objs - - -def _unwrap(response): - locator = response["Locator"] - return response[locator] if locator in response else [] - - -def _unwrap_iter(response): - obj = _unwrap(response) - if isinstance(obj, list): - return obj - return (obj,) + return objs