[500px] fix extraction and update URL patterns (fixes #956)

- rewrite most API calls to GraphQL queries
- match '500px.com/p/<user>' URLs

gallery_dl/extractor/500px.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2019 Mike Fährmann
+# Copyright 2019-2020 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -9,7 +9,7 @@
 """Extractors for https://500px.com/"""
 
 from .common import Extractor, Message
-from .. import text
+import json
 
 
 BASE_PATTERN = r"(?:https?://)?(?:web\.)?500px\.com"
@@ -48,7 +48,7 @@ class _500pxExtractor(Extractor):
     def photos(self):
         """Returns an iterable containing all relevant photo IDs"""
 
-    def _extend(self, photos):
+    def _extend(self, edges):
         """Extend photos with additional metadata and higher resolution URLs"""
         url = "https://api.500px.com/v1/photos"
         params = {
@@ -62,40 +62,42 @@ class _500pxExtractor(Extractor):
             "liked_by"              : "1",
             "following_sample"      : "100",
             "image_size"            : "4096",
-            "ids"                   : ",".join(str(p["id"]) for p in photos),
+            "ids"                   : ",".join(
+                str(edge["node"]["legacyId"]) for edge in edges),
         }
 
-        data = self._api_call(url, params)["photos"]
+        data = self._request_api(url, params)["photos"]
-        for photo in photos:
+        return [
-            pid = str(photo["id"])
+            data[str(edge["node"]["legacyId"])]
-            photo.update(data[pid])
+            for edge in edges
-        return photos
+        ]
 
-    def _api_call(self, url, params, csrf_token=None):
+    def _request_api(self, url, params, csrf_token=None):
         headers = {"Origin": self.root, "X-CSRF-Token": csrf_token}
         return self.request(url, headers=headers, params=params).json()
 
-    def _pagination(self, url, params, csrf):
+    def _request_graphql(self, opname, variables, query_hash):
-        params["page"] = 1
+        url = "https://api.500px.com/graphql"
-        while True:
+        params = {
-            data = self._api_call(url, params, csrf)
+            "operationName": opname,
-            yield from self._extend(data["photos"])
+            "variables"    : json.dumps(variables),
-
+            "extensions"   : '{"persistedQuery":{"version":1'
-            if params["page"] >= data["total_pages"]:
+                             ',"sha256Hash":"' + query_hash + '"}}',
-                return
+        }
-            params["page"] += 1
+        return self.request(url, params=params).json()["data"]
 
 
 class _500pxUserExtractor(_500pxExtractor):
     """Extractor for photos from a user's photostream on 500px.com"""
     subcategory = "user"
-    pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/?(?:$|\?|#)"
+    pattern = BASE_PATTERN + r"/(?!photo/)(?:p/)?([^/?&#]+)/?(?:$|\?|#)"
     test = (
-        ("https://500px.com/light_expression_photography", {
+        ("https://500px.com/p/light_expression_photography", {
             "pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2",
             "range": "1-99",
             "count": 99,
         }),
+        ("https://500px.com/light_expression_photography"),
         ("https://web.500px.com/light_expression_photography"),
     )
 
@@ -104,72 +106,97 @@ class _500pxUserExtractor(_500pxExtractor):
         self.user = match.group(1)
 
     def photos(self):
-        # get csrf token and user id from webpage
+        variables = {"username": self.user, "pageSize": 20}
-        url = "{}/{}".format(self.root, self.user)
+        photos = self._request_graphql(
-        page = self.request(url).text
+            "OtherPhotosQuery", variables,
-        csrf_token, pos = text.extract(page, 'csrf-token" content="', '"')
+            "54524abbdc809f8d4e10d37839e8ab2d"
-        user_id   , pos = text.extract(page, '/user/', '"', pos)
+            "3035413688cad9c7fbece13b66637e9d",
+        )["user"]["photos"]
 
-        # get user photos
+        while True:
-        url = "https://api.500px.com/v1/photos"
+            yield from self._extend(photos["edges"])
-        params = {
+
-            "feature"       : "user",
+            if not photos["pageInfo"]["hasNextPage"]:
-            "stream"        : "photos",
+                return
-            "rpp"           : "50",
+
-            "user_id"       : user_id,
+            variables["cursor"] = photos["pageInfo"]["endCursor"]
-        }
+            photos = self._request_graphql(
-        return self._pagination(url, params, csrf_token)
+                "OtherPhotosPaginationContainerQuery", variables,
+                "6d31e01104456ce642a2c6fc2f936812"
+                "b0f2a65c442d03e1521d769c20efe507",
+            )["userByUsername"]["photos"]
 
 
 class _500pxGalleryExtractor(_500pxExtractor):
     """Extractor for photo galleries on 500px.com"""
     subcategory = "gallery"
     directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}")
-    pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/galleries/([^/?&#]+)"
+    pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?"
-    test = ("https://500px.com/fashvamp/galleries/lera", {
+               r"([^/?&#]+)/galleries/([^/?&#]+)")
-        "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6",
+    test = (
-        "count": 3,
+        ("https://500px.com/p/fashvamp/galleries/lera", {
-        "keyword": {
+            "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6",
-            "gallery": dict,
+            "count": 3,
-            "user": dict,
+            "keyword": {
-        },
+                "gallery": dict,
-    })
+                "user": dict,
+            },
+        }),
+        ("https://500px.com/fashvamp/galleries/lera"),
+    )
 
     def __init__(self, match):
         _500pxExtractor.__init__(self, match)
         self.user_name, self.gallery_name = match.groups()
-        self.user_id = self.gallery_id = self.csrf_token = None
+        self.user_id = self._photos = None
 
     def metadata(self):
-        # get csrf token and user id from webpage
+        user = self._request_graphql(
-        url = "{}/{}/galleries/{}".format(
+            "ProfileRendererQuery", {"username": self.user_name},
-            self.root, self.user_name, self.gallery_name)
+            "db1dba2cb7b7e94916d1005db16fea1a39d6211437b691c4de2f1a606c21c5fb",
-        page = self.request(url).text
+        )["profile"]
-        self.csrf_token, pos = text.extract(page, 'csrf-token" content="', '"')
+        self.user_id = str(user["legacyId"])
-        self.user_id   , pos = text.extract(page, 'App.CuratorId =', '\n', pos)
-        self.user_id = self.user_id.strip(" '\";")
 
-        # get gallery metadata; transform gallery name into id
+        variables = {
-        url = "https://api.500px.com/v1/users/{}/galleries/{}".format(
+            "galleryOwnerLegacyId": self.user_id,
-            self.user_id, self.gallery_name)
+            "ownerLegacyId"       : self.user_id,
-        params = {
+            "slug"                : self.gallery_name,
-            #  "include_user": "true",
+            "token"               : None,
-            "include_cover": "1",
+            "pageSize"            : 20,
-            "cover_size": "2048",
+        }
+        gallery = self._request_graphql(
+            "GalleriesDetailQueryRendererQuery", variables,
+            "1afc7dede86ff73456b4defbc5aeb593e330b990943d114cbef7da5be0d7ce2f",
+        )["gallery"]
+
+        self._photos = gallery["photos"]
+        del gallery["photos"]
+        return {
+            "gallery": gallery,
+            "user"   : user,
         }
-        data = self._api_call(url, params, self.csrf_token)
-        self.gallery_id = data["gallery"]["id"]
-        return data
 
     def photos(self):
-        url = "https://api.500px.com/v1/users/{}/galleries/{}/items".format(
+        photos = self._photos
-            self.user_id, self.gallery_id)
+        variables = {
-        params = {
+            "ownerLegacyId": self.user_id,
-            "sort"             : "position",
+            "slug"         : self.gallery_name,
-            "sort_direction"   : "asc",
+            "token"        : None,
-            "rpp"              : "50",
+            "pageSize"     : 20,
         }
-        return self._pagination(url, params, self.csrf_token)
+
+        while True:
+            yield from self._extend(photos["edges"])
+
+            if not photos["pageInfo"]["hasNextPage"]:
+                return
+
+            variables["cursor"] = photos["pageInfo"]["endCursor"]
+            photos = self._request_graphql(
+                "GalleriesDetailPaginationContainerQuery", variables,
+                "3fcbc9ea1589f31c86fc43a0a02c2163"
+                "cab070f9d376651f270de9f30f031539",
+            )["galleryByOwnerIdAndSlugOrToken"]["photos"]
 
 
 class _500pxImageExtractor(_500pxExtractor):
@@ -226,5 +253,5 @@ class _500pxImageExtractor(_500pxExtractor):
         self.photo_id = match.group(1)
 
     def photos(self):
-        photos = ({"id": self.photo_id},)
+        edges = ({"node": {"legacyId": self.photo_id}},)
-        return self._extend(photos)
+        return self._extend(edges)