[vk] fix extraction (#2512)

Use a different API endpoint, since thumbnail URLs from the old one
cannot be transformed into URLs for "original" photos anymore.

gallery_dl/extractor/vk.py

@@ -10,7 +10,6 @@
 
 from .common import Extractor, Message
 from .. import text
-import re
 
 BASE_PATTERN = r"(?:https://)?(?:www\.|m\.)?vk\.com"
 
@@ -25,47 +24,63 @@ class VkExtractor(Extractor):
     request_interval = 1.0
 
     def items(self):
+        sizes = "wzyxrqpo"
+
         data = self.metadata()
         yield Message.Directory, data
+
         for photo in self.photos():
+
+            for size in sizes:
+                size += "_"
+                if size in photo:
+                    break
+            else:
+                self.log.warning("no photo URL found (%s)", photo.get("id"))
+                continue
+
             photo.update(data)
+            photo["url"], photo["width"], photo["height"] = photo[size]
+            photo["id"] = photo["id"].rpartition("_")[2]
+
+            text.nameext_from_url(photo["url"], photo)
             yield Message.Url, photo["url"], photo
 
-    def _pagination(self, photos_url, user_id):
+    def _pagination(self, photos_id):
-        sub = re.compile(r"/imp[fg]/").sub
+        url = self.root + "/al_photos.php"
-        needle = 'data-id="{}_'.format(user_id)
-
         headers = {
             "X-Requested-With": "XMLHttpRequest",
             "Origin"          : self.root,
-            "Referer"         : photos_url,
+            "Referer"         : self.root + "/" + photos_id,
         }
-        params = {
+        data = {
-            "al"    : "1",
+            "act"      : "show",
-            "al_ad" : "0",
+            "al"       : "1",
-            "offset": 0,
+            "direction": "1",
-            "part"  : "1",
+            "list"     : photos_id,
+            "offset"   : 0,
         }
 
         while True:
             payload = self.request(
-                photos_url, method="POST", headers=headers, data=params
+                url, method="POST", headers=headers, data=data,
             ).json()["payload"][1]
 
-            offset = payload[0]
+            total = payload[1]
-            html = payload[1]
+            photos = payload[3]
 
-            cnt = 0
+            data["offset"] += len(photos)
-            for photo in text.extract_iter(html, needle, ')'):
+            if data["offset"] >= total:
-                cnt += 1
+                # the last chunk of photos also contains the first few photos
-                pid = photo[:photo.find('"')]
+                # again if 'total' is not a multiple of 10
-                url = photo[photo.rindex("(")+1:]
+                extra = total - data["offset"]
-                url = sub("/", url.partition("?")[0])
+                if extra:
-                yield text.nameext_from_url(url, {"url": url, "id": pid})
+                    del photos[extra:]
 
-            if cnt <= 20 or offset == params["offset"]:
+                yield from photos
                 return
-            params["offset"] = offset
+
+            yield from photos
 
 
 class VkPhotosExtractor(VkExtractor):
@@ -76,8 +91,8 @@ class VkPhotosExtractor(VkExtractor):
                r"|(?!album-?\d+_)([^/?#]+))")
     test = (
         ("https://vk.com/id398982326", {
-            "pattern": r"https://sun\d+-\d+\.userapi\.com"
+            "pattern": r"https://sun\d+-\d+\.userapi\.com/s/v1/if1"
-                       r"/\w+/v\d+/[0-9a-f]+/[\w-]+\.jpg",
+                       r"/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album",
             "count": ">= 35",
             "keywords": {
                 "id": r"re:\d+",
@@ -90,7 +105,7 @@ class VkPhotosExtractor(VkExtractor):
             },
         }),
         ("https://vk.com/cosplayinrussia", {
-            "range": "75-100",
+            "range": "25-35",
             "keywords": {
                 "id": r"re:\d+",
                 "user": {
@@ -112,8 +127,7 @@ class VkPhotosExtractor(VkExtractor):
         self.user_id, self.user_name = match.groups()
 
     def photos(self):
-        url = "{}/photos{}".format(self.root, self.user_id)
+        return self._pagination("photos" + self.user_id)
-        return self._pagination(url, self.user_id)
 
     def metadata(self):
         if self.user_id:
@@ -159,8 +173,8 @@ class VkAlbumExtractor(VkExtractor):
         self.user_id, self.album_id = match.groups()
 
     def photos(self):
-        url = "{}/album{}_{}".format(self.root, self.user_id, self.album_id)
+        return self._pagination("album{}_{}".format(
-        return self._pagination(url, self.user_id)
+            self.user_id, self.album_id))
 
     def metadata(self):
         return {