[artstation] add challenge extractor (#80)

docs/supportedsites.rst

@@ -91,7 +91,7 @@ Turboimagehost       https://turboimagehost.com/         individual Images
 ==================== =================================== ================================================== ================
 
 .. |http://www.thes-0| replace:: http://www.thespectrum.net/manga_scans/
-.. |Images from Use-1| replace:: Images from Users, Albums, individual Images, Likes
+.. |Images from Use-1| replace:: Images from Users, Albums, Contests, individual Images, Likes, Search Results
 .. |Collections, De-2| replace:: Collections, Deviations, Favorites, Folders, Galleries, Journals
 .. |Images from Use-3| replace:: Images from Users, Albums, Favorites, Galleries, Groups, individual Images, Search Results
 .. |Images from Use-4| replace:: Images from Users, Bookmarks, Favorites, pixiv.me Links, Rankings, Individual Images

gallery_dl/extractor/artstation.py

@@ -21,7 +21,6 @@ class ArtstationExtractor(Extractor):
     directory_fmt = ["{category}", "{userinfo[username]}"]
     archive_fmt = "{asset[id]}"
     root = "https://www.artstation.com"
-    per_page = 50
 
     def __init__(self, match=None):
         Extractor.__init__(self)
@@ -33,8 +32,8 @@ class ArtstationExtractor(Extractor):
         yield Message.Version, 1
         yield Message.Directory, data
 
-        for project_id in self.projects():
-            for asset in self.get_project_assets(project_id):
+        for project in self.projects():
+            for asset in self.get_project_assets(project["hash_id"]):
                 adict = asset["asset"]
                 if data:
                     asset.update(data)
@@ -83,13 +82,16 @@ class ArtstationExtractor(Extractor):
         if not params:
             params = {}
         params["page"] = 1
+        total = 0
 
         while True:
-            projects = self.request(url, params=params).json()["data"]
-            for project in projects:
-                yield project["hash_id"]
-            if len(projects) < self.per_page:
+            data = self.request(url, params=params).json()
+            yield from data["data"]
+
+            total += len(data["data"])
+            if total >= data["total_count"]:
                 return
+
             params["page"] += 1
 
     @staticmethod
@@ -201,6 +203,62 @@ class ArtstationLikesExtractor(ArtstationExtractor):
         return self._pagination(url)
 
 
+class ArtstationChallengeExtractor(ArtstationExtractor):
+    """Extractor for submissions of artstation challenges"""
+    subcategory = "challenge"
+    filename_fmt = "{submission_id}_{asset_id}_{name}.{extension}"
+    directory_fmt = ["{category}", "Challenges",
+                     "{challenge[id]} - {challenge[title]}"]
+    archive_fmt = "c_{challenge[id]}_{asset_id}"
+    pattern = [r"(?:https?://)?(?:www\.)?artstation\.com"
+               r"/contests/[^/?&#]+/challenges/(\d+)"
+               r"/?(?:\?sorting=([a-z]+))?"]
+    test = [
+        (("https://www.artstation.com/contests/thu-2017/challenges/20"), None),
+        (("https://www.artstation.com/contests/beyond-human"
+          "/challenges/23?sorting=winners"), {
+            "count": 203,
+        }),
+    ]
+
+    def __init__(self, match):
+        ArtstationExtractor.__init__(self)
+        self.challenge_id = match.group(1)
+        self.sorting = match.group(2) or "popular"
+
+    def items(self):
+        challenge_url = "{}/contests/_/challenges/{}.json".format(
+            self.root, self.challenge_id)
+        submission_url = "{}/contests/_/challenges/{}/submissions.json".format(
+            self.root, self.challenge_id)
+        update_url = "{}/contests/submission_updates.json".format(
+            self.root)
+
+        challenge = self.request(challenge_url).json()
+        yield Message.Version, 1
+        yield Message.Directory, {"challenge": challenge}
+
+        params = {"sorting": self.sorting}
+        for submission in self._pagination(submission_url, params):
+
+            params = {"submission_id": submission["id"]}
+            for update in self._pagination(update_url, params=params):
+
+                del update["replies"]
+                update["challenge"] = challenge
+                for url in text.extract_iter(
+                        update["body_presentation_html"], ' href="', '"'):
+                    update["asset_id"] = self._id_from_url(url)
+                    text.nameext_from_url(url, update)
+                    yield Message.Url, self._no_cache(url), update
+
+    @staticmethod
+    def _id_from_url(url):
+        """Get an image's submission ID from its URL"""
+        parts = url.split("/")
+        return util.safe_int("".join(parts[7:10]))
+
+
 class ArtstationSearchExtractor(ArtstationExtractor):
     """Extractor for artstation search results"""
     subcategory = "search"
@@ -274,7 +332,7 @@ class ArtstationImageExtractor(ArtstationExtractor):
         return ArtstationExtractor.metadata(self)
 
     def projects(self):
-        return (self.project_id,)
+        return ({"hash_id": self.project_id},)
 
     def get_project_assets(self, project_id):
         return self.assets

gallery_dl/extractor/fallenangels.py

@@ -68,8 +68,8 @@ class FallenangelsMangaExtractor(MangaExtractor):
     scheme = "https"
     test = [
         ("http://manga.fascans.com/manga/trinity-seven", {
-            "url": "1e7bcabcd2902520fbf21bf0743183916a483883",
-            "keyword": "47ab722fc79bab4bfa0dbcdbced5f21d5affdb32",
+            "url": "94eb803094f80e0e7725fdd3f8b826d6afdabb2a",
+            "keyword": "2c14cb4fb04af3f049b519336dbcb22bedbf461d",
         }),
         ("https://truyen.fascans.com/manga/rakudai-kishi-no-eiyuutan", {
             "url": "51a731a6b82d5eb7a335fbae6b02d06aeb2ab07b",