From c70b21248ddb07eb4c48e317e938b88c6273d486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 2 Apr 2019 17:34:57 +0200 Subject: [PATCH] [wikiart] add extractors (#179) for - artists: https://www.wikiart.org/en/thomas-cole - artist-listings: https://www.wikiart.org/en/artists-by-century/12 - artwork-listings: https://www.wikiart.org/en/paintings-by-media/grisaille --- docs/supportedsites.rst | 1 + gallery_dl/extractor/__init__.py | 1 + gallery_dl/extractor/wikiart.py | 135 +++++++++++++++++++++++++++++++ scripts/supportedsites.py | 2 + test/test_results.py | 3 +- 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 gallery_dl/extractor/wikiart.py diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index e2c8314f..de28bf03 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -93,6 +93,7 @@ Twitter https://twitter.com/ Media Timelines, Timeli Wallhaven https://alpha.wallhaven.cc/ individual Images, Search Results Optional Warosu https://warosu.org/ Threads Weibo https://www.weibo.com/ Images from Users, Images from Statuses +WikiArt.org https://www.wikiart.org/ Artists, Artworks World Three http://www.slide.world-three.org/ Chapters, Manga XVideos https://www.xvideos.com/ Images from Users, Galleries Yandere https://yande.re/ Pools, Popular Images, Posts, Tag-Searches diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index e0c9c37d..acb0da6e 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -83,6 +83,7 @@ modules = [ "wallhaven", "warosu", "weibo", + "wikiart", "yandere", "xvideos", "yuki", diff --git a/gallery_dl/extractor/wikiart.py b/gallery_dl/extractor/wikiart.py new file mode 100644 index 00000000..e4d96fc4 --- /dev/null +++ b/gallery_dl/extractor/wikiart.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://www.wikiart.org/""" + +from .common import Extractor, Message +from .. import text + +BASE_PATTERN = r"(?:https://)?(?:www\.)?wikiart\.org/([a-z]+)" + + +class WikiartExtractor(Extractor): + """Base class for wikiart extractors""" + category = "wikiart" + filename_fmt = "{id}_{title}.{extension}" + archive_fmt = "{id}" + root = "https://www.wikiart.org" + + def __init__(self, match): + Extractor.__init__(self, match) + self.lang = match.group(1) + + def items(self): + data = self.metadata() + yield Message.Version, 1 + yield Message.Directory, data + for painting in self.paintings(): + url = painting["image"] + painting.update(data) + yield Message.Url, url, text.nameext_from_url(url, painting) + + def metadata(self): + """ """ + + def paintings(self): + """ """ + + def _pagination(self, url, key="Paintings", extra_params=None): + headers = { + "X-Requested-With": "XMLHttpRequest", + "Referer": url, + } + params = { + "json": "2", + "layout": "new", + "page": 1, + "resultType": "masonry", + } + if extra_params: + params.update(extra_params) + + while True: + data = self.request(url, headers=headers, params=params).json() + items = data.get(key) + if not items: + return + yield from items + params["page"] += 1 + + +class WikiartArtistExtractor(WikiartExtractor): + """Extractor for an artist's paintings on wikiart.org""" + subcategory = "artist" + directory_fmt = ("{category}", "{artist[artistName]}") + pattern = BASE_PATTERN + r"/(?!\w+-by-)([\w-]+)" + test = ("https://www.wikiart.org/en/thomas-cole", { + "url": "f1eee8158f5b8b7380382ab730a8f53884715c8b", + "keyword": "b62678394ce645815963883d5c9642255307225f", + }) + + def __init__(self, match): + WikiartExtractor.__init__(self, match) + self.artist = match.group(2) + + def metadata(self): + url = "{}/{}/{}?json=2".format(self.root, self.lang, self.artist) + return {"artist": self.request(url).json()} + + def paintings(self): + """ """ + url = "{}/{}/{}/mode/all-paintings".format( + self.root, self.lang, self.artist) + return self._pagination(url) + + +class WikiartArtworksExtractor(WikiartExtractor): + """Extractor for artwork collections on wikiart.org""" + subcategory = "artworks" + directory_fmt = ("{category}", "Artworks by {group!c}", "{type}") + pattern = BASE_PATTERN + r"/paintings-by-([\w-]+)/([\w-]+)" + test = ("https://www.wikiart.org/en/paintings-by-media/grisaille", { + "url": "f92d55669fa949491c26a5437527adb14b35b8cc", + }) + + def __init__(self, match): + WikiartExtractor.__init__(self, match) + self.group = match.group(2) + self.type = match.group(3) + + def metadata(self): + return {"group": self.group, "type": self.type} + + def paintings(self): + url = "{}/{}/paintings-by-{}/{}".format( + self.root, self.lang, self.group, self.type) + return self._pagination(url) + + +class WikiartArtistsExtractor(WikiartExtractor): + """Extractor for artist collections on wikiart.org""" + subcategory = "artists" + pattern = (BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)") + test = ("https://www.wikiart.org/en/artists-by-century/12", { + "pattern": WikiartArtistExtractor.pattern, + "count": 7, + }) + + def __init__(self, match): + WikiartExtractor.__init__(self, match) + self.group = match.group(2) + self.type = match.group(3) + + def items(self): + url = "{}/{}/App/Search/Artists-by-{}".format( + self.root, self.lang, self.group) + params = {"json": "3", "searchterm": self.type} + + for artist in self._pagination(url, "Artists", params): + artist["_extractor"] = WikiartArtistExtractor + yield Message.Queue, self.root + artist["artistUrl"], artist diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 572abec9..1b05fa09 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -63,6 +63,7 @@ CATEGORY_MAP = { "slideshare" : "SlideShare", "smugmug" : "SmugMug", "thebarchive" : "The /b/ Archive", + "wikiart" : "WikiArt.org", "worldthree" : "World Three", "xvideos" : "XVideos", "yuki" : "yuki.la 4chan archive", @@ -70,6 +71,7 @@ CATEGORY_MAP = { SUBCATEGORY_MAP = { "artwork": "Artwork Listings", + "artists": "", "doujin" : "Doujin", "gallery": "Galleries", "image" : "individual Images", diff --git a/test/test_results.py b/test/test_results.py index 1ee77163..15217a31 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -25,8 +25,9 @@ TRAVIS_SKIP = { # temporary issues, etc. BROKEN = { + "komikcast", + "mangahere", "mangapark", - "nyafuu", }