diff --git a/CHANGELOG.md b/CHANGELOG.md index 1edb2263..844de878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## Unreleased + ## 1.3.4 - 2018-04-20 - Added support for custom OAuth2 credentials for `pinterest` - Improved rate limit handling for `tumblr` extractors diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index ee156342..aa117cc0 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -178,6 +178,9 @@ class DeviantartGalleryExtractor(DeviantartExtractor): "url": "fa6ecb2c3aa78872f762d43f7809b7f0580debc1", "keyword": "b29746bac291d8c8e339f0256a2bd7bb3ebe1741", }), + ("http://shimoda8.deviantart.com/gallery/", { + "exception": exception.NotFoundError, + }), ("http://shimoda7.deviantart.com/gallery/?catpath=/", None), ] @@ -306,7 +309,6 @@ class DeviantartCollectionExtractor(DeviantartExtractor): test = [(("https://pencilshadings.deviantart.com" "/favourites/70595441/3D-Favorites"), { "url": "742f92199d5bc6a89cda6ec6133d46c7a523824d", - "keyword": "d258ca05424b3586c4feec940158bca00acdf511", "options": (("original", False),), })] @@ -355,7 +357,7 @@ class DeviantartAPI(): self.session = extractor.session self.log = extractor.log self.headers = {} - self.delay = 0 + self.delay = -1 self.mature = extractor.config("mature", "true") if not isinstance(self.mature, str): @@ -465,35 +467,33 @@ class DeviantartAPI(): """Call an API endpoint""" url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint while True: - if self.delay > 0: - time.sleep(2 ** (self.delay-1)) + if self.delay >= 0: + time.sleep(2 ** self.delay) self.authenticate() response = self.session.get( url, headers=self.headers, params=params) + data = response.json() + status = response.status_code - if response.status_code == 200: - if self.delay > 2: + if 200 <= status < 400: + if self.delay > 1: self.delay -= 1 - break + return data + elif expect_error: + return None + elif data.get("error_description") == "User not found.": + raise exception.NotFoundError("user or group") + self.log.debug(response.text) + msg = "API responded with {} {}".format( + status, response.reason) + if 400 <= status < 500 and status != 429: + self.log.error(msg) + return data else: - if response.status_code == 429: - msg = "Rate limit reached" - else: - if expect_error: - return None - msg = "API responded with {} {}".format( - response.status_code, response.reason) self.delay += 1 - self.log.warning( - "%s. Using %ds delay.", msg, 2 ** (self.delay-1)) - self.log.debug(response.text) - try: - return response.json() - except ValueError: - self.log.error("Failed to parse API response") - return {} + self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay) def _pagination(self, endpoint, params=None): while True: diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index 4529a735..677d4049 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -27,13 +27,13 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor): pattern = [r"(?:https?://)?(?:www\.)?mangadex\.(?:org|com)/chapter/(\d+)"] test = [ ("https://mangadex.org/chapter/122094", { - "keyword": "b4c83fe41f125eae745c2e00d29e087cc4eb78df", + "keyword": "fe9f66f61ef3a31d9e5a0bd47c672f1b2433a682", "content": "7ab3bef5caccb62b881f8e6e70359d3c7be8137f", }), # oneshot ("https://mangadex.org/chapter/138086", { "count": 64, - "keyword": "9b1b7292f7dbcf10983fbdc34b8cdceeb47328ee", + "keyword": "0e27e78e498debf905199ff9540cffe5c352ae21", }), # NotFoundError ("https://mangadex.org/chapter/1", { @@ -42,8 +42,7 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor): ] def __init__(self, match): - self.chapter_id = match.group(1) - url = self.root + "/chapter/" + self.chapter_id + url = self.root + "/chapter/" + match.group(1) ChapterExtractor.__init__(self, url) def get_metadata(self, page): @@ -51,12 +50,15 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor): raise exception.NotFoundError("chapter") info , pos = text.extract(page, '="og:title" content="', '"') - manga_id, pos = text.extract(page, '/images/manga/', '.', pos) _ , pos = text.extract(page, ' id="jump_group"', '', pos) _ , pos = text.extract(page, ' selected ', '', pos) language, ___ = text.extract(page, " title='", "'", pos-100) group , pos = text.extract(page, '>', '<', pos) + data = json.loads( + text.extract(page, 'data-type="chapter">', '<', pos)[0] + ) + info = text.unescape(info) match = re.match( r"(?:(?:Vol\. (\d+) )?Ch\. (\d+)([^ ]*)|(.*)) " @@ -64,12 +66,13 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor): info) return { - "manga": match.group(5), - "manga_id": text.parse_int(manga_id), + "manga": data["manga_title"], + "manga_id": data["manga_id"], + "title": data["chapter_title"], "volume": text.parse_int(match.group(1)), "chapter": text.parse_int(match.group(2)), "chapter_minor": match.group(3) or "", - "chapter_id": text.parse_int(self.chapter_id), + "chapter_id": data["chapter_id"], "chapter_string": info.replace(" - MangaDex", ""), "group": text.unescape(group), "lang": util.language_to_code(language), diff --git a/gallery_dl/extractor/seaotterscans.py b/gallery_dl/extractor/seaotterscans.py index f24fb5aa..290d2cbe 100644 --- a/gallery_dl/extractor/seaotterscans.py +++ b/gallery_dl/extractor/seaotterscans.py @@ -14,7 +14,7 @@ from . import foolslide class SeaotterscansChapterExtractor(foolslide.FoolslideChapterExtractor): """Extractor for manga-chapters from reader.seaotterscans.com""" category = "seaotterscans" - pattern = foolslide.chapter_pattern("reader\.seaotterscans\.com") + pattern = foolslide.chapter_pattern(r"reader\.seaotterscans\.com") test = [("https://reader.seaotterscans.com/read/100_days/en/0/5/", { "url": "63d46b8883cc652dfe8bd5be4492160dd31f06a8", "keyword": "5349c2fbaa88070e6af600de17a6c4e212243e8e", @@ -24,7 +24,7 @@ class SeaotterscansChapterExtractor(foolslide.FoolslideChapterExtractor): class SeaotterscansMangaExtractor(foolslide.FoolslideMangaExtractor): """Extractor for manga from reader.seaotterscans.com""" category = "seaotterscans" - pattern = foolslide.manga_pattern("reader\.seaotterscans\.com") + pattern = foolslide.manga_pattern(r"reader\.seaotterscans\.com") test = [("https://reader.seaotterscans.com/series/marry_me/", { "url": "fdbacabfa566a6baeb3f01bb46cbda0577bd4bbe", "keyword": "61d3388d73df12f64361892b47a9398df4a5947c", diff --git a/gallery_dl/extractor/worldthree.py b/gallery_dl/extractor/worldthree.py index e00eadce..6a2f0208 100644 --- a/gallery_dl/extractor/worldthree.py +++ b/gallery_dl/extractor/worldthree.py @@ -14,7 +14,7 @@ from . import foolslide class WorldthreeChapterExtractor(foolslide.FoolslideChapterExtractor): """Extractor for manga-chapters from slide.world-three.org""" category = "worldthree" - pattern = foolslide.chapter_pattern("(?:www\.)?slide\.world-three\.org") + pattern = foolslide.chapter_pattern(r"(?:www\.)?slide\.world-three\.org") test = [ (("http://www.slide.world-three.org" "/read/black_bullet/en/2/7/page/1"), { @@ -33,7 +33,7 @@ class WorldthreeChapterExtractor(foolslide.FoolslideChapterExtractor): class WorldthreeMangaExtractor(foolslide.FoolslideMangaExtractor): """Extractor for manga from slide.world-three.org""" category = "worldthree" - pattern = foolslide.manga_pattern("(?:www\.)?slide\.world-three\.org") + pattern = foolslide.manga_pattern(r"(?:www\.)?slide\.world-three\.org") test = [("http://www.slide.world-three.org/series/black_bullet/", { "url": "5743b93512d26e6b540d90a7a5d69208b6d4a738", "keyword": "3a24f1088b4d7f3b798a96163f21ca251293a120", diff --git a/gallery_dl/version.py b/gallery_dl/version.py index 6cd64a90..191fe110 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,4 +6,4 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.3.4" +__version__ = "1.3.5-dev" diff --git a/setup.cfg b/setup.cfg index cdd1b0ce..c8d5ceaa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ [flake8] exclude = gallery_dl/__init__.py,gallery_dl/__main__.py,setup.py,build,scripts,archive -ignore = E203,E226 +ignore = E203,E226,W504