Merge branch 'master' into 1.4-dev

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changelog
 
+## Unreleased
+
 ## 1.3.4 - 2018-04-20
 - Added support for custom OAuth2 credentials for `pinterest`
 - Improved rate limit handling for `tumblr` extractors

gallery_dl/extractor/deviantart.py

@@ -178,6 +178,9 @@ class DeviantartGalleryExtractor(DeviantartExtractor):
             "url": "fa6ecb2c3aa78872f762d43f7809b7f0580debc1",
             "keyword": "b29746bac291d8c8e339f0256a2bd7bb3ebe1741",
         }),
+        ("http://shimoda8.deviantart.com/gallery/", {
+            "exception": exception.NotFoundError,
+        }),
         ("http://shimoda7.deviantart.com/gallery/?catpath=/", None),
     ]
 
@@ -306,7 +309,6 @@ class DeviantartCollectionExtractor(DeviantartExtractor):
     test = [(("https://pencilshadings.deviantart.com"
               "/favourites/70595441/3D-Favorites"), {
         "url": "742f92199d5bc6a89cda6ec6133d46c7a523824d",
-        "keyword": "d258ca05424b3586c4feec940158bca00acdf511",
         "options": (("original", False),),
     })]
 
@@ -355,7 +357,7 @@ class DeviantartAPI():
         self.session = extractor.session
         self.log = extractor.log
         self.headers = {}
-        self.delay = 0
+        self.delay = -1
 
         self.mature = extractor.config("mature", "true")
         if not isinstance(self.mature, str):
@@ -465,35 +467,33 @@ class DeviantartAPI():
         """Call an API endpoint"""
         url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
         while True:
-            if self.delay > 0:
+            if self.delay >= 0:
-                time.sleep(2 ** (self.delay-1))
+                time.sleep(2 ** self.delay)
 
             self.authenticate()
             response = self.session.get(
                 url, headers=self.headers, params=params)
+            data = response.json()
+            status = response.status_code
 
-            if response.status_code == 200:
+            if 200 <= status < 400:
-                if self.delay > 2:
+                if self.delay > 1:
                     self.delay -= 1
-                break
+                return data
+            elif expect_error:
+                return None
+            elif data.get("error_description") == "User not found.":
+                raise exception.NotFoundError("user or group")
 
+            self.log.debug(response.text)
+            msg = "API responded with {} {}".format(
+                status, response.reason)
+            if 400 <= status < 500 and status != 429:
+                self.log.error(msg)
+                return data
             else:
-                if response.status_code == 429:
-                    msg = "Rate limit reached"
-                else:
-                    if expect_error:
-                        return None
-                    msg = "API responded with {} {}".format(
-                        response.status_code, response.reason)
                 self.delay += 1
-                self.log.warning(
+                self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
-                    "%s. Using %ds delay.", msg, 2 ** (self.delay-1))
-                self.log.debug(response.text)
-        try:
-            return response.json()
-        except ValueError:
-            self.log.error("Failed to parse API response")
-            return {}
 
     def _pagination(self, endpoint, params=None):
         while True:

gallery_dl/extractor/mangadex.py

@@ -27,13 +27,13 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor):
     pattern = [r"(?:https?://)?(?:www\.)?mangadex\.(?:org|com)/chapter/(\d+)"]
     test = [
         ("https://mangadex.org/chapter/122094", {
-            "keyword": "b4c83fe41f125eae745c2e00d29e087cc4eb78df",
+            "keyword": "fe9f66f61ef3a31d9e5a0bd47c672f1b2433a682",
             "content": "7ab3bef5caccb62b881f8e6e70359d3c7be8137f",
         }),
         # oneshot
         ("https://mangadex.org/chapter/138086", {
             "count": 64,
-            "keyword": "9b1b7292f7dbcf10983fbdc34b8cdceeb47328ee",
+            "keyword": "0e27e78e498debf905199ff9540cffe5c352ae21",
         }),
         # NotFoundError
         ("https://mangadex.org/chapter/1", {
@@ -42,8 +42,7 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor):
     ]
 
     def __init__(self, match):
-        self.chapter_id = match.group(1)
+        url = self.root + "/chapter/" + match.group(1)
-        url = self.root + "/chapter/" + self.chapter_id
         ChapterExtractor.__init__(self, url)
 
     def get_metadata(self, page):
@@ -51,12 +50,15 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor):
             raise exception.NotFoundError("chapter")
 
         info    , pos = text.extract(page, '="og:title" content="', '"')
-        manga_id, pos = text.extract(page, '/images/manga/', '.', pos)
         _       , pos = text.extract(page, ' id="jump_group"', '', pos)
         _       , pos = text.extract(page, ' selected ', '', pos)
         language, ___ = text.extract(page, " title='", "'", pos-100)
         group   , pos = text.extract(page, '>', '<', pos)
 
+        data = json.loads(
+            text.extract(page, 'data-type="chapter">', '<', pos)[0]
+        )
+
         info = text.unescape(info)
         match = re.match(
             r"(?:(?:Vol\. (\d+) )?Ch\. (\d+)([^ ]*)|(.*)) "
@@ -64,12 +66,13 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor):
             info)
 
         return {
-            "manga": match.group(5),
+            "manga": data["manga_title"],
-            "manga_id": text.parse_int(manga_id),
+            "manga_id": data["manga_id"],
+            "title": data["chapter_title"],
             "volume": text.parse_int(match.group(1)),
             "chapter": text.parse_int(match.group(2)),
             "chapter_minor": match.group(3) or "",
-            "chapter_id": text.parse_int(self.chapter_id),
+            "chapter_id": data["chapter_id"],
             "chapter_string": info.replace(" - MangaDex", ""),
             "group": text.unescape(group),
             "lang": util.language_to_code(language),

gallery_dl/extractor/seaotterscans.py

@@ -14,7 +14,7 @@ from . import foolslide
 class SeaotterscansChapterExtractor(foolslide.FoolslideChapterExtractor):
     """Extractor for manga-chapters from reader.seaotterscans.com"""
     category = "seaotterscans"
-    pattern = foolslide.chapter_pattern("reader\.seaotterscans\.com")
+    pattern = foolslide.chapter_pattern(r"reader\.seaotterscans\.com")
     test = [("https://reader.seaotterscans.com/read/100_days/en/0/5/", {
         "url": "63d46b8883cc652dfe8bd5be4492160dd31f06a8",
         "keyword": "5349c2fbaa88070e6af600de17a6c4e212243e8e",
@@ -24,7 +24,7 @@ class SeaotterscansChapterExtractor(foolslide.FoolslideChapterExtractor):
 class SeaotterscansMangaExtractor(foolslide.FoolslideMangaExtractor):
     """Extractor for manga from reader.seaotterscans.com"""
     category = "seaotterscans"
-    pattern = foolslide.manga_pattern("reader\.seaotterscans\.com")
+    pattern = foolslide.manga_pattern(r"reader\.seaotterscans\.com")
     test = [("https://reader.seaotterscans.com/series/marry_me/", {
         "url": "fdbacabfa566a6baeb3f01bb46cbda0577bd4bbe",
         "keyword": "61d3388d73df12f64361892b47a9398df4a5947c",

gallery_dl/extractor/worldthree.py

@@ -14,7 +14,7 @@ from . import foolslide
 class WorldthreeChapterExtractor(foolslide.FoolslideChapterExtractor):
     """Extractor for manga-chapters from slide.world-three.org"""
     category = "worldthree"
-    pattern = foolslide.chapter_pattern("(?:www\.)?slide\.world-three\.org")
+    pattern = foolslide.chapter_pattern(r"(?:www\.)?slide\.world-three\.org")
     test = [
         (("http://www.slide.world-three.org"
           "/read/black_bullet/en/2/7/page/1"), {
@@ -33,7 +33,7 @@ class WorldthreeChapterExtractor(foolslide.FoolslideChapterExtractor):
 class WorldthreeMangaExtractor(foolslide.FoolslideMangaExtractor):
     """Extractor for manga from slide.world-three.org"""
     category = "worldthree"
-    pattern = foolslide.manga_pattern("(?:www\.)?slide\.world-three\.org")
+    pattern = foolslide.manga_pattern(r"(?:www\.)?slide\.world-three\.org")
     test = [("http://www.slide.world-three.org/series/black_bullet/", {
         "url": "5743b93512d26e6b540d90a7a5d69208b6d4a738",
         "keyword": "3a24f1088b4d7f3b798a96163f21ca251293a120",

gallery_dl/version.py

@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.3.4"
+__version__ = "1.3.5-dev"

setup.cfg

@@ -1,3 +1,3 @@
 [flake8]
 exclude = gallery_dl/__init__.py,gallery_dl/__main__.py,setup.py,build,scripts,archive
-ignore = E203,E226
+ignore = E203,E226,W504