diff --git a/docs/configuration.rst b/docs/configuration.rst index 24aa8a31..fd459293 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2504,8 +2504,23 @@ Type Default ``null`` Description - Sets a custom image download limit and - stops extraction when it gets exceeded. + Set a custom image download limit and perform + `limits-action `__ + when it gets exceeded. + + +extractor.exhentai.limits-action +-------------------------------- +Type + ``string`` +Default + ``"stop"`` +Description + Action to perform when the image limit is exceeded. + + * `"stop"`: Stop the current extractor run. + * `"wait"`: Wait for user input. + * `"reset"`: Spend GP to reset your account's image limits. extractor.exhentai.metadata @@ -2518,8 +2533,8 @@ Description Load extended gallery metadata from the `API `_. - Adds ``archiver_key``, ``posted``, and ``torrents``. - Makes ``date`` and ``filesize`` more precise. + * Adds ``archiver_key``, ``posted``, and ``torrents`` + * Provides exact ``date`` and ``filesize`` extractor.exhentai.original diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 029d2053..74980823 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -275,11 +275,12 @@ "domain" : "auto", "fav" : null, "gp" : "resized", - "limits" : null, "metadata": false, "original": true, "source" : null, "tags" : false, + "limits" : null, + "limits-action" : "stop", "fallback-retries": 2 }, "fanbox": diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index c4bcfdf5..8b997219 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -140,7 +140,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): limits = self.config("limits", False) if limits and limits.__class__ is int: self.limits = limits - self._remaining = 0 + self._limits_remaining = 0 else: self.limits = False @@ -198,7 +198,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): for url, image in images: data.update(image) if self.limits: - self._check_limits(data) + self._limits_check(data) if "/fullimg" in url: data["_http_validate"] = self._validate_response else: @@ -386,28 +386,30 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): request["imgkey"] = nextkey def _validate_response(self, response): - if not response.history and response.headers.get( + if response.history or not response.headers.get( "content-type", "").startswith("text/html"): - page = response.text - self.log.warning("'%s'", page) + return True - if " requires GP" in page: - gp = self.config("gp") - if gp == "stop": - raise exception.StopExtraction("Not enough GP") - elif gp == "wait": - input("Press ENTER to continue.") - return response.url + page = response.text + self.log.warning("'%s'", page) - self.log.info("Falling back to non-original downloads") - self.original = False - return self.data["_url_1280"] + if " requires GP" in page: + gp = self.config("gp") + if gp == "stop": + raise exception.StopExtraction("Not enough GP") + elif gp == "wait": + self.input("Press ENTER to continue.") + return response.url - if " temporarily banned " in page: - raise exception.AuthorizationError("Temporarily Banned") + self.log.info("Falling back to non-original downloads") + self.original = False + return self.data["_url_1280"] - self._report_limits() - return True + if " temporarily banned " in page: + raise exception.AuthorizationError("Temporarily Banned") + + self._limits_exceeded() + return response.url def _validate_signature(self, signature): """Return False if all file signature bytes are zero""" @@ -424,16 +426,22 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): return True return False - def _report_limits(self): - ExhentaiExtractor.LIMIT = True - raise exception.StopExtraction("Image limit reached!") + def _request_home(self, **kwargs): + url = "https://e-hentai.org/home.php" + kwargs["cookies"] = { + cookie.name: cookie.value + for cookie in self.cookies + if cookie.domain == self.cookies_domain and + cookie.name != "igneous" + } + page = self.request(url, **kwargs).text - def _check_limits(self, data): - if not self._remaining or data["num"] % 25 == 0: - self._update_limits() - self._remaining -= data["cost"] - if self._remaining <= 0: - self._report_limits() + # update image limits + current = text.extr(page, "", "").replace(",", "") + self.log.debug("Image Limits: %s/%s", current, self.limits) + self._limits_remaining = self.limits - text.parse_int(current) + + return page def _check_509(self, url): # full 509.gif URLs @@ -442,21 +450,40 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): if url.endswith(("hentai.org/img/509.gif", "ehgt.org/g/509.gif")): self.log.debug(url) - self._report_limits() + self._limits_exceeded() - def _update_limits(self): - url = "https://e-hentai.org/home.php" - cookies = { - cookie.name: cookie.value - for cookie in self.cookies - if cookie.domain == self.cookies_domain and - cookie.name != "igneous" - } + def _limits_exceeded(self): + msg = "Image limit exceeded!" + action = self.config("limits-action") - page = self.request(url, cookies=cookies).text - current = text.extr(page, "", "").replace(",", "") - self.log.debug("Image Limits: %s/%s", current, self.limits) - self._remaining = self.limits - text.parse_int(current) + if not action or action == "stop": + ExhentaiExtractor.LIMIT = True + raise exception.StopExtraction(msg) + + self.log.warning(msg) + if action == "wait": + self.input("Press ENTER to continue.") + self._limits_update() + elif action == "reset": + self._limits_reset() + else: + self.log.error("Invalid 'limits-action' value '%s'", action) + + def _limits_check(self, data): + if not self._limits_remaining or data["num"] % 25 == 0: + self._limits_update() + self._limits_remaining -= data["cost"] + if self._limits_remaining <= 0: + self._limits_exceeded() + + def _limits_reset(self): + self.log.info("Resetting image limits") + self._request_home( + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data=b"reset_imagelimit=Reset+Quota") + + _limits_update = _request_home def _gallery_page(self): url = "{}/g/{}/{}/".format(