From c47482b1106f0197bc89e36efc045e86922bfcf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 16 Nov 2018 18:02:24 +0100 Subject: [PATCH] smaller changes, missing docs, etc. - make 'netrc' extractor-specific - rename 'downloader.enable' to 'enabled' - document 'downloader.ytdl.format' - consistent newlines in configuration.rst --- docs/configuration.rst | 70 +++++++++++++++++++++------------- gallery_dl/extractor/common.py | 8 ++-- gallery_dl/job.py | 4 +- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 84054d32..4b3e25ad 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -13,9 +13,11 @@ Contents 7) `API Tokens & IDs`_ + Extractor Options ================= + Each extractor is identified by its ``category`` and ``subcategory``. The ``category`` is the lowercase site name without any spaces or special characters, which is usually just the module name @@ -155,6 +157,15 @@ Description The username and password to use when attempting to log in to =========== ===== +extractor.*.netrc +----------------- +=========== ===== +Type ``bool`` +Default ``false`` +Description Enable the use of |.netrc|_ authentication data. +=========== ===== + + extractor.*.cookies ------------------- =========== ===== @@ -356,9 +367,11 @@ Description Like `image-filter`__, but applies to delegated URLs __ `extractor.*.image-filter`_ + Extractor-specific Options ========================== + extractor.artstation.external ----------------------------- =========== ===== @@ -769,8 +782,18 @@ Description Categorize tags by their respective types Downloader Options ================== -downloader.part ---------------- + +downloader.*.enabled +-------------------- +=========== ===== +Type ``bool`` +Default ``true`` +Description Enable/Disable this downloader module. +=========== ===== + + +downloader.*.part +----------------- =========== ===== Type ``bool`` Default ``true`` @@ -784,8 +807,8 @@ Description Controls the use of ``.part`` files during file downloads. =========== ===== -downloader.part-directory -------------------------- +downloader.*.part-directory +--------------------------- =========== ===== Type |Path|_ Default ``null`` @@ -797,8 +820,8 @@ Description Alternate location for ``.part`` files. =========== ===== -downloader.rate ---------------- +downloader.*.rate +----------------- =========== ===== Type ``string`` Default ``null`` @@ -811,8 +834,8 @@ Description Maximum download rate in bytes per second. =========== ===== -downloader.retries ------------------- +downloader.*.retries +-------------------- =========== ===== Type ``integer`` Default `extractor.*.retries`_ @@ -820,8 +843,8 @@ Description Number of retries during file downloads. =========== ===== -downloader.timeout ------------------- +downloader.*.timeout +-------------------- =========== ===== Type ``float`` or ``null`` Default `extractor.*.timeout`_ @@ -829,8 +852,8 @@ Description Connection timeout during file downloads. =========== ===== -downloader.verify ------------------ +downloader.*.verify +------------------- =========== ===== Type ``bool`` or ``string`` Default `extractor.*.verify`_ @@ -838,12 +861,14 @@ Description Certificate validation during file downloads. =========== ===== -downloader.*.enable -------------------- +downloader.ytdl.format +---------------------- =========== ===== -Type ``bool`` -Default ``true`` -Description Enable/Disable this downloader module. +Type ``string`` +Default youtube-dl's default, currently ``"bestvideo+bestaudio/best"`` +Description Video `format selection + `__ + directly passed to youtube-dl. =========== ===== @@ -882,6 +907,7 @@ Description | Additional options passed directly to the ``YoutubeDL`` constructo Output Options ============== + output.mode ----------- =========== ===== @@ -1053,7 +1079,6 @@ Default ``"ffmpeg"`` Description Location of the ``ffmpeg`` (or ``avconv``) executable to use. =========== ===== - ugoira.ffmpeg-output -------------------- =========== ===== @@ -1145,14 +1170,6 @@ Description Keep the actual files after writing them to a ZIP archive. Miscellaneous Options ===================== -netrc ------ -=========== ===== -Type ``bool`` -Default ``false`` -Description Enable the use of |.netrc|_ authentication data. -=========== ===== - cache.file ---------- @@ -1171,6 +1188,7 @@ Description Path of the SQLite3 database used to cache login sessions, API Tokens & IDs ================ + All configuration keys listed in this section have fully functional default values embedded into *gallery-dl* itself, but if things unexpectedly break or you want to use your own personal client credentials, you can follow these diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index ab8b8c72..5f5e612b 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -60,6 +60,7 @@ class Extractor(): retries = retries or self._retries kwargs.setdefault("timeout", self._timeout) kwargs.setdefault("verify", self._verify) + while True: try: response = self.session.request(method, url, **kwargs) @@ -74,14 +75,13 @@ class Extractor(): response.encoding = encoding return response - msg = "{}: {} for url: {}".format( - code, response.reason, url) + msg = "{}: {} for url: {}".format(code, response.reason, url) if code < 500 and code != 429: break + self.log.debug("%s (%d/%d)", msg, tries + 1, retries) if tries >= retries: break - self.log.debug("%s (%d/%d)", msg, tries + 1, retries) time.sleep(2 ** tries) tries += 1 @@ -94,7 +94,7 @@ class Extractor(): if username: password = self.config("password") - elif config.get(("netrc",), False): + elif self.config("netrc", False): try: info = netrc.netrc().authenticators(self.category) username, _, password = info diff --git a/gallery_dl/job.py b/gallery_dl/job.py index dfbc6fdc..f5e091f0 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -272,11 +272,11 @@ class DownloadJob(Job): pass klass = downloader.find(scheme) - if klass and config.get(("downloader", scheme, "enable"), True): + if klass and config.get(("downloader", scheme, "enabled"), True): instance = klass(self.extractor, self.out) else: instance = None - self.log.error("'%s:' URLs are not supported", scheme) + self.log.error("'%s:' URLs are not supported/enabled", scheme) self.downloaders[scheme] = instance return instance