From d3b04076f71d3073f75333d54756f7c9a4813c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 24 Jun 2017 12:17:26 +0200 Subject: [PATCH] add .netrc support (#22) Use the '--netrc' cmdline option or set the 'netrc' config option to 'true' to enable the use of .netrc authentication data. The 'machine' names for the .netrc info are the lowercase extractor names (or categories): batoto, exhentai, nijie, pixiv, seiga. --- docs/configuration.rst | 13 ++++++++++++- docs/gallery-dl.conf | 1 + gallery_dl/extractor/batoto.py | 3 +-- gallery_dl/extractor/common.py | 19 +++++++++++++++++++ gallery_dl/extractor/exhentai.py | 3 +-- gallery_dl/extractor/nijie.py | 3 +-- gallery_dl/extractor/pixiv.py | 3 +-- gallery_dl/extractor/seiga.py | 3 +-- gallery_dl/option.py | 5 +++++ 9 files changed, 42 insertions(+), 11 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 0ae614d1..00cf39b6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -23,6 +23,15 @@ Description Directory path used as the base for all download destinations. =========== ===== +netrc +----- +=========== ===== +Type ``bool`` +Default ``false`` +Description Enable the use of |.netrc|_ authentication data. +=========== ===== + + cache.file ---------- =========== ===== @@ -331,7 +340,7 @@ Description The value of the ``limit`` parameter when loading extractor.reddit.morecomments -------------------------- +----------------------------- =========== ===== Type ``bool`` Default ``false`` @@ -392,11 +401,13 @@ Description The ``refresh_token`` value you get from linking your Reddit account =========== ===== +.. |.netrc| replace:: ``.netrc`` .. |tempfile.gettempdir()| replace:: ``tempfile.gettempdir()`` .. |requests.get()| replace:: ``requests.get()`` .. |mature_content| replace:: ``mature_content`` .. |webbrowser.open()| replace:: ``webbrowser.open()`` +.. _.netrc: https://stackoverflow.com/tags/.netrc/info .. _tempfile.gettempdir(): https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir .. _requests.get(): https://docs.python-requests.org/en/latest/user/advanced/#timeouts .. _format string: https://docs.python.org/3/library/string.html#formatstrings diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 14cf7be7..93067e05 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -1,5 +1,6 @@ { "base-directory": "/tmp/", + "netrc": false, "downloader": { "http": diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py index ced9b4a7..00d9a6d4 100644 --- a/gallery_dl/extractor/batoto.py +++ b/gallery_dl/extractor/batoto.py @@ -22,8 +22,7 @@ class BatotoExtractor(): def login(self): """Login and set necessary cookies""" - username = self.config("username") - password = self.config("password") + username, password = self.auth_info() if username: cookies = self._login_impl(username, password) for key, value in cookies.items(): diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 320b170f..abd556ab 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -10,6 +10,7 @@ import os import time +import netrc import queue import logging import requests @@ -42,6 +43,24 @@ class Extractor(): return config.interpolate( ("extractor", self.category, self.subcategory, key), default) + def auth_info(self): + """Return authentication information as (username, password) tuple""" + username = self.config("username") + password = None + + if username: + password = self.config("password") + elif config.get(("netrc",), False): + try: + info = netrc.netrc().authenticators(self.category) + username, _, password = info + except (OSError, netrc.NetrcParseError) as exc: + self.log.error("netrc: %s", exc) + except TypeError: + self.log.warning("netrc: No authentication info") + + return username, password + def request(self, url, encoding=None, *args, **kwargs): response = safe_request(self.session, url, *args, **kwargs) if encoding: diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index ed04bd6c..6a6c658b 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -182,13 +182,12 @@ class ExhentaiGalleryExtractor(Extractor): def login(self): """Login and set necessary cookies""" - username = self.config("username") + username, password = self.auth_info() if not username: self.log.info("no username given; using e-hentai.org") self.root = "https://e-hentai.org" self.original = False return - password = self.config("password") cookies = self._login_impl(username, password) for key, value in cookies.items(): self.session.cookies.set( diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py index c5f4e22e..8da28f72 100644 --- a/gallery_dl/extractor/nijie.py +++ b/gallery_dl/extractor/nijie.py @@ -62,8 +62,7 @@ class NijieExtractor(AsynchronousExtractor): def login(self): """Login and obtain session cookie""" - username = self.config("username") - password = self.config("password") + username, password = self.auth_info() self.session.cookies = self._login_impl(username, password) @cache(maxage=30*24*60*60, keyarg=1) diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index 034efa20..113a9345 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -233,8 +233,7 @@ class PixivAPI(): def __init__(self, extractor): self.session = extractor.session self.log = extractor.log - self.username = extractor.config("username") - self.password = extractor.config("password") + self.username, self.password = extractor.auth_info() self.user_info = None self.session.headers.update({ "Referer": "https://www.pixiv.net/", diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py index 570e9a01..d2b2483d 100644 --- a/gallery_dl/extractor/seiga.py +++ b/gallery_dl/extractor/seiga.py @@ -47,8 +47,7 @@ class SeigaExtractor(Extractor): def login(self): """Login and set necessary cookies""" - username = self.config("username") - password = self.config("password") + username, password = self.auth_info() self.session.cookies = self._login_impl(username, password) @cache(maxage=7*24*60*60, keyarg=1) diff --git a/gallery_dl/option.py b/gallery_dl/option.py index 8f2d70f7..2ffab289 100644 --- a/gallery_dl/option.py +++ b/gallery_dl/option.py @@ -86,6 +86,11 @@ def build_parser(): "-p", "--password", metavar="PASS", action=ConfigAction, dest="password", ) + parser.add_argument( + "--netrc", + action=ConfigConstAction, nargs=0, dest="netrc", const=True, + help="Use .netrc authentication data", + ) parser.add_argument( "-i", "--input-file", metavar="FILE", dest="inputfile",