From 3eaae19345b219c21f7b0e7bc803bc581a90380f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 10 Nov 2015 01:56:31 +0100 Subject: [PATCH 1/7] cmd line switch to set options --- gallery_dl/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index 147e874a..114d56f6 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -32,6 +32,11 @@ def parse_cmdline_options(): metavar="DEST", help="destination directory" ) + parser.add_argument( + "-o", "--option", + metavar="OPT", action="append", + help="option value", + ) parser.add_argument( "urls", nargs="+", metavar="URL", @@ -40,8 +45,11 @@ def parse_cmdline_options(): return parser.parse_args() def main(): + args = parse_cmdline_options() config.load() - opts = parse_cmdline_options() + for opt in args.option: + key, value = opt.split("=", 1) + config.set(key.split("."), value) dlmgr = download.DownloadManager(opts) try: From 21a6416737b95603200950aaea5558d353931579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 12 Nov 2015 02:26:27 +0100 Subject: [PATCH 2/7] bugfixes --- gallery_dl/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index 114d56f6..de2c143e 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -34,7 +34,7 @@ def parse_cmdline_options(): ) parser.add_argument( "-o", "--option", - metavar="OPT", action="append", + metavar="OPT", action="append", default=[], help="option value", ) parser.add_argument( @@ -45,15 +45,18 @@ def parse_cmdline_options(): return parser.parse_args() def main(): - args = parse_cmdline_options() config.load() + args = parse_cmdline_options() for opt in args.option: - key, value = opt.split("=", 1) - config.set(key.split("."), value) - dlmgr = download.DownloadManager(opts) + try: + key, value = opt.split("=", 1) + config.set(key.split("."), value) + except TypeError: + pass + dlmgr = download.DownloadManager(args) try: - for url in opts.urls: + for url in args.urls: dlmgr.add(url) except KeyboardInterrupt: print("\nKeyboardInterrupt") From 352950eebeed9fb2ff6d6746e5905df196dbf0b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 12 Nov 2015 02:29:59 +0100 Subject: [PATCH 3/7] new method to import downloaders --- gallery_dl/downloader/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/gallery_dl/downloader/__init__.py b/gallery_dl/downloader/__init__.py index 8b137891..0750caa8 100644 --- a/gallery_dl/downloader/__init__.py +++ b/gallery_dl/downloader/__init__.py @@ -1 +1,27 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import importlib + +def find(scheme): + """Return downloader class suitable for handling the given scheme""" + try: + return _cache[scheme] + except KeyError: + try: + module = importlib.import_module("."+scheme, __package__) + klass = getattr(module, "Downloader") + _cache[scheme] = klass + return klass + except ImportError: + return None + +# -------------------------------------------------------------------- +# internals + +_cache = {} From 47f016a016fb2f51c6805086156d67e31d7a7829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 12 Nov 2015 02:35:30 +0100 Subject: [PATCH 4/7] remove DownloadManager class --- gallery_dl/download.py | 57 +++++++++++++----------------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/gallery_dl/download.py b/gallery_dl/download.py index 0e737efb..b5ffef20 100644 --- a/gallery_dl/download.py +++ b/gallery_dl/download.py @@ -8,45 +8,17 @@ import os import sys -import importlib -from . import config, extractor, text +from . import config, extractor, downloader, text from .extractor.common import Message -class DownloadManager(): - - def __init__(self, opts): - self.opts = opts - self.modules = {} - - def add(self, url): - job = DownloadJob(self, url) - job.run() - - def get_downloader_module(self, scheme): - """Return a downloader module suitable for 'scheme'""" - module = self.modules.get(scheme) - if module is None: - module = importlib.import_module(".downloader."+scheme, __package__) - self.modules[scheme] = module - return module - - def get_base_directory(self): - """Return the base-destination-directory for downloads""" - if self.opts.dest: - return self.opts.dest - else: - return config.get(("base-directory",), default="/tmp/") - - class DownloadJob(): - def __init__(self, mngr, url): - self.mngr = mngr + def __init__(self, url): self.extractor, self.info = extractor.find(url) if self.extractor is None: print(url, ": No extractor found", sep="", file=sys.stderr) return - self.directory = mngr.get_base_directory() + self.directory = self.get_base_directory() self.downloaders = {} self.filename_fmt = config.get( ("extractor", self.info["category"], "filename"), @@ -91,15 +63,15 @@ class DownloadJob(): if os.path.exists(path): self.print_skip(path) return - downloader = self.get_downloader(url) + dlinstance = self.get_downloader(url) self.print_start(path) - tries = downloader.download(url, path) + tries = dlinstance.download(url, path) self.print_success(path, tries) def set_directory(self, msg): """Set and create the target directory for downloads""" self.directory = os.path.join( - self.mngr.get_base_directory(), + self.get_base_directory(), self.directory_fmt.format(**{ key: text.clean_path(value) for key, value in msg[1].items() }) @@ -112,12 +84,17 @@ class DownloadJob(): scheme = url[:pos] if pos != -1 else "http" if scheme == "https": scheme = "http" - downloader = self.downloaders.get(scheme) - if downloader is None: - module = self.mngr.get_downloader_module(scheme) - downloader = module.Downloader() - self.downloaders[scheme] = downloader - return downloader + instance = self.downloaders.get(scheme) + if instance is None: + klass = downloader.find(scheme) + instance = klass() + self.downloaders[scheme] = instance + return instance + + @staticmethod + def get_base_directory(): + """Return the base-destination-directory for downloads""" + return config.get(("base-directory",), default="/tmp/") @staticmethod def print_start(path): From db113bd87d0fd4402f8b105c875e2875259ae33c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 12 Nov 2015 03:28:01 +0100 Subject: [PATCH 5/7] rename download.py --- gallery_dl/__init__.py | 6 ++---- gallery_dl/{download.py => jobs.py} | 0 2 files changed, 2 insertions(+), 4 deletions(-) rename gallery_dl/{download.py => jobs.py} (100%) diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index de2c143e..7633a8f6 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -17,7 +17,7 @@ __email__ = "mike_faehrmann@web.de" import os import sys import argparse -from . import config, download +from . import config, jobs def parse_cmdline_options(): parser = argparse.ArgumentParser( @@ -53,10 +53,8 @@ def main(): config.set(key.split("."), value) except TypeError: pass - dlmgr = download.DownloadManager(args) - try: for url in args.urls: - dlmgr.add(url) + jobs.DownloadJob(url).run() except KeyboardInterrupt: print("\nKeyboardInterrupt") diff --git a/gallery_dl/download.py b/gallery_dl/jobs.py similarity index 100% rename from gallery_dl/download.py rename to gallery_dl/jobs.py From edf3f7b69fbede314229ac165f38f1ab3360b1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 13 Nov 2015 01:02:49 +0100 Subject: [PATCH 6/7] add KeywordJob class --- gallery_dl/jobs.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/gallery_dl/jobs.py b/gallery_dl/jobs.py index b5ffef20..cc61c8ca 100644 --- a/gallery_dl/jobs.py +++ b/gallery_dl/jobs.py @@ -113,3 +113,32 @@ class DownloadJob(): if tries == 0: print("\r", end="") print("\r\033[1;32m", path, "\033[0m", sep="") + + +class KeywordJob(): + + def __init__(self, url): + self.extractor, self.info = extractor.find(url) + if self.extractor is None: + print(url, ": No extractor found", sep="", file=sys.stderr) + return + + def run(self): + """Execute/Run the download job""" + if self.extractor is None: + return + for msg in self.extractor: + if msg[0] == Message.Url: + print("Keywords for filenames:") + self.print_keywords(msg[2]) + return + elif msg[0] == Message.Directory: + print("Keywords for directory names:") + self.print_keywords(msg[1]) + + @staticmethod + def print_keywords(keywords): + offset = max(map(len, keywords.keys())) + 1 + for key, value in sorted(keywords.items()): + print(key, ":", " "*(offset-len(key)), value, sep="") + print() From 90247977c93af40d210a156420fb8420c3596db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 13 Nov 2015 01:19:01 +0100 Subject: [PATCH 7/7] implement --list-keywords cmd-line option --- gallery_dl/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index 7633a8f6..acdad000 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -37,6 +37,10 @@ def parse_cmdline_options(): metavar="OPT", action="append", default=[], help="option value", ) + parser.add_argument( + "--list-keywords", dest="keywords", action="store_true", + help="print a list of available keywords", + ) parser.add_argument( "urls", nargs="+", metavar="URL", @@ -53,8 +57,9 @@ def main(): config.set(key.split("."), value) except TypeError: pass + jobtype = jobs.KeywordJob if args.keywords else jobs.DownloadJob try: for url in args.urls: - jobs.DownloadJob(url).run() + jobtype(url).run() except KeyboardInterrupt: print("\nKeyboardInterrupt")