github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

smaller code and text changes

Browse Source
This commit is contained in:
Mike Fährmann
2017-09-30 18:52:23 +02:00
parent ad4580800c
commit b319f4bab3
7 changed files with 31 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
gallery_dl/config.py
View File

@@ -66,9 +66,8 @@ def load(*files, format="json", strict=False):
if strict:
log.error("Configuration file '%s' not found", path)
sys.exit(1)
except Exception as exception:
log.warning("Could not parse '%s'", path)
log.warning(exception)
except Exception as exc:
log.warning("Could not parse '%s': %s", path, exc)
if strict:
sys.exit(2)

8
gallery_dl/extractor/common.py
View File

@@ -117,8 +117,8 @@ class AsynchronousExtractor(Extractor):
def __init__(self):
Extractor.__init__(self)
queue_size = int(config.get(("queue-size",), default=5))
self.__queue = queue.Queue(maxsize=queue_size)
queue_size = int(config.get(("queue-size",), 5))
self.__queue = queue.Queue(queue_size)
self.__thread = threading.Thread(target=self.async_items, daemon=True)
def __iter__(self):
@@ -140,8 +140,8 @@ class AsynchronousExtractor(Extractor):
try:
for task in self.items():
put(task)
except Exception as e:
put(e)
except Exception as exc:
put(exc)
put(None)

4
gallery_dl/extractor/pixiv.py
View File

@@ -268,9 +268,7 @@ class PixivRankingExtractor(PixivExtractor):
r"ranking\.php(?:\?([^#]*))?"]
test = [
(("https://www.pixiv.net/ranking.php"
"?mode=daily&content=illust&date=20170818"), {
"pattern": r"^https?://i\d*\.pixiv\.net/img-original/img/.+/\d+_p",
}),
"?mode=daily&content=illust&date=20170818"), None),
("https://www.pixiv.net/ranking.php", None),
]

29
gallery_dl/job.py
View File

@@ -17,7 +17,7 @@ class Job():
"""Base class for Job-types"""
ufile = None
def __init__(self, url):
def __init__(self, url, parent=None):
self.url = url
self.extractor = extractor.find(url)
if self.extractor is None:
@@ -46,6 +46,11 @@ class Job():
predicates.append(util.RangePredicate(chapter["range"]))
self.pred_queue = util.build_predicate(predicates)
# category transfer
if parent and parent.extractor.categorytransfer:
self.extractor.category = parent.extractor.category
self.extractor.subcategory = parent.extractor.subcategory
def run(self):
"""Execute or run the job"""
try:
@@ -121,12 +126,6 @@ class Job():
kwdict["category"] = self.extractor.category
kwdict["subcategory"] = self.extractor.subcategory
def _prepare(self, job):
if self.extractor.categorytransfer:
job.extractor.category = self.extractor.category
job.extractor.subcategory = self.extractor.subcategory
return job
def _write_unsupported(self, url):
if self.ufile:
print(url, file=self.ufile, flush=True)
@@ -135,7 +134,7 @@ class Job():
class DownloadJob(Job):
"""Download images into appropriate directory/filename locations"""
def __init__(self, url):
def __init__(self, url, parent=None):
Job.__init__(self, url)
self.pathfmt = util.PathFormat(self.extractor)
self.downloaders = {}
@@ -156,7 +155,7 @@ class DownloadJob(Job):
def handle_queue(self, url, keywords):
try:
self._prepare(DownloadJob(url)).run()
DownloadJob(url, self).run()
except exception.NoExtractorError:
self._write_unsupported(url)
@@ -178,8 +177,8 @@ class KeywordJob(Job):
"""Print available keywords"""
def handle_url(self, url, keywords):
print("\nKeywords for filenames:")
print("-----------------------")
print("\nKeywords for filenames and --filter:")
print("------------------------------------")
self.print_keywords(keywords)
raise exception.StopExtraction()
@@ -200,7 +199,7 @@ class KeywordJob(Job):
self.print_keywords(keywords)
if self.extractor.categorytransfer:
print()
self._prepare(KeywordJob(url)).run()
KeywordJob(url, self).run()
raise exception.StopExtraction()
@staticmethod
@@ -230,8 +229,8 @@ class UrlJob(Job):
"""Print download urls"""
maxdepth = -1
def __init__(self, url, depth=1):
Job.__init__(self, url)
def __init__(self, url, parent=None, depth=1):
Job.__init__(self, url, parent)
self.depth = depth
if depth == self.maxdepth:
self.handle_queue = self.handle_url
@@ -242,7 +241,7 @@ class UrlJob(Job):
def handle_queue(self, url, _):
try:
self._prepare(UrlJob(url, self.depth + 1)).run()
UrlJob(url, self, self.depth + 1).run()
except exception.NoExtractorError:
self._write_unsupported(url)

15
gallery_dl/option.py
View File

@@ -71,7 +71,7 @@ def build_parser():
general.add_argument(
"-h", "--help",
action="help",
help="Print this help message and exit"
help="Print this help message and exit",
)
general.add_argument(
"--version",
@@ -121,7 +121,8 @@ def build_parser():
)
output.add_argument(
"-K", "--list-keywords", dest="list_keywords", action="store_true",
help="Print a list of available keywords for the given URLs",
help=("Print a list of available keywords and example values "
"for the given URLs"),
)
output.add_argument(
"--list-modules", dest="list_modules", action="store_true",
@@ -148,7 +149,7 @@ def build_parser():
"--abort-on-skip",
action=ConfigConstAction, nargs=0, dest="skip", const="abort",
help=("Abort extractor run if a file download would normally be "
"skipped, i.e. if a file with the same filename already exists")
"skipped, i.e. if a file with the same filename already exists"),
)
configuration = parser.add_argument_group("Configuration Options")
@@ -193,13 +194,14 @@ def build_parser():
metavar="RANGE", dest="image_range",
help=("Specify which images to download through a comma seperated list"
" of indices or index-ranges; "
"for example '--images -2,4,6-8,10-' will download images with "
"for example '--range -2,4,6-8,10-' will download images with "
"index 1, 2, 4, 6, 7, 8 and 10 up to the last one"),
)
selection.add_argument(
"--chapter-range",
metavar="RANGE", dest="chapter_range",
help="Same as '--range' except for chapters and other delegated URLs",
help=("Same as '--range' except for chapters "
"and other transferred URLs"),
)
selection.add_argument(
"--filter",
@@ -213,7 +215,8 @@ def build_parser():
selection.add_argument(
"--chapter-filter",
metavar="EXPR", dest="chapter_filter",
help="Same as '--filter' except for chapters and other delegated URLs",
help=("Same as '--filter' except for chapters "
"and other transferred URLs"),
)
selection.add_argument(
"--images", dest="depr_images",

2
gallery_dl/util.py
View File

@@ -237,7 +237,7 @@ class Formatter():
Extra Format Specifiers:
- "?<before>/<after>/":
Adds <before> and <after> to the actual value if it evaluates to True.
Otherwise the whole replacement field beomes an empty string.
Otherwise the whole replacement field becomes an empty string.
Example: {f:?-+/+-/} -> "-+Example+-" (if "f" contains "Example")
-> "" (if "f" is None, 0, "")
"""

1
test/test_extractors.py
View File

@@ -69,7 +69,6 @@ skip = [
"imgtrex",
"luscious", # "high load"
"pawoo",
"mangahere", # unstable; sometimes sends empty responses
]
# enable selective testing for direct calls
if __name__ == '__main__' and len(sys.argv) > 1: