diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 2180fbee..d5d2bfe1 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -33,6 +33,9 @@ class Extractor(): def items(self): yield Message.Version, 1 + def skip(self, num): + return 0 + def request(self, url, encoding=None, *args, **kwargs): response = safe_request(self.session, url, *args, **kwargs) if encoding: diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 5a992267..28350376 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -23,7 +23,13 @@ class Job(): raise exception.NoExtractorError(url) items = config.get(("images",)) - self.pred_url = util.RangePredicate(items) if items else True + if items: + pred = util.RangePredicate(items) + if pred.lower > 1: + pred.index += self.extractor.skip(pred.lower - 1) + self.pred_url = pred + else: + self.pred_url = True items = config.get(("chapters",)) self.pred_queue = util.RangePredicate(items) if items else True diff --git a/gallery_dl/util.py b/gallery_dl/util.py index c1fe005e..e4910688 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -13,17 +13,14 @@ from . import exception def parse_range(rangespec): - """Parse an integer range and return the resulting ranges and upper limit + """Parse an integer range string and return the resulting ranges Examples - parse_range("-2,4,6-8,10-") - -> [(1,2), (4,4), (6,8), (10,INTMAX)], INTMAX - - parse_range(" - 3 , 4- 4, 2-6") - -> [(1,3), (4,4), (2,6)], 6 + parse_range("-2,4,6-8,10-") -> [(1,2), (4,4), (6,8), (10,INTMAX)] + parse_range(" - 3 , 4- 4, 2-6") -> [(1,3), (4,4), (2,6)] """ ranges = [] - limit = 0 + for group in rangespec.split(","): parts = group.split("-", maxsplit=1) try: @@ -33,23 +30,52 @@ def parse_range(rangespec): else: beg = int(parts[0]) if parts[0].strip() else 1 end = int(parts[1]) if parts[1].strip() else sys.maxsize - ranges.append((beg, end)) - limit = max(limit, end) + ranges.append((beg, end) if beg <= end else (end, beg)) except ValueError: pass - return ranges, limit + + return ranges + + +def optimize_range(ranges): + """Simplify/Combine a parsed list of ranges + + Examples + optimize_range([(2,4), (4,6), (5,8)]) -> [(2,8)] + optimize_range([(1,1), (2,2), (3,6), (8,9))]) -> [(1,6), (8-9)] + """ + if len(ranges) <= 1: + return ranges + + ranges.sort() + riter = iter(ranges) + result = [] + + beg, end = next(riter) + for lower, upper in riter: + if lower > end+1: + result.append((beg, end)) + beg, end = lower, upper + elif upper > end: + end = upper + result.append((beg, end)) + return result class RangePredicate(): """Predicate; is True if the current index is in the given range""" def __init__(self, rangespec): - self.ranges, self.limit = parse_range(rangespec) + self.ranges = optimize_range(parse_range(rangespec)) self.index = 0 + if self.ranges: + self.lower, self.upper = self.ranges[0][0], self.ranges[-1][1] + else: + self.lower, self.upper = 0, 0 def __bool__(self): self.index += 1 - if self.index > self.limit: + if self.index > self.upper: raise exception.StopExtraction() for lower, upper in self.ranges: