implement basic way to tell extractors to skip ahead

gallery_dl/extractor/common.py

@@ -33,6 +33,9 @@ class Extractor():
     def items(self):
         yield Message.Version, 1
 
+    def skip(self, num):
+        return 0
+
     def request(self, url, encoding=None, *args, **kwargs):
         response = safe_request(self.session, url, *args, **kwargs)
         if encoding:

gallery_dl/job.py

@@ -23,7 +23,13 @@ class Job():
             raise exception.NoExtractorError(url)
 
         items = config.get(("images",))
-        self.pred_url = util.RangePredicate(items) if items else True
+        if items:
+            pred = util.RangePredicate(items)
+            if pred.lower > 1:
+                pred.index += self.extractor.skip(pred.lower - 1)
+            self.pred_url = pred
+        else:
+            self.pred_url = True
 
         items = config.get(("chapters",))
         self.pred_queue = util.RangePredicate(items) if items else True

gallery_dl/util.py

@@ -13,17 +13,14 @@ from . import exception
 
 
 def parse_range(rangespec):
-    """Parse an integer range and return the resulting ranges and upper limit
+    """Parse an integer range string and return the resulting ranges
 
     Examples
-        parse_range("-2,4,6-8,10-")
+        parse_range("-2,4,6-8,10-")      -> [(1,2), (4,4), (6,8), (10,INTMAX)]
-            -> [(1,2), (4,4), (6,8), (10,INTMAX)], INTMAX
+        parse_range(" - 3 , 4-  4, 2-6") -> [(1,3), (4,4), (2,6)]
-
-        parse_range(" - 3 , 4-  4, 2-6")
-            -> [(1,3), (4,4), (2,6)], 6
     """
     ranges = []
-    limit = 0
+
     for group in rangespec.split(","):
         parts = group.split("-", maxsplit=1)
         try:
@@ -33,23 +30,52 @@ def parse_range(rangespec):
             else:
                 beg = int(parts[0]) if parts[0].strip() else 1
                 end = int(parts[1]) if parts[1].strip() else sys.maxsize
-            ranges.append((beg, end))
+            ranges.append((beg, end) if beg <= end else (end, beg))
-            limit = max(limit, end)
         except ValueError:
             pass
-    return ranges, limit
+
+    return ranges
+
+
+def optimize_range(ranges):
+    """Simplify/Combine a parsed list of ranges
+
+    Examples
+        optimize_range([(2,4), (4,6), (5,8)])         -> [(2,8)]
+        optimize_range([(1,1), (2,2), (3,6), (8,9))]) -> [(1,6), (8-9)]
+    """
+    if len(ranges) <= 1:
+        return ranges
+
+    ranges.sort()
+    riter = iter(ranges)
+    result = []
+
+    beg, end = next(riter)
+    for lower, upper in riter:
+        if lower > end+1:
+            result.append((beg, end))
+            beg, end = lower, upper
+        elif upper > end:
+            end = upper
+    result.append((beg, end))
+    return result
 
 
 class RangePredicate():
     """Predicate; is True if the current index is in the given range"""
     def __init__(self, rangespec):
-        self.ranges, self.limit = parse_range(rangespec)
+        self.ranges = optimize_range(parse_range(rangespec))
         self.index = 0
+        if self.ranges:
+            self.lower, self.upper = self.ranges[0][0], self.ranges[-1][1]
+        else:
+            self.lower, self.upper = 0, 0
 
     def __bool__(self):
         self.index += 1
 
-        if self.index > self.limit:
+        if self.index > self.upper:
             raise exception.StopExtraction()
 
         for lower, upper in self.ranges: