implement restarting an extractor (#3338)

gallery_dl/__init__.py

@@ -250,9 +250,13 @@ def main():
             pformat = config.get(("output",), "progress", True)
             if pformat and len(urls) > 1 and args.loglevel < logging.ERROR:
                 urls = progress(urls, pformat)
+            else:
+                urls = iter(urls)
 
             retval = 0
-            for url in urls:
+            url = next(urls, None)
+
+            while url is not None:
                 try:
                     log.debug("Starting %s for '%s'", jobtype.__name__, url)
                     if isinstance(url, util.ExtendedUrl):
@@ -264,9 +268,15 @@ def main():
                         retval |= jobtype(url).run()
                 except exception.TerminateExtraction:
                     pass
+                except exception.RestartExtraction:
+                    log.debug("Restarting '%s'", url)
+                    continue
                 except exception.NoExtractorError:
                     log.error("Unsupported URL '%s'", url)
                     retval |= 64
+
+                url = next(urls, None)
+
             return retval
 
     except KeyboardInterrupt:

gallery_dl/exception.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2022 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -24,6 +24,7 @@ Exception
       +-- NoExtractorError
       +-- StopExtraction
       +-- TerminateExtraction
+      +-- RestartExtraction
 """
 
 
@@ -115,3 +116,8 @@ class StopExtraction(GalleryDLException):
 class TerminateExtraction(GalleryDLException):
     """Terminate data extraction"""
     code = 0
+
+
+class RestartExtraction(GalleryDLException):
+    """Restart data extraction"""
+    code = 0

gallery_dl/job.py

@@ -93,7 +93,7 @@ class Job():
             if exc.message:
                 log.error(exc.message)
             self.status |= exc.code
-        except exception.TerminateExtraction:
+        except (exception.TerminateExtraction, exception.RestartExtraction):
             raise
         except exception.GalleryDLException as exc:
             log.error("%s: %s", exc.__class__.__name__, exc)
@@ -343,12 +343,18 @@ class DownloadJob(Job):
                     if kwdict:
                         job.kwdict.update(kwdict)
 
-            if pextr.config("parent-skip"):
-                job._skipcnt = self._skipcnt
-                self.status |= job.run()
-                self._skipcnt = job._skipcnt
-            else:
-                self.status |= job.run()
+            while True:
+                try:
+                    if pextr.config("parent-skip"):
+                        job._skipcnt = self._skipcnt
+                        self.status |= job.run()
+                        self._skipcnt = job._skipcnt
+                    else:
+                        self.status |= job.run()
+                    break
+                except exception.RestartExtraction:
+                    pass
+
         else:
             self._write_unsupported(url)
 

gallery_dl/util.py

@@ -592,6 +592,7 @@ GLOBALS = {
     "timedelta": datetime.timedelta,
     "abort"    : raises(exception.StopExtraction),
     "terminate": raises(exception.TerminateExtraction),
+    "restart"  : raises(exception.RestartExtraction),
     "re"       : re,
 }