diff --git a/docs/configuration.rst b/docs/configuration.rst index cf5381b7..5965da7d 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -6939,6 +6939,26 @@ Description See `metadata.event`_ for a list of available events. +exec.session +------------ +Type + ``bool`` +Default + ``false`` +Description + Start subprocesses in a new session. + + On Windows, this means passing + `CREATE_NEW_PROCESS_GROUP `__ + as a ``creationflags`` argument to + `subprocess.Popen `__ + + On POSIX systems, this means enabling the + ``start_new_session`` argument of + `subprocess.Popen `__ + to have it call ``setsid()``. + + hash.chunk-size --------------- Type diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py index 280b9bc3..55fdd1ac 100644 --- a/gallery_dl/postprocessor/exec.py +++ b/gallery_dl/postprocessor/exec.py @@ -10,12 +10,14 @@ from .common import PostProcessor from .. import util, formatter +import subprocess import os if util.WINDOWS: def quote(s): - return '"' + s.replace('"', '\\"') + '"' + s = s.replace('"', '\\"') + return f'"{s}"' else: from shlex import quote @@ -25,14 +27,21 @@ class ExecPP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) - cmds = options.get("commands") - if cmds: + if cmds := options.get("commands"): self.cmds = [self._prepare_cmd(c) for c in cmds] execute = self.exec_many else: execute, self.args = self._prepare_cmd(options["command"]) if options.get("async", False): - self._exec = self._exec_async + self._exec = self._popen + + self.session = None + self.creationflags = 0 + if options.get("session"): + if util.WINDOWS: + self.creationflags = subprocess.CREATE_NEW_PROCESS_GROUP + else: + self.session = True events = options.get("event") if events is None: @@ -83,8 +92,7 @@ class ExecPP(PostProcessor): return retcode def exec_many(self, pathfmt): - archive = self.archive - if archive: + if archive := self.archive: if archive.check(pathfmt.kwdict): return self.archive = False @@ -92,8 +100,7 @@ class ExecPP(PostProcessor): retcode = 0 for execute, args in self.cmds: self.args = args - retcode = execute(pathfmt) - if retcode: + if retcode := execute(pathfmt): # non-zero exit status break @@ -103,16 +110,19 @@ class ExecPP(PostProcessor): return retcode def _exec(self, args, shell): - self.log.debug("Running '%s'", args) - retcode = util.Popen(args, shell=shell).wait() - if retcode: + if retcode := self._popen(args, shell).wait(): self.log.warning("'%s' returned with non-zero exit status (%d)", args, retcode) return retcode - def _exec_async(self, args, shell): + def _popen(self, args, shell): self.log.debug("Running '%s'", args) - util.Popen(args, shell=shell) + return util.Popen( + args, + shell=shell, + creationflags=self.creationflags, + start_new_session=self.session, + ) def _replace(self, match): name = match[1] diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 91503344..dbc61bc7 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -20,7 +20,7 @@ import collections from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, output, path # noqa E402 +from gallery_dl import extractor, output, path, util # noqa E402 from gallery_dl import postprocessor, config # noqa E402 from gallery_dl.postprocessor.common import PostProcessor # noqa E402 @@ -209,7 +209,10 @@ class ExecTest(BasePostprocessorTest): self.pathfmt.realpath, self.pathfmt.realdirectory, self.pathfmt.filename), - shell=True) + shell=True, + creationflags=0, + start_new_session=None, + ) i.wait.assert_called_once_with() def test_command_list(self): @@ -231,6 +234,8 @@ class ExecTest(BasePostprocessorTest): self.pathfmt.realdirectory.upper(), ], shell=False, + creationflags=0, + start_new_session=None, ) def test_command_many(self): @@ -254,6 +259,8 @@ class ExecTest(BasePostprocessorTest): self.pathfmt.realdirectory, self.pathfmt.filename), shell=True, + creationflags=0, + start_new_session=None, ), call( [ @@ -262,6 +269,8 @@ class ExecTest(BasePostprocessorTest): self.pathfmt.realdirectory.upper(), ], shell=False, + creationflags=0, + start_new_session=None, ), ]) @@ -296,6 +305,47 @@ class ExecTest(BasePostprocessorTest): self.assertTrue(p.called) self.assertFalse(i.wait.called) + @unittest.skipIf(util.WINDOWS, "not POSIX") + def test_session_posix(self): + self._create({ + "session": True, + "command": ["echo", "foobar"], + }) + + with patch("gallery_dl.util.Popen") as p: + i = Mock() + p.return_value = i + self._trigger(("after",)) + + p.assert_called_once_with( + ["echo", "foobar"], + shell=False, + creationflags=0, + start_new_session=True, + ) + i.wait.assert_called_once_with() + + @unittest.skipIf(not util.WINDOWS, "not Windows") + def test_session_windows(self): + self._create({ + "session": True, + "command": ["echo", "foobar"], + }) + + with patch("gallery_dl.util.Popen") as p: + i = Mock() + p.return_value = i + self._trigger(("after",)) + + import subprocess + p.assert_called_once_with( + ["echo", "foobar"], + shell=False, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + start_new_session=False, + ) + i.wait.assert_called_once_with() + class HashTest(BasePostprocessorTest):