add '--filter' command-line option

This allows for image filtering via Python expressions by the same
metadata that is also used to build filenames (--list-keywords).

The usually shunned eval() function is used to evaluate
filter-expressions, but it seemed quite appropriate in this case and
shouldn't introduce any new security issues, as any attacker that could do
> gallery-dl --filter "delete-everything()" ...
could as well do
> python -c "delete-everything()"

gallery_dl/__init__.py

@@ -57,6 +57,25 @@ def sanatize_input(file):
             yield line
 
 
+def prepare_range(rangespec, target):
+    if rangespec:
+        range = util.optimize_range(util.parse_range(rangespec))
+        if range:
+            config.set(("_", target, "range"), range)
+        else:
+            log.warning("invalid/empty %s range", target)
+
+
+def prepare_filter(filterexpr, target):
+    if filterexpr:
+        try:
+            name = "<{} filter>".format(target)
+            codeobj = compile(filterexpr, name, "eval")
+            config.set(("_", target, "filter"), codeobj)
+        except (SyntaxError, ValueError, TypeError) as exc:
+            log.warning(exc)
+
+
 def main():
     try:
         initialize_logging()
@@ -64,6 +83,7 @@ def main():
         args = parser.parse_args()
         logging.getLogger().setLevel(args.loglevel)
 
+        # configuration
         if args.load_config:
             config.load()
         if args.cfgfiles:
@@ -72,6 +92,9 @@ def main():
             config.load(*args.yamlfiles, format="yaml", strict=True)
         for key, value in args.options:
             config.set(key, value)
+        config.set(("_",), {})
+
+        # logging
         if args.loglevel >= logging.ERROR:
             config.set(("output", "mode"), "null")
         elif args.loglevel <= logging.DEBUG:
@@ -128,6 +151,11 @@ def main():
                 except OSError as exc:
                     log.warning("unsupported-URL file: %s", exc)
 
+            prepare_range(args.image_range, "image")
+            prepare_range(args.chapter_range, "chapter")
+            prepare_filter(args.image_filter, "image")
+            # prepare_filter(args.chapter_filter, "chapter")
+
             pformat = config.get(("output", "progress"), True)
             if pformat and len(urls) > 1 and args.loglevel < logging.ERROR:
                 urls = progress(urls, pformat)

gallery_dl/exception.py

@@ -19,6 +19,7 @@ Exception
       |    +-- HttpError
       +-- NoExtractorError
       +-- FormatError
+      +-- FilterError
       +-- StopExtraction
 """
 
@@ -55,5 +56,9 @@ class FormatError(GalleryDLException):
     """Error while building output path"""
 
 
+class FilterError(GalleryDLException):
+    """Error while evaluating a filter expression"""
+
+
 class StopExtraction(GalleryDLException):
     """Extraction should stop"""

gallery_dl/extractor/deviantart.py

@@ -291,7 +291,7 @@ class DeviantartCollectionExtractor(DeviantartExtractor):
                r"/favourites/(\d+)/([^/?&#]+)"]
     test = [("http://rosuuri.deviantart.com/favourites/58951174/Useful", {
         "url": "f0c12581060aab9699289817b39804d9eb88f675",
-        "keyword": "e0ed920fb3dfdad9294be592be2eeb3dc1258a6a",
+        "keyword": "2778b4abaac240ff6fb1d630d7b04b8e983ef9c4",
     })]
 
     def __init__(self, match):

gallery_dl/job.py

@@ -27,23 +27,23 @@ class Job():
 
         # url predicates
         predicates = [util.UniquePredicate()]
-
-        items = config.get(("images",))
-        if items:
-            pred = util.RangePredicate(items)
+        image = config.get(("_", "image"), {})
+        if "range" in image:
+            pred = util.RangePredicate(image["range"])
             if pred.lower > 1:
                 pred.index += self.extractor.skip(pred.lower - 1)
             predicates.append(pred)
-
+        if "filter" in image:
+            predicates.append(util.FilterPredicate(image["filter"]))
         self.pred_url = util.build_predicate(predicates)
 
         # queue predicates
         predicates = []
-
-        items = config.get(("chapters",))
-        if items:
-            predicates.append(util.RangePredicate(items))
-
+        chapter = config.get(("_", "chapter"), {})
+        if "range" in chapter:
+            predicates.append(util.RangePredicate(chapter["range"]))
+        if "filter" in chapter:
+            predicates.append(util.FilterPredicate(chapter["filter"]))
         self.pred_queue = util.build_predicate(predicates)
 
     def run(self):
@@ -65,8 +65,12 @@ class Job():
             log.error("HTTP request failed:\n%s", exc)
         except exception.FormatError as exc:
             err, obj = exc.args
-            log.error("Applying %s format string failed:\n%s: %s",
+            log.error("Applying %s format string failed:  %s: %s",
                       obj, err.__class__.__name__, err)
+        except exception.FilterError as exc:
+            err = exc.args[0]
+            log.error("Evaluating filter expression failed:  %s: %s",
+                      err.__class__.__name__, err)
         except exception.StopExtraction:
             pass
         except OSError as exc:

gallery_dl/option.py

@@ -190,16 +190,30 @@ def build_parser():
     selection = parser.add_argument_group("Selection Options")
     selection.add_argument(
         "--images",
-        metavar="ITEM-SPEC", action=ConfigAction, dest="images",
+        metavar="ITEM-SPEC", dest="image_range",
         help=("Specify which images to download through a comma seperated list"
               " of indices or index-ranges; "
               "for example '--images -2,4,6-8,10-' will download images with "
-              "index 1, 2, 4, 6, 7, 8 and 10 up to the last one")
+              "index 1, 2, 4, 6, 7, 8 and 10 up to the last one"),
     )
     selection.add_argument(
         "--chapters",
-        metavar="ITEM-SPEC", action=ConfigAction, dest="chapters",
-        help=("Same as '--images' except for chapters")
+        metavar="ITEM-SPEC", dest="chapter_range",
+        help="Same as '--images' except for chapters",
+    )
+    selection.add_argument(
+        "--filter",
+        metavar="EXPR", dest="image_filter",
+        help=("Python expression controlling which images to download. Images "
+              "for which the expression evaluates to False are ignored. "
+              "Available keys are the filename-specific ones listed by "
+              "'--list-keywords'. Example: --filter \"image_width >= 1000 and "
+              "rating in ('s', 'q')\""),
+    )
+    selection.add_argument(
+        "--chapter-filter",
+        metavar="EXPR", dest="chapter_filter",
+        help="Same as '--filter' except for chapters (not yet implemented)",
     )
 
     parser.add_argument(

gallery_dl/util.py

@@ -150,8 +150,8 @@ def build_predicate(predicates):
 
 class RangePredicate():
     """Predicate; True if the current index is in the given range"""
-    def __init__(self, rangespec):
-        self.ranges = optimize_range(parse_range(rangespec))
+    def __init__(self, ranges):
+        self.ranges = ranges
         self.index = 0
         if self.ranges:
             self.lower, self.upper = self.ranges[0][0], self.ranges[-1][1]
@@ -182,6 +182,20 @@ class UniquePredicate():
         return False
 
 
+class FilterPredicate():
+    """Predicate; True if evaluating the given expression returns True"""
+    globalsdict = {"__builtins__": {}}
+
+    def __init__(self, codeobj):
+        self.codeobj = codeobj
+
+    def __call__(self, url, kwds):
+        try:
+            return eval(self.codeobj, self.globalsdict, kwds)
+        except Exception as exc:
+            raise exception.FilterError(exc)
+
+
 class ChainPredicate():
     """Predicate; True if all of its predicates return True"""
     def __init__(self, predicates):