[deviantart] refactor 'extra' (#1356)

- change its expected type to string
- let users specify a list of sources (stash, posts) or 'all'

docs/configuration.rst

@@ -745,15 +745,20 @@ Description
 extractor.deviantart.extra
 --------------------------
 Type
-    ``bool`` or ``string``
+    ``string`` or ``list`` of ``strings``
 Default
-    ``false``
+    ``null``
+Example
+    ``stash,posts``
 Description
-    Download embedded Deviations and Sta.sh resources from
+    A (comma-separated) list of extra content to extract
-    description texts and journals.
+    from description texts and journals.
 
-    Set this option to ``"stash"`` or ``"deviations"``
+    Possible values are
-    to select only one of them as a source.
+    ``"posts"`` (embedded DeviantArt posts) and
+    ``"stash"`` (Sta.sh resources).
+
+    You can use ``"all"`` instead of listing all values separately.
 
     Note: Enabling this option also enables
     `deviantart.metadata <extractor.deviantart.metadata_>`_.

docs/gallery-dl.conf

@@ -58,7 +58,7 @@
         },
         "deviantart":
         {
-            "extra": false,
+            "extra": null,
             "flat": true,
             "folders": false,
             "include": "gallery",

gallery_dl/extractor/deviantart.py

@@ -80,18 +80,19 @@ class DeviantartExtractor(Extractor):
 
         extra = self.extra
         if extra:
-            if extra == "stash":
+            if extra is True or extra == "all":
-                extra_stash = DeviantartStashExtractor.pattern.finditer
+                extra = (DeviantartStashExtractor,
-                extra_deviation = None
+                         DeviantartDeviationExtractor)
-            elif extra == "deviations":
-                extra_deviation = DeviantartDeviationExtractor.pattern.finditer
-                extra_stash = None
             else:
-                extra_stash = DeviantartStashExtractor.pattern.finditer
+                items = extra
-                extra_deviation = DeviantartDeviationExtractor.pattern.finditer
+                extra = []
-            extra = True
+                if isinstance(items, str):
+                    items = items.split(",")
+                if "stash" in items:
+                    extra.append(DeviantartStashExtractor)
+                if "posts" in items:
+                    extra.append(DeviantartDeviationExtractor)
 
-        yield Message.Version, 1
         for deviation in self.deviations():
             if isinstance(deviation, tuple):
                 url, data = deviation
@@ -147,15 +148,10 @@ class DeviantartExtractor(Extractor):
             if extra:
                 txt = (deviation.get("description", "") +
                        deviation.get("_journal", ""))
-                if extra_stash:
+                for extr in extra:
-                    for match in extra_stash(txt):
+                    for match in extr.pattern.finditer(txt):
                         url = text.ensure_http_scheme(match.group(0))
-                        deviation["_extractor"] = DeviantartStashExtractor
+                        deviation["_extractor"] = extr
-                        yield Message.Queue, url, deviation
-                if extra_deviation:
-                    for match in extra_deviation(txt):
-                        url = text.ensure_http_scheme(match.group(0))
-                        deviation["_extractor"] = DeviantartDeviationExtractor
                         yield Message.Queue, url, deviation
 
     def deviations(self):
@@ -784,7 +780,7 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
         }),
         # sta.sh URLs from description (#302)
         (("https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498"), {
-            "options": (("extra", 1), ("original", 0)),
+            "options": (("extra", "stash"), ("original", False)),
             "pattern": DeviantartStashExtractor.pattern,
             "range": "2-",
             "count": 4,