Revert "[deviantart] extend 'extra' option"

This reverts commit
5ad2b9c82b,
5c32a7bf58, and
83f465faca.

(#1387, #1356)
This commit is contained in:
Mike Fährmann
2021-03-19 16:24:23 +01:00
parent 58b93635ee
commit b0438c8f99
3 changed files with 15 additions and 38 deletions

View File

@@ -755,23 +755,14 @@ Description
extractor.deviantart.extra extractor.deviantart.extra
-------------------------- --------------------------
Type Type
``string`` or ``list`` of ``strings`` ``bool``
Default Default
``null`` ``false``
Example
``stash,posts``
Description Description
A (comma-separated) list of extra content to extract Download extra Sta.sh resources from
from description texts and journals. description texts and journals.
Possible values are Note: Enabling this option also enables deviantart.metadata_.
``"posts"`` (embedded DeviantArt posts) and
``"stash"`` (Sta.sh resources).
You can use ``"all"`` instead of listing all values separately.
Note: Enabling this option also enables
`deviantart.metadata <extractor.deviantart.metadata_>`_.
extractor.deviantart.flat extractor.deviantart.flat
@@ -2793,6 +2784,7 @@ Description
.. _base-directory: `extractor.*.base-directory`_ .. _base-directory: `extractor.*.base-directory`_
.. _date-format: `extractor.*.date-format`_ .. _date-format: `extractor.*.date-format`_
.. _deviantart.metadata: `extractor.deviantart.metadata`_
.. _postprocessors: `extractor.*.postprocessors`_ .. _postprocessors: `extractor.*.postprocessors`_
.. _download archive: `extractor.*.archive`_ .. _download archive: `extractor.*.archive`_

View File

@@ -58,7 +58,7 @@
}, },
"deviantart": "deviantart":
{ {
"extra": null, "extra": false,
"flat": true, "flat": true,
"folders": false, "folders": false,
"include": "gallery", "include": "gallery",

View File

@@ -78,21 +78,7 @@ class DeviantartExtractor(Extractor):
else: else:
self.user = profile["user"]["username"] self.user = profile["user"]["username"]
extra = self.extra yield Message.Version, 1
if extra:
if extra is True or extra == "all":
extra = (DeviantartStashExtractor,
DeviantartDeviationExtractor)
else:
items = extra
extra = []
if isinstance(items, str):
items = items.split(",")
if "stash" in items:
extra.append(DeviantartStashExtractor)
if "posts" in items:
extra.append(DeviantartDeviationExtractor)
for deviation in self.deviations(): for deviation in self.deviations():
if isinstance(deviation, tuple): if isinstance(deviation, tuple):
url, data = deviation url, data = deviation
@@ -141,18 +127,17 @@ class DeviantartExtractor(Extractor):
if "excerpt" in deviation and self.commit_journal: if "excerpt" in deviation and self.commit_journal:
journal = self.api.deviation_content(deviation["deviationid"]) journal = self.api.deviation_content(deviation["deviationid"])
if extra: if self.extra:
deviation["_journal"] = journal["html"] deviation["_journal"] = journal["html"]
yield self.commit_journal(deviation, journal) yield self.commit_journal(deviation, journal)
if extra: if self.extra:
txt = (deviation.get("description", "") + txt = (deviation.get("description", "") +
deviation.get("_journal", "")) deviation.get("_journal", ""))
for extr in extra: for match in DeviantartStashExtractor.pattern.finditer(txt):
for match in extr.pattern.finditer(txt): url = text.ensure_http_scheme(match.group(0))
url = text.ensure_http_scheme(match.group(0)) deviation["_extractor"] = DeviantartStashExtractor
deviation["_extractor"] = extr yield Message.Queue, url, deviation
yield Message.Queue, url, deviation
def deviations(self): def deviations(self):
"""Return an iterable containing all relevant Deviation-objects""" """Return an iterable containing all relevant Deviation-objects"""
@@ -780,7 +765,7 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
}), }),
# sta.sh URLs from description (#302) # sta.sh URLs from description (#302)
(("https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498"), { (("https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498"), {
"options": (("extra", "stash"), ("original", False)), "options": (("extra", 1), ("original", 0)),
"pattern": DeviantartStashExtractor.pattern, "pattern": DeviantartStashExtractor.pattern,
"range": "2-", "range": "2-",
"count": 4, "count": 4,