[deviantart] rename 'external' to 'stash' (#302)
restrict extracted URLs to ones from https://sta.sh/...
This commit is contained in:
@@ -395,18 +395,6 @@ Description Try to follow external URLs of embedded players.
|
|||||||
=========== =====
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
extractor.deviantart.external
|
|
||||||
-----------------------------
|
|
||||||
=========== =====
|
|
||||||
Type ``bool``
|
|
||||||
Default ``false``
|
|
||||||
Description Try to follow external URLs in description fields.
|
|
||||||
|
|
||||||
Note: deviantart.metadata_ needs to be enabled to make descriptions
|
|
||||||
available.
|
|
||||||
=========== =====
|
|
||||||
|
|
||||||
|
|
||||||
extractor.deviantart.flat
|
extractor.deviantart.flat
|
||||||
-------------------------
|
-------------------------
|
||||||
=========== =====
|
=========== =====
|
||||||
@@ -499,6 +487,17 @@ Description The ``refresh_token`` value you get from linking your
|
|||||||
=========== =====
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
|
extractor.deviantart.stash
|
||||||
|
--------------------------
|
||||||
|
=========== =====
|
||||||
|
Type ``bool``
|
||||||
|
Default ``false``
|
||||||
|
Description Extract Sta.sh resources from description texts.
|
||||||
|
|
||||||
|
Note: Enabling this option also enables deviantart.metadata_.
|
||||||
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
extractor.deviantart.wait-min
|
extractor.deviantart.wait-min
|
||||||
-----------------------------
|
-----------------------------
|
||||||
=========== =====
|
=========== =====
|
||||||
|
|||||||
@@ -22,13 +22,13 @@
|
|||||||
"deviantart":
|
"deviantart":
|
||||||
{
|
{
|
||||||
"refresh-token": null,
|
"refresh-token": null,
|
||||||
"external": false,
|
|
||||||
"flat": true,
|
"flat": true,
|
||||||
"folders": false,
|
"folders": false,
|
||||||
"journals": "html",
|
"journals": "html",
|
||||||
"mature": true,
|
"mature": true,
|
||||||
"metadata": false,
|
"metadata": false,
|
||||||
"original": true,
|
"original": true,
|
||||||
|
"stash": false,
|
||||||
"wait-min": 0
|
"wait-min": 0
|
||||||
},
|
},
|
||||||
"exhentai":
|
"exhentai":
|
||||||
|
|||||||
@@ -38,11 +38,14 @@ class DeviantartExtractor(Extractor):
|
|||||||
self.api = DeviantartAPI(self)
|
self.api = DeviantartAPI(self)
|
||||||
self.offset = 0
|
self.offset = 0
|
||||||
self.flat = self.config("flat", True)
|
self.flat = self.config("flat", True)
|
||||||
|
self.stash = self.config("stash", False)
|
||||||
self.original = self.config("original", True)
|
self.original = self.config("original", True)
|
||||||
self.external = self.config("external", False)
|
|
||||||
self.user = match.group(1) or match.group(2)
|
self.user = match.group(1) or match.group(2)
|
||||||
self.group = False
|
self.group = False
|
||||||
|
|
||||||
|
if self.stash:
|
||||||
|
self.api.metadata = True
|
||||||
|
|
||||||
self.commit_journal = {
|
self.commit_journal = {
|
||||||
"html": self._commit_journal_html,
|
"html": self._commit_journal_html,
|
||||||
"text": self._commit_journal_text,
|
"text": self._commit_journal_text,
|
||||||
@@ -96,12 +99,11 @@ class DeviantartExtractor(Extractor):
|
|||||||
journal = self.api.deviation_content(deviation["deviationid"])
|
journal = self.api.deviation_content(deviation["deviationid"])
|
||||||
yield self.commit_journal(deviation, journal)
|
yield self.commit_journal(deviation, journal)
|
||||||
|
|
||||||
if self.external:
|
if self.stash:
|
||||||
for url in text.extract_iter(
|
for match in DeviantartStashExtractor.pattern.finditer(
|
||||||
deviation.get("description", ""), 'href="', '"'):
|
deviation.get("description", "")):
|
||||||
if "deviantart.com/users/outgoing?" in url:
|
deviation["_extractor"] = DeviantartStashExtractor
|
||||||
url = text.unquote(url.partition("?")[2])
|
yield Message.Queue, match.group(0), deviation
|
||||||
yield Message.Queue, url, deviation
|
|
||||||
|
|
||||||
def deviations(self):
|
def deviations(self):
|
||||||
"""Return an iterable containing all relevant Deviation-objects"""
|
"""Return an iterable containing all relevant Deviation-objects"""
|
||||||
@@ -372,10 +374,10 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
|
|||||||
# external URLs from description (#302)
|
# external URLs from description (#302)
|
||||||
(("https://www.deviantart.com/uotapo/art/"
|
(("https://www.deviantart.com/uotapo/art/"
|
||||||
"INANAKI-Memorial-Humane7-590297498"), {
|
"INANAKI-Memorial-Humane7-590297498"), {
|
||||||
"options": (("external", 1), ("metadata", 1), ("original", 0)),
|
"options": (("stash", 1), ("original", 0)),
|
||||||
"pattern": r"https?://(sta\.sh|youtu\.be)/\w+$",
|
"pattern": r"https?://sta\.sh/\w+$",
|
||||||
"range": "2-",
|
"range": "2-",
|
||||||
"count": 6,
|
"count": 4,
|
||||||
}),
|
}),
|
||||||
# old-style URLs
|
# old-style URLs
|
||||||
("https://shimoda7.deviantart.com"
|
("https://shimoda7.deviantart.com"
|
||||||
|
|||||||
Reference in New Issue
Block a user