[deviantart] add 'external' option (#302)

If a description is available, this will extract URLs from the
description text and try to find Extractors for them.
This commit is contained in:
Mike Fährmann
2019-06-06 18:53:50 +02:00
parent f85e42cffc
commit 2fb85178da
3 changed files with 30 additions and 0 deletions

View File

@@ -39,6 +39,7 @@ class DeviantartExtractor(Extractor):
self.offset = 0
self.flat = self.config("flat", True)
self.original = self.config("original", True)
self.external = self.config("external", False)
self.user = match.group(1) or match.group(2)
self.group = False
@@ -95,6 +96,13 @@ class DeviantartExtractor(Extractor):
journal = self.api.deviation_content(deviation["deviationid"])
yield self.commit_journal(deviation, journal)
if self.external:
for url in text.extract_iter(
deviation.get("description", ""), 'href="', '"'):
if "deviantart.com/users/outgoing?" in url:
url = text.unquote(url.partition("?")[2])
yield Message.Queue, url, deviation
def deviations(self):
"""Return an iterable containing all relevant Deviation-objects"""
return []
@@ -361,6 +369,14 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
"pattern": (r"https://images-wixmp-\w+\.wixmp\.com"
r"/f/[^/]+/[^.]+\.gif\?token="),
}),
# external URLs from description (#302)
(("https://www.deviantart.com/uotapo/art/"
"INANAKI-Memorial-Humane7-590297498"), {
"options": (("external", 1), ("metadata", 1), ("original", 0)),
"pattern": r"https?://(sta\.sh|youtu\.be)/\w+$",
"range": "2-",
"count": 6,
}),
# old-style URLs
("https://shimoda7.deviantart.com"
"/art/For-the-sake-of-a-memory-10073852"),