github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[reddit] add metadata to extracted URLs

Browse Source
This commit is contained in:
Mike Fährmann
2018-12-29 17:52:43 +01:00
parent 2f4f60de33
commit 4ab0960083
1 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
gallery_dl/extractor/reddit.py
View File

@@ -36,7 +36,7 @@ class RedditExtractor(Extractor):
util.SPECIAL_EXTRACTORS, [RedditSubredditExtractor]):
while True:
extra = []
for url in self._urls(submissions):
for url, data in self._urls(submissions):
if url[0] == "#":
continue
if url[0] == "/":
@@ -46,7 +46,7 @@ class RedditExtractor(Extractor):
if match:
extra.append(match.group(1))
else:
yield Message.Queue, text.unescape(url), {}
yield Message.Queue, text.unescape(url), data
if not extra or depth == self.max_depth:
return
@@ -62,11 +62,18 @@ class RedditExtractor(Extractor):
def _urls(self, submissions):
for submission, comments in submissions:
self._visited.add(submission["id"])
if not submission["is_self"]:
yield submission["url"]
strings = [submission["selftext_html"] or ""]
strings += [c["body_html"] or "" for c in comments]
yield from text.extract_iter("".join(strings), ' href="', '"')
yield submission["url"], submission
for url in text.extract_iter(
submission["selftext_html"] or "", ' href="', '"'):
yield url, submission
for comment in comments:
for url in text.extract_iter(
comment["body_html"] or "", ' href="', '"'):
yield url, comment
class RedditSubredditExtractor(RedditExtractor):
@@ -103,7 +110,6 @@ class RedditSubmissionExtractor(RedditExtractor):
test = [
("https://www.reddit.com/r/lavaporn/comments/2a00np/", {
"pattern": r"https?://i\.imgur\.com/AaAUCgy\.jpg",
"count": 1,
}),
("https://old.reddit.com/r/lavaporn/comments/2a00np/", None),
("https://np.reddit.com/r/lavaporn/comments/2a00np/", None),