32 lines
949 B
Python
32 lines
949 B
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2015-2025 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Recursive extractor"""
|
|
|
|
from .common import Extractor, Message
|
|
from .. import text
|
|
|
|
|
|
class RecursiveExtractor(Extractor):
|
|
"""Extractor that fetches URLs from a remote or local source"""
|
|
category = "recursive"
|
|
pattern = r"r(?:ecursive)?:"
|
|
example = "recursive:https://pastebin.com/raw/FLwrCYsT"
|
|
|
|
def items(self):
|
|
url = self.url.partition(":")[2]
|
|
|
|
if url.startswith("file://"):
|
|
with open(url[7:], encoding="utf-8") as fp:
|
|
page = fp.read()
|
|
else:
|
|
page = self.request(text.ensure_http_scheme(url)).text
|
|
|
|
for match in text.re(r"https?://[^\s\"']+").finditer(page):
|
|
yield Message.Queue, match[0], {}
|