[recursive] simplify
This commit is contained in:
@@ -9,7 +9,6 @@
|
|||||||
"""Recursive extractor"""
|
"""Recursive extractor"""
|
||||||
|
|
||||||
from .common import Extractor, Message
|
from .common import Extractor, Message
|
||||||
import requests
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
|
||||||
@@ -20,29 +19,13 @@ class RecursiveExtractor(Extractor):
|
|||||||
example = "recursive:https://pastebin.com/raw/FLwrCYsT"
|
example = "recursive:https://pastebin.com/raw/FLwrCYsT"
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
self.session.mount("file://", FileAdapter())
|
url = self.url.partition(":")[2]
|
||||||
page = self.request(self.url.partition(":")[2]).text
|
|
||||||
del self.session.adapters["file://"]
|
if url.startswith("file://"):
|
||||||
|
with open(url[7:]) as fp:
|
||||||
|
page = fp.read()
|
||||||
|
else:
|
||||||
|
page = self.request(url).text
|
||||||
|
|
||||||
for match in re.finditer(r"https?://[^\s\"']+", page):
|
for match in re.finditer(r"https?://[^\s\"']+", page):
|
||||||
yield Message.Queue, match.group(0), {}
|
yield Message.Queue, match.group(0), {}
|
||||||
|
|
||||||
|
|
||||||
class FileAdapter(requests.adapters.BaseAdapter):
|
|
||||||
"""Requests adapter for local files"""
|
|
||||||
|
|
||||||
def send(self, request, **kwargs):
|
|
||||||
response = requests.Response()
|
|
||||||
try:
|
|
||||||
response.raw = open(request.url[7:], "rb")
|
|
||||||
except OSError:
|
|
||||||
import io
|
|
||||||
response.raw = io.BytesIO()
|
|
||||||
response.status_code = requests.codes.bad_request
|
|
||||||
else:
|
|
||||||
response.raw.release_conn = response.raw.close
|
|
||||||
response.status_code = requests.codes.ok
|
|
||||||
return response
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
pass
|
|
||||||
|
|||||||
Reference in New Issue
Block a user