replace 'wait-min/-max' with 'sleep-request'

on exhentai, idolcomplex, reactor
This commit is contained in:
Mike Fährmann
2021-03-02 22:55:45 +01:00
parent e165e6c265
commit bae874f370
4 changed files with 5 additions and 70 deletions

View File

@@ -11,11 +11,8 @@
from .common import Extractor, Message
from .. import text
import urllib.parse
import random
import time
import json
BASE_PATTERN = r"(?:https?://)?((?:[^/.]+\.)?reactor\.cc)"
@@ -25,17 +22,13 @@ class ReactorExtractor(Extractor):
filename_fmt = "{post_id}_{num:>02}{title[:100]:?_//}.{extension}"
archive_fmt = "{post_id}_{num}"
instances = ()
request_interval = 5.0
def __init__(self, match):
Extractor.__init__(self, match)
self.root = "http://" + match.group(1)
self.session.headers["Referer"] = self.root
self.wait_min = self.config("wait-min", 3)
self.wait_max = self.config("wait-max", 6)
if self.wait_max < self.wait_min:
self.wait_max = self.wait_min
if not self.category:
# set category based on domain name
netloc = urllib.parse.urlsplit(self.root).netloc
@@ -61,8 +54,6 @@ class ReactorExtractor(Extractor):
def _pagination(self, url):
while True:
time.sleep(random.uniform(self.wait_min, self.wait_max))
response = self.request(url)
if response.history:
# sometimes there is a redirect from