replace remaining time.sleep() calls

with Extractor.sleep() or request_interval
This commit is contained in:
Mike Fährmann
2023-02-23 00:35:37 +01:00
parent 6ed4309aba
commit 51301e0c31
3 changed files with 4 additions and 8 deletions

View File

@@ -366,9 +366,7 @@ class DeviantartExtractor(Extractor):
kwargs["fatal"] = None kwargs["fatal"] = None
diff = time.time() - DeviantartExtractor._last_request diff = time.time() - DeviantartExtractor._last_request
if diff < 2.0: if diff < 2.0:
delay = 2.0 - diff self.sleep(2.0 - diff, "request")
self.log.debug("Sleeping %.2f seconds", delay)
time.sleep(delay)
while True: while True:
response = self.request(url, **kwargs) response = self.request(url, **kwargs)
@@ -1496,7 +1494,7 @@ class DeviantartOAuthAPI():
while True: while True:
if self.delay: if self.delay:
time.sleep(self.delay) self.extractor.sleep(self.delay, "api")
self.authenticate(None if public else self.refresh_token_key) self.authenticate(None if public else self.refresh_token_key)
kwargs["headers"] = self.headers kwargs["headers"] = self.headers

View File

@@ -12,7 +12,6 @@ from .common import Extractor, Message
from .. import text, util, exception from .. import text, util, exception
from ..cache import cache from ..cache import cache
import itertools import itertools
import time
BASE_PATTERN = r"(?:https?://)?(?:www\.)?erome\.com" BASE_PATTERN = r"(?:https?://)?(?:www\.)?erome\.com"
@@ -75,7 +74,7 @@ class EromeExtractor(Extractor):
if response.content.find( if response.content.find(
b"<title>Please wait a few moments</title>", 0, 600) < 0: b"<title>Please wait a few moments</title>", 0, 600) < 0:
return response return response
time.sleep(5) self.sleep(5.0, "check")
def _pagination(self, url, params): def _pagination(self, url, params):
for params["page"] in itertools.count(1): for params["page"] in itertools.count(1):

View File

@@ -11,7 +11,6 @@
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, util, exception from .. import text, util, exception
import datetime import datetime
import time
import re import re
@@ -19,6 +18,7 @@ class PlurkExtractor(Extractor):
"""Base class for plurk extractors""" """Base class for plurk extractors"""
category = "plurk" category = "plurk"
root = "https://www.plurk.com" root = "https://www.plurk.com"
request_interval = 1.0
def items(self): def items(self):
urls = self._urls_ex if self.config("comments", False) else self._urls urls = self._urls_ex if self.config("comments", False) else self._urls
@@ -58,7 +58,6 @@ class PlurkExtractor(Extractor):
return return
elif info["has_newer"] < 200: elif info["has_newer"] < 200:
del data["count"] del data["count"]
time.sleep(1)
data["from_response_id"] = info["responses"][-1]["id"] + 1 data["from_response_id"] = info["responses"][-1]["id"] + 1
@staticmethod @staticmethod