replace remaining time.sleep() calls
with Extractor.sleep() or request_interval
This commit is contained in:
@@ -366,9 +366,7 @@ class DeviantartExtractor(Extractor):
|
|||||||
kwargs["fatal"] = None
|
kwargs["fatal"] = None
|
||||||
diff = time.time() - DeviantartExtractor._last_request
|
diff = time.time() - DeviantartExtractor._last_request
|
||||||
if diff < 2.0:
|
if diff < 2.0:
|
||||||
delay = 2.0 - diff
|
self.sleep(2.0 - diff, "request")
|
||||||
self.log.debug("Sleeping %.2f seconds", delay)
|
|
||||||
time.sleep(delay)
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
response = self.request(url, **kwargs)
|
response = self.request(url, **kwargs)
|
||||||
@@ -1496,7 +1494,7 @@ class DeviantartOAuthAPI():
|
|||||||
|
|
||||||
while True:
|
while True:
|
||||||
if self.delay:
|
if self.delay:
|
||||||
time.sleep(self.delay)
|
self.extractor.sleep(self.delay, "api")
|
||||||
|
|
||||||
self.authenticate(None if public else self.refresh_token_key)
|
self.authenticate(None if public else self.refresh_token_key)
|
||||||
kwargs["headers"] = self.headers
|
kwargs["headers"] = self.headers
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ from .common import Extractor, Message
|
|||||||
from .. import text, util, exception
|
from .. import text, util, exception
|
||||||
from ..cache import cache
|
from ..cache import cache
|
||||||
import itertools
|
import itertools
|
||||||
import time
|
|
||||||
|
|
||||||
BASE_PATTERN = r"(?:https?://)?(?:www\.)?erome\.com"
|
BASE_PATTERN = r"(?:https?://)?(?:www\.)?erome\.com"
|
||||||
|
|
||||||
@@ -75,7 +74,7 @@ class EromeExtractor(Extractor):
|
|||||||
if response.content.find(
|
if response.content.find(
|
||||||
b"<title>Please wait a few moments</title>", 0, 600) < 0:
|
b"<title>Please wait a few moments</title>", 0, 600) < 0:
|
||||||
return response
|
return response
|
||||||
time.sleep(5)
|
self.sleep(5.0, "check")
|
||||||
|
|
||||||
def _pagination(self, url, params):
|
def _pagination(self, url, params):
|
||||||
for params["page"] in itertools.count(1):
|
for params["page"] in itertools.count(1):
|
||||||
|
|||||||
@@ -11,7 +11,6 @@
|
|||||||
from .common import Extractor, Message
|
from .common import Extractor, Message
|
||||||
from .. import text, util, exception
|
from .. import text, util, exception
|
||||||
import datetime
|
import datetime
|
||||||
import time
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
|
||||||
@@ -19,6 +18,7 @@ class PlurkExtractor(Extractor):
|
|||||||
"""Base class for plurk extractors"""
|
"""Base class for plurk extractors"""
|
||||||
category = "plurk"
|
category = "plurk"
|
||||||
root = "https://www.plurk.com"
|
root = "https://www.plurk.com"
|
||||||
|
request_interval = 1.0
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
urls = self._urls_ex if self.config("comments", False) else self._urls
|
urls = self._urls_ex if self.config("comments", False) else self._urls
|
||||||
@@ -58,7 +58,6 @@ class PlurkExtractor(Extractor):
|
|||||||
return
|
return
|
||||||
elif info["has_newer"] < 200:
|
elif info["has_newer"] < 200:
|
||||||
del data["count"]
|
del data["count"]
|
||||||
time.sleep(1)
|
|
||||||
data["from_response_id"] = info["responses"][-1]["id"] + 1
|
data["from_response_id"] = info["responses"][-1]["id"] + 1
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
Reference in New Issue
Block a user