[deviantart] implement 'skip' method
This commit is contained in:
@@ -30,11 +30,16 @@ class DeviantartUserExtractor(Extractor):
|
|||||||
Extractor.__init__(self)
|
Extractor.__init__(self)
|
||||||
self.api = DeviantartAPI(self)
|
self.api = DeviantartAPI(self)
|
||||||
self.user = match.group(1)
|
self.user = match.group(1)
|
||||||
|
self.offset = 0
|
||||||
|
|
||||||
|
def skip(self, num):
|
||||||
|
self.offset += num
|
||||||
|
return num
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
first = True
|
first = True
|
||||||
yield Message.Version, 1
|
yield Message.Version, 1
|
||||||
for deviation in self.api.gallery_all(self.user):
|
for deviation in self.api.gallery_all(self.user, self.offset):
|
||||||
if "content" not in deviation:
|
if "content" not in deviation:
|
||||||
continue
|
continue
|
||||||
if first:
|
if first:
|
||||||
@@ -151,6 +156,7 @@ class DeviantartAPI():
|
|||||||
params["offset"] = data["next_offset"]
|
params["offset"] = data["next_offset"]
|
||||||
else:
|
else:
|
||||||
self.log.error("Unexpected API response: %s", data)
|
self.log.error("Unexpected API response: %s", data)
|
||||||
|
return
|
||||||
|
|
||||||
def authenticate(self):
|
def authenticate(self):
|
||||||
"""Authenticate the application by requesting a bearer token"""
|
"""Authenticate the application by requesting a bearer token"""
|
||||||
@@ -175,12 +181,12 @@ class DeviantartAPI():
|
|||||||
|
|
||||||
def _call(self, url, params={}):
|
def _call(self, url, params={}):
|
||||||
"""Call an API endpoint"""
|
"""Call an API endpoint"""
|
||||||
self.authenticate()
|
tries = 1
|
||||||
tries = 0
|
|
||||||
while True:
|
while True:
|
||||||
if self.delay:
|
if self.delay:
|
||||||
time.sleep(self.delay)
|
time.sleep(self.delay)
|
||||||
|
|
||||||
|
self.authenticate()
|
||||||
response = self.session.get(url, params=params)
|
response = self.session.get(url, params=params)
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
@@ -190,8 +196,10 @@ class DeviantartAPI():
|
|||||||
self.log.debug("rate limit (delay: %d)", self.delay)
|
self.log.debug("rate limit (delay: %d)", self.delay)
|
||||||
else:
|
else:
|
||||||
self.delay = 1
|
self.delay = 1
|
||||||
|
self.log.debug("http status code %d (%d/3)",
|
||||||
|
response.status_code, tries)
|
||||||
tries += 1
|
tries += 1
|
||||||
if tries >= 3:
|
if tries > 3:
|
||||||
raise Exception(response.text)
|
raise Exception(response.text)
|
||||||
try:
|
try:
|
||||||
return response.json()
|
return response.json()
|
||||||
|
|||||||
Reference in New Issue
Block a user