github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[pixiv] update API interface

Browse Source 
- start all endpoints with '/'
- use extractor.wait() for rate limit
- retry with while loop instead of recursion
- in case of error, write entire response to debug log
This commit is contained in:
Mike Fährmann
2022-04-29 16:29:12 +02:00
parent 1bc77efa02
commit 82eee72b39
1 changed files with 26 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
gallery_dl/extractor/pixiv.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2014-2021 Mike Fährmann # Copyright 2014-2022 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -10,11 +10,10 @@
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, util, exception from .. import text, util, exception
from ..cache import cache from ..cache import cache, memcache
from datetime import datetime, timedelta from datetime import datetime, timedelta
import itertools import itertools
import hashlib import hashlib
import time
class PixivExtractor(Extractor): class PixivExtractor(Extractor):
@@ -735,66 +734,70 @@ class PixivAppAPI():
def illust_detail(self, illust_id): def illust_detail(self, illust_id):
params = {"illust_id": illust_id} params = {"illust_id": illust_id}
return self._call("v1/illust/detail", params)["illust"] return self._call("/v1/illust/detail", params)["illust"]
def illust_follow(self, restrict="all"): def illust_follow(self, restrict="all"):
params = {"restrict": restrict} params = {"restrict": restrict}
return self._pagination("v2/illust/follow", params) return self._pagination("/v2/illust/follow", params)
def illust_ranking(self, mode="day", date=None): def illust_ranking(self, mode="day", date=None):
params = {"mode": mode, "date": date} params = {"mode": mode, "date": date}
return self._pagination("v1/illust/ranking", params) return self._pagination("/v1/illust/ranking", params)
def illust_related(self, illust_id): def illust_related(self, illust_id):
params = {"illust_id": illust_id} params = {"illust_id": illust_id}
return self._pagination("v2/illust/related", params) return self._pagination("/v2/illust/related", params)
def search_illust(self, word, sort=None, target=None, duration=None, def search_illust(self, word, sort=None, target=None, duration=None,
date_start=None, date_end=None): date_start=None, date_end=None):
params = {"word": word, "search_target": target, params = {"word": word, "search_target": target,
"sort": sort, "duration": duration, "sort": sort, "duration": duration,
"start_date": date_start, "end_date": date_end} "start_date": date_start, "end_date": date_end}
return self._pagination("v1/search/illust", params) return self._pagination("/v1/search/illust", params)
def user_bookmarks_illust(self, user_id, tag=None, restrict="public"): def user_bookmarks_illust(self, user_id, tag=None, restrict="public"):
params = {"user_id": user_id, "tag": tag, "restrict": restrict} params = {"user_id": user_id, "tag": tag, "restrict": restrict}
return self._pagination("v1/user/bookmarks/illust", params) return self._pagination("/v1/user/bookmarks/illust", params)
@memcache()
def user_detail(self, user_id): def user_detail(self, user_id):
params = {"user_id": user_id} params = {"user_id": user_id}
return self._call("v1/user/detail", params) return self._call("/v1/user/detail", params)
def user_following(self, user_id, restrict="public"): def user_following(self, user_id, restrict="public"):
params = {"user_id": user_id, "restrict": restrict} params = {"user_id": user_id, "restrict": restrict}
return self._pagination("v1/user/following", params, "user_previews") return self._pagination("/v1/user/following", params, "user_previews")
def user_illusts(self, user_id): def user_illusts(self, user_id):
params = {"user_id": user_id} params = {"user_id": user_id}
return self._pagination("v1/user/illusts", params) return self._pagination("/v1/user/illusts", params)
def ugoira_metadata(self, illust_id): def ugoira_metadata(self, illust_id):
params = {"illust_id": illust_id} params = {"illust_id": illust_id}
return self._call("v1/ugoira/metadata", params)["ugoira_metadata"] return self._call("/v1/ugoira/metadata", params)["ugoira_metadata"]
def _call(self, endpoint, params=None): def _call(self, endpoint, params=None):
url = "https://app-api.pixiv.net/" + endpoint url = "https://app-api.pixiv.net" + endpoint
self.login() while True:
response = self.extractor.request(url, params=params, fatal=False) self.login()
data = response.json() response = self.extractor.request(url, params=params, fatal=False)
data = response.json()
if "error" not in data:
return data
self.log.debug(data)
if "error" in data:
if response.status_code == 404: if response.status_code == 404:
raise exception.NotFoundError() raise exception.NotFoundError()
error = data["error"] error = data["error"]
if "rate limit" in (error.get("message") or "").lower(): if "rate limit" in (error.get("message") or "").lower():
self.log.info("Waiting two minutes for API rate limit reset.") self.extractor.wait(seconds=300)
time.sleep(120) continue
return self._call(endpoint, params)
raise exception.StopExtraction("API request failed: %s", error)
return data raise exception.StopExtraction("API request failed: %s", error)
def _pagination(self, endpoint, params, key="illusts"): def _pagination(self, endpoint, params, key="illusts"):
while True: while True: