github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[shopify] skip deleted products (#175)

Browse Source 
Product pages which return a 4xx status code will now be skipped instead
of raising an exception.
This commit is contained in:
Mike Fährmann
2019-06-05 11:12:10 +02:00
parent 096009367b
commit aa1ca4ed35
1 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
gallery_dl/extractor/shopify.py
View File

@@ -24,12 +24,12 @@ class ShopifyExtractor(SharedConfigMixin, Extractor):
Extractor.__init__(self, match) Extractor.__init__(self, match)
self.item_url = self.root + match.group(1) self.item_url = self.root + match.group(1)
def request(self, url, method="GET", expect=(429, 430), **kwargs): def request(self, url, method="GET", expect=range(400, 500), **kwargs):
tries = 0 tries = 0
kwargs["expect"] = expect kwargs["expect"] = expect
while True: while True:
response = Extractor.request(self, url, method, **kwargs) response = Extractor.request(self, url, method, **kwargs)
if response.status_code not in expect: if response.status_code not in (429, 430):
return response return response
tries += 1 tries += 1
waittime = 2 ** (tries + 2) waittime = 2 ** (tries + 2)
@@ -45,8 +45,12 @@ class ShopifyExtractor(SharedConfigMixin, Extractor):
headers = {"X-Requested-With": "XMLHttpRequest"} headers = {"X-Requested-With": "XMLHttpRequest"}
for url in self.products(): for url in self.products():
product = self.request( response = self.request(url + ".json", headers=headers)
url + ".json", headers=headers).json()["product"] if response.status_code >= 400:
self.log.warning('Skipping %s ("%d: %s")',
url, response.status_code, response.reason)
continue
product = response.json()["product"]
del product["image"] del product["image"]
for num, image in enumerate(product.pop("images"), 1): for num, image in enumerate(product.pop("images"), 1):