github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[shopify] skip consecutive duplicate products

Browse Source 
Not filtering duplicate URLs anymore caused the archive ID uniqueness
test to fail.
This commit is contained in:
Mike Fährmann
2019-07-01 20:04:57 +02:00
parent b89f0d8d3c
commit bc6b0cfddc
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gallery_dl/extractor/shopify.py
View File

@@ -89,10 +89,14 @@ class ShopifyCollectionExtractor(ShopifyExtractor):
while True: while True:
page = self.request(self.item_url, params=params).text page = self.request(self.item_url, params=params).text
urls = search_re.findall(page) urls = search_re.findall(page)
last = None
if not urls: if not urls:
return return
for path in urls: for path in urls:
if last == path:
continue
last = path
yield self.root + path yield self.root + path
params["page"] += 1 params["page"] += 1