[util] don't add text: URLs to list of downloaded URLs
This commit is contained in:
@@ -223,6 +223,8 @@ class UniquePredicate():
|
||||
self.urls = set()
|
||||
|
||||
def __call__(self, url, kwds):
|
||||
if url.startswith("text:"):
|
||||
return True
|
||||
if url not in self.urls:
|
||||
self.urls.add(url)
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user