[util] don't add text: URLs to list of downloaded URLs

This commit is contained in:
Mike Fährmann
2018-02-20 18:14:27 +01:00
parent 8704d850bf
commit ac3da8115e
2 changed files with 10 additions and 2 deletions

View File

@@ -223,6 +223,8 @@ class UniquePredicate():
self.urls = set()
def __call__(self, url, kwds):
if url.startswith("text:"):
return True
if url not in self.urls:
self.urls.add(url)
return True