[util] don't add text: URLs to list of downloaded URLs

gallery_dl/util.py

@@ -223,6 +223,8 @@ class UniquePredicate():
         self.urls = set()
 
     def __call__(self, url, kwds):
+        if url.startswith("text:"):
+            return True
         if url not in self.urls:
             self.urls.add(url)
             return True

test/test_util.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2017 Mike Fährmann
+# Copyright 2015-2018 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -76,8 +76,9 @@ class TestPredicate(unittest.TestCase):
 
     def test_unique_predicate(self):
         dummy = None
-
         pred = util.UniquePredicate()
+
+        # no duplicates
         self.assertTrue(pred("1", dummy))
         self.assertTrue(pred("2", dummy))
         self.assertFalse(pred("1", dummy))
@@ -85,6 +86,11 @@ class TestPredicate(unittest.TestCase):
         self.assertTrue(pred("3", dummy))
         self.assertFalse(pred("3", dummy))
 
+        # duplicates for "text:"
+        self.assertTrue(pred("text:123", dummy))
+        self.assertTrue(pred("text:123", dummy))
+        self.assertTrue(pred("text:123", dummy))
+
     def test_build_predicate(self):
         pred = util.build_predicate([])
         self.assertIsInstance(pred, type(lambda: True))