github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[util] don't add text: URLs to list of downloaded URLs

Browse Source
This commit is contained in:
Mike Fährmann
2018-02-20 18:14:27 +01:00
parent 8704d850bf
commit ac3da8115e
2 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
gallery_dl/util.py
View File

@@ -223,6 +223,8 @@ class UniquePredicate():
self.urls = set() self.urls = set()
def __call__(self, url, kwds): def __call__(self, url, kwds):
if url.startswith("text:"):
return True
if url not in self.urls: if url not in self.urls:
self.urls.add(url) self.urls.add(url)
return True return True

10
test/test_util.py
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2015-2017 Mike Fährmann # Copyright 2015-2018 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -76,8 +76,9 @@ class TestPredicate(unittest.TestCase):
def test_unique_predicate(self): def test_unique_predicate(self):
dummy = None dummy = None
pred = util.UniquePredicate() pred = util.UniquePredicate()
# no duplicates
self.assertTrue(pred("1", dummy)) self.assertTrue(pred("1", dummy))
self.assertTrue(pred("2", dummy)) self.assertTrue(pred("2", dummy))
self.assertFalse(pred("1", dummy)) self.assertFalse(pred("1", dummy))
@@ -85,6 +86,11 @@ class TestPredicate(unittest.TestCase):
self.assertTrue(pred("3", dummy)) self.assertTrue(pred("3", dummy))
self.assertFalse(pred("3", dummy)) self.assertFalse(pred("3", dummy))
# duplicates for "text:"
self.assertTrue(pred("text:123", dummy))
self.assertTrue(pred("text:123", dummy))
self.assertTrue(pred("text:123", dummy))
def test_build_predicate(self): def test_build_predicate(self):
pred = util.build_predicate([]) pred = util.build_predicate([])
self.assertIsInstance(pred, type(lambda: True)) self.assertIsInstance(pred, type(lambda: True))