diff --git a/gallery_dl/util.py b/gallery_dl/util.py index d85d2b37..09f851ea 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -48,12 +48,22 @@ def bdecode(data, alphabet="0123456789"): def advance(iterable, num): - """"Advance the iterable by 'num' steps""" + """"Advance 'iterable' by 'num' steps""" iterator = iter(iterable) next(itertools.islice(iterator, num, num), None) return iterator +def unique(iterable): + """Yield unique elements from 'iterable' while preserving order""" + seen = set() + add = seen.add + for element in iterable: + if element not in seen: + add(element) + yield element + + def raises(cls): """Returns a function that raises 'cls' as exception""" def wrap(*args): diff --git a/test/test_util.py b/test/test_util.py index 08ecd643..fd659a07 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -446,6 +446,16 @@ class TestOther(unittest.TestCase): self.assertCountEqual( util.advance(util.advance(items, 1), 2), range(3, 5)) + def test_unique(self): + self.assertSequenceEqual( + list(util.unique("")), "") + self.assertSequenceEqual( + list(util.unique("AABBCC")), "ABC") + self.assertSequenceEqual( + list(util.unique("ABABABCAABBCC")), "ABC") + self.assertSequenceEqual( + list(util.unique([1, 2, 1, 3, 2, 1])), [1, 2, 3]) + def test_raises(self): func = util.raises(Exception) with self.assertRaises(Exception):