27eab4e467
- test more edge cases - consistently return an empty string for invalid arguments - remove the ungreedy-flag in 'remove_html()'
249 lines
8.0 KiB
Python
249 lines
8.0 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2015-2018 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
import unittest
|
|
import sys
|
|
|
|
from gallery_dl import text
|
|
|
|
|
|
INVALID = ((), [], {}, None, 1, 2.3)
|
|
|
|
|
|
class TestText(unittest.TestCase):
|
|
|
|
def test_clean_xml(self, f=text.clean_xml):
|
|
# standard usage
|
|
self.assertEqual(f(""), "")
|
|
self.assertEqual(f("foo"), "foo")
|
|
self.assertEqual(f("\tfoo\nbar\r"), "\tfoo\nbar\r")
|
|
self.assertEqual(f("<foo>\ab\ba\fr\v</foo>"), "<foo>bar</foo>")
|
|
|
|
# 'repl' argument
|
|
repl = "#"
|
|
self.assertEqual(f("", repl), "")
|
|
self.assertEqual(f("foo", repl), "foo")
|
|
self.assertEqual(f("\tfoo\nbar\r", repl), "\tfoo\nbar\r")
|
|
self.assertEqual(
|
|
f("<foo>\ab\ba\fr\v</foo>", repl), "<foo>#b#a#r#</foo>")
|
|
|
|
# removal of all illegal control characters
|
|
value = "".join(chr(x) for x in range(32))
|
|
self.assertEqual(f(value), "\t\n\r")
|
|
|
|
# 'invalid' arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value), "")
|
|
|
|
def test_remove_html(self, f=text.remove_html):
|
|
result = "Hello World."
|
|
|
|
# standard usage
|
|
self.assertEqual(f(""), "")
|
|
self.assertEqual(f("Hello World."), result)
|
|
self.assertEqual(f(" Hello World. "), result)
|
|
self.assertEqual(f("Hello<br/>World."), result)
|
|
self.assertEqual(
|
|
f("<div><b class='a'>Hello</b><i>World.</i></div>"), result)
|
|
|
|
# empty HTML
|
|
self.assertEqual(f("<div></div>"), "")
|
|
self.assertEqual(f(" <div> </div> "), "")
|
|
|
|
# malformed HTML
|
|
self.assertEqual(f("<div</div>"), "")
|
|
self.assertEqual(f("<div<Hello World.</div>"), "")
|
|
|
|
# invalid arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value), "")
|
|
|
|
def test_filename_from_url(self, f=text.filename_from_url):
|
|
result = "filename.ext"
|
|
|
|
# standard usage
|
|
self.assertEqual(f(""), "")
|
|
self.assertEqual(f("filename.ext"), result)
|
|
self.assertEqual(f("/filename.ext"), result)
|
|
self.assertEqual(f("example.org/filename.ext"), result)
|
|
self.assertEqual(f("http://example.org/v2/filename.ext"), result)
|
|
self.assertEqual(
|
|
f("http://example.org/v2/filename.ext?param=value#frag"), result)
|
|
|
|
# invalid arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value), "")
|
|
|
|
def test_nameext_from_url(self, f=text.nameext_from_url):
|
|
empty = {"filename": "", "name": "", "extension": ""}
|
|
result = {"filename": "filename.ext",
|
|
"name": "filename", "extension": "ext"}
|
|
|
|
# standard usage
|
|
self.assertEqual(f(""), empty)
|
|
self.assertEqual(f("filename.ext"), result)
|
|
self.assertEqual(f("/filename.ext"), result)
|
|
self.assertEqual(f("example.org/filename.ext"), result)
|
|
self.assertEqual(f("http://example.org/v2/filename.ext"), result)
|
|
self.assertEqual(
|
|
f("http://example.org/v2/filename.ext?param=value#frag"), result)
|
|
|
|
# invalid arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value), empty)
|
|
|
|
def test_clean_path_windows(self, f=text.clean_path_windows):
|
|
self.assertEqual(f(""), "")
|
|
self.assertEqual(f("foo"), "foo")
|
|
self.assertEqual(f("foo/bar"), "foo_bar")
|
|
self.assertEqual(f("foo<>:\"\\/|?*bar"), "foo_________bar")
|
|
|
|
# invalid arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value), "")
|
|
|
|
def test_clean_path_posix(self, f=text.clean_path_posix):
|
|
self.assertEqual(f(""), "")
|
|
self.assertEqual(f("foo"), "foo")
|
|
self.assertEqual(f("foo/bar"), "foo_bar")
|
|
self.assertEqual(f("foo<>:\"\\/|?*bar"), "foo<>:\"\\_|?*bar")
|
|
|
|
# invalid arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value), "")
|
|
|
|
def test_shorten_path(self):
|
|
cases = {
|
|
"dirname": "dirname",
|
|
"X"*255: "X"*255,
|
|
"X"*256: "X"*255,
|
|
"Ä"*255: "Ä"*127,
|
|
}
|
|
enc = sys.getfilesystemencoding()
|
|
for case, result in cases.items():
|
|
self.assertEqual(text.shorten_path(case), result)
|
|
self.assertTrue(len(text.shorten_path(case).encode(enc)) <= 255)
|
|
|
|
def test_shorten_filename(self):
|
|
self.maxDiff = None
|
|
cases = {
|
|
"filename.ext": "filename.ext",
|
|
"X"*251 + ".ext": "X"*251 + ".ext",
|
|
"X"*255 + ".ext": "X"*251 + ".ext",
|
|
"Ä"*251 + ".ext": "Ä"*125 + ".ext",
|
|
}
|
|
enc = sys.getfilesystemencoding()
|
|
for case, result in cases.items():
|
|
fname = text.shorten_filename(case)
|
|
self.assertEqual(fname, result)
|
|
self.assertTrue(len(fname.encode(enc)) <= 255)
|
|
|
|
def test_extract(self, f=text.extract):
|
|
txt = "<a><b>"
|
|
self.assertEqual(f(txt, "<", ">"), ("a", 3))
|
|
self.assertEqual(f(txt, "X", ">"), (None, 0))
|
|
self.assertEqual(f(txt, "<", "X"), (None, 0))
|
|
|
|
# 'pos' argument
|
|
for i in range(1, 4):
|
|
self.assertEqual(f(txt, "<", ">", i), ("b", 6))
|
|
for i in range(4, 10):
|
|
self.assertEqual(f(txt, "<", ">", i), (None, i))
|
|
|
|
# invalid arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value , "<" , ">") , (None, 0))
|
|
self.assertEqual(f(txt, value, ">") , (None, 0))
|
|
self.assertEqual(f(txt, "<" , value), (None, 0))
|
|
|
|
def test_extract_all(self, f=text.extract_all):
|
|
txt = "[c][b][a]: xyz! [d][e"
|
|
|
|
self.assertEqual(
|
|
f(txt, ()), ({}, 0))
|
|
self.assertEqual(
|
|
f(txt, (("C", "[", "]"), ("B", "[", "]"), ("A", "[", "]"))),
|
|
({"A": "a", "B": "b", "C": "c"}, 9),
|
|
)
|
|
|
|
# 'None' as field name
|
|
self.assertEqual(
|
|
f(txt, ((None, "[", "]"), (None, "[", "]"), ("A", "[", "]"))),
|
|
({"A": "a"}, 9),
|
|
)
|
|
self.assertEqual(
|
|
f(txt, ((None, "[", "]"), (None, "[", "]"), (None, "[", "]"))),
|
|
({}, 9),
|
|
)
|
|
|
|
# failed matches
|
|
self.assertEqual(
|
|
f(txt, (("C", "[", "]"), ("X", "X", "X"), ("B", "[", "]"))),
|
|
({"B": "b", "C": "c", "X": None}, 6),
|
|
)
|
|
|
|
# 'pos' argument
|
|
self.assertEqual(
|
|
f(txt, (("B", "[", "]"), ("A", "[", "]")), pos=1),
|
|
({"A": "a", "B": "b"}, 9),
|
|
)
|
|
|
|
# 'values' argument
|
|
self.assertEqual(
|
|
f(txt, (("C", "[", "]"),), values={"A": "a", "B": "b"}),
|
|
({"A": "a", "B": "b", "C": "c"}, 3),
|
|
)
|
|
|
|
vdict = {}
|
|
rdict, pos = f(txt, (), values=vdict)
|
|
self.assertIs(vdict, rdict)
|
|
|
|
def test_extract_iter(self, f=text.extract_iter):
|
|
txt = "[c][b][a]: xyz! [d][e"
|
|
|
|
def g(*args):
|
|
return list(f(*args))
|
|
|
|
self.assertEqual(
|
|
g("", "[", "]"), [])
|
|
self.assertEqual(
|
|
g("[a]", "[", "]"), ["a"])
|
|
self.assertEqual(
|
|
g(txt, "[", "]"), ["c", "b", "a", "d"])
|
|
self.assertEqual(
|
|
g(txt, "X", "X"), [])
|
|
self.assertEqual(
|
|
g(txt, "[", "]", 6), ["a", "d"])
|
|
|
|
def test_parse_query(self, f=text.parse_query):
|
|
# standard usage
|
|
self.assertEqual(f(""), {})
|
|
self.assertEqual(f("foo=1"), {"foo": "1"})
|
|
self.assertEqual(f("foo=1&bar=2"), {"foo": "1", "bar": "2"})
|
|
|
|
# missing value
|
|
self.assertEqual(f("bar"), {})
|
|
self.assertEqual(f("foo=1&bar"), {"foo": "1"})
|
|
self.assertEqual(f("foo=1&bar&baz=3"), {"foo": "1", "baz": "3"})
|
|
|
|
# keys with identical names
|
|
self.assertEqual(f("foo=1&foo=2"), {"foo": "1"})
|
|
self.assertEqual(
|
|
f("foo=1&bar=2&foo=3&bar=4"),
|
|
{"foo": "1", "bar": "2"},
|
|
)
|
|
|
|
# invalid arguments
|
|
for value in INVALID:
|
|
self.assertEqual(f(value), {})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|