[scripts] remove 'create_test_data' & 'export_tests' (#4504)
- create_test_data.py was superseded by generate_test_result.py - export_tests.py has done its job and is no longer needed
This commit is contained in:
Mike Fährmann
@@ -6,5 +6,5 @@
|
|||||||
# it under the terms of the GNU General Public License version 2 as
|
# it under the terms of the GNU General Public License version 2 as
|
||||||
# published by the Free Software Foundation.
|
# published by the Free Software Foundation.
|
||||||
|
|
||||||
__version__ = "1.30.8"
|
__version__ = "1.31.0-dev"
|
||||||
__variant__ = None
|
__variant__ = None
|
||||||
|
|||||||
@@ -1,69 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
# Copyright 2015-2019 Mike Fährmann
|
|
||||||
#
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License version 2 as
|
|
||||||
# published by the Free Software Foundation.
|
|
||||||
|
|
||||||
"""Create testdata for extractor tests"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
import util # noqa
|
|
||||||
from gallery_dl import extractor
|
|
||||||
from test.test_results import ResultJob, setup_test_config
|
|
||||||
|
|
||||||
|
|
||||||
TESTDATA_FMT = """
|
|
||||||
test = ("{}", {{
|
|
||||||
"url": "{}",
|
|
||||||
"keyword": "{}",
|
|
||||||
"content": "{}",
|
|
||||||
}})
|
|
||||||
"""
|
|
||||||
|
|
||||||
TESTDATA_EXCEPTION_FMT = """
|
|
||||||
test = ("{}", {{
|
|
||||||
"exception": exception.{},
|
|
||||||
}})
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("--content", action="store_true")
|
|
||||||
parser.add_argument("--recreate", action="store_true")
|
|
||||||
parser.add_argument("urls", nargs="*")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.recreate:
|
|
||||||
urls = [
|
|
||||||
test[0]
|
|
||||||
for extr in extractor.extractors() if extr.category in args.urls
|
|
||||||
for test in extr.test
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
urls = args.urls
|
|
||||||
|
|
||||||
setup_test_config()
|
|
||||||
|
|
||||||
for url in urls:
|
|
||||||
tjob = ResultJob(url, content=args.content)
|
|
||||||
try:
|
|
||||||
tjob.run()
|
|
||||||
except Exception as exc:
|
|
||||||
fmt = TESTDATA_EXCEPTION_FMT
|
|
||||||
data = (exc.__class__.__name__,)
|
|
||||||
else:
|
|
||||||
fmt = TESTDATA_FMT
|
|
||||||
data = (tjob.url_hash.hexdigest(),
|
|
||||||
tjob.kwdict_hash.hexdigest(),
|
|
||||||
tjob.content_hash.hexdigest())
|
|
||||||
print(tjob.extractor.__class__.__name__)
|
|
||||||
print(fmt.format(url, *data))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,247 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
# Copyright 2023 Mike Fährmann
|
|
||||||
#
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License version 2 as
|
|
||||||
# published by the Free Software Foundation.
|
|
||||||
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
import itertools
|
|
||||||
import collections
|
|
||||||
|
|
||||||
import util
|
|
||||||
from pyprint import pyprint
|
|
||||||
from gallery_dl import extractor
|
|
||||||
|
|
||||||
|
|
||||||
FORMAT = '''\
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License version 2 as
|
|
||||||
# published by the Free Software Foundation.
|
|
||||||
|
|
||||||
{imports}
|
|
||||||
|
|
||||||
|
|
||||||
__tests__ = (
|
|
||||||
{tests}\
|
|
||||||
)
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
def extract_tests_from_source(lines):
|
|
||||||
tests = {}
|
|
||||||
|
|
||||||
match_url = re.compile(
|
|
||||||
r''' (?:test = | )?\(\(?"([^"]+)"(.*)''').match
|
|
||||||
match_end = re.compile(
|
|
||||||
r" (\}\)| \}\),)\n$").match
|
|
||||||
first = 0
|
|
||||||
url = ""
|
|
||||||
|
|
||||||
for index, line in enumerate(lines):
|
|
||||||
if first and match_end(line):
|
|
||||||
tests[url] = lines[first-1:index+1]
|
|
||||||
first = 0
|
|
||||||
|
|
||||||
elif (m := match_url(line)):
|
|
||||||
offset = index
|
|
||||||
while not m[2]:
|
|
||||||
offset += 1
|
|
||||||
next = lines[offset]
|
|
||||||
line = line[:-2] + next[next.index('"')+1:]
|
|
||||||
m = match_url(line)
|
|
||||||
url = m[1]
|
|
||||||
if m[2] in (",)", "),"):
|
|
||||||
tests[url] = lines[index-1:index+1]
|
|
||||||
first = 0
|
|
||||||
else:
|
|
||||||
first = index
|
|
||||||
|
|
||||||
return tests
|
|
||||||
|
|
||||||
|
|
||||||
def get_test_source(extr, *, cache={}):
|
|
||||||
try:
|
|
||||||
tests = cache[extr.__module__]
|
|
||||||
except KeyError:
|
|
||||||
path = sys.modules[extr.__module__].__file__
|
|
||||||
with util.open(path) as fp:
|
|
||||||
lines = fp.readlines()
|
|
||||||
tests = cache[extr.__module__] = extract_tests_from_source(lines)
|
|
||||||
return tests.get(extr.url) or ("",)
|
|
||||||
return tests[extr.url]
|
|
||||||
|
|
||||||
|
|
||||||
def comment_from_source(source):
|
|
||||||
match = re.match(r"\s+#\s*(.+)", source[0])
|
|
||||||
return match[1] if match else ""
|
|
||||||
|
|
||||||
|
|
||||||
def build_test(extr, data):
|
|
||||||
source = get_test_source(extr)
|
|
||||||
comment = comment_from_source(source)
|
|
||||||
|
|
||||||
head = {
|
|
||||||
"#url" : extr.url,
|
|
||||||
"#comment" : comment.replace('"', "'"),
|
|
||||||
"#category": (extr.basecategory,
|
|
||||||
extr.category,
|
|
||||||
extr.subcategory),
|
|
||||||
"#class" : extr.__class__,
|
|
||||||
}
|
|
||||||
|
|
||||||
if not comment:
|
|
||||||
del head["#comment"]
|
|
||||||
|
|
||||||
instr = {}
|
|
||||||
|
|
||||||
if not data:
|
|
||||||
data = {}
|
|
||||||
if (options := data.pop("options", None)):
|
|
||||||
instr["#options"] = {
|
|
||||||
name: value
|
|
||||||
for name, value in options
|
|
||||||
}
|
|
||||||
if (pattern := data.pop("pattern", None)):
|
|
||||||
if pattern in PATTERNS:
|
|
||||||
cls = PATTERNS[pattern]
|
|
||||||
pattern = f"lit:{pyprint(cls)}.pattern"
|
|
||||||
instr["#pattern"] = pattern
|
|
||||||
if (exception := data.pop("exception", None)):
|
|
||||||
instr["#exception"] = exception
|
|
||||||
if (range := data.pop("range", None)):
|
|
||||||
instr["#range"] = range
|
|
||||||
if (count := data.pop("count", None)) is not None:
|
|
||||||
instr["#count"] = count
|
|
||||||
if (archive := data.pop("archive", None)) is not None:
|
|
||||||
instr["#archive"] = archive
|
|
||||||
if (extractor := data.pop("extractor", None)) is not None:
|
|
||||||
instr["#extractor"] = extractor
|
|
||||||
if (url := data.pop("url", None)):
|
|
||||||
instr["#sha1_url"] = url
|
|
||||||
if (metadata := data.pop("keyword", None)):
|
|
||||||
if isinstance(metadata, str) and len(metadata) == 40:
|
|
||||||
instr["#sha1_metadata"] = metadata
|
|
||||||
metadata = {}
|
|
||||||
if (content := data.pop("content", None)):
|
|
||||||
if isinstance(content, tuple):
|
|
||||||
content = list(content)
|
|
||||||
instr["#sha1_content"] = content
|
|
||||||
|
|
||||||
if data:
|
|
||||||
print(extr)
|
|
||||||
for k in data:
|
|
||||||
print(k)
|
|
||||||
exit()
|
|
||||||
|
|
||||||
return head, instr, metadata
|
|
||||||
|
|
||||||
|
|
||||||
def collect_patterns():
|
|
||||||
return {
|
|
||||||
cls.pattern.pattern: cls
|
|
||||||
for cls in extractor._list_classes()
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def collect_tests(whitelist=None):
|
|
||||||
tests = collections.defaultdict(list)
|
|
||||||
|
|
||||||
for cls in extractor._list_classes():
|
|
||||||
for url, data in cls._get_tests():
|
|
||||||
|
|
||||||
extr = cls.from_url(url)
|
|
||||||
if whitelist and extr.category not in whitelist:
|
|
||||||
continue
|
|
||||||
test = build_test(extr, data)
|
|
||||||
tests[extr.category].append(test)
|
|
||||||
|
|
||||||
return tests
|
|
||||||
|
|
||||||
|
|
||||||
def export_tests(data):
|
|
||||||
imports = {}
|
|
||||||
tests = []
|
|
||||||
|
|
||||||
for head, instr, metadata in data:
|
|
||||||
|
|
||||||
for v in itertools.chain(
|
|
||||||
head.values(),
|
|
||||||
instr.values() if instr else (),
|
|
||||||
metadata.values() if metadata else (),
|
|
||||||
):
|
|
||||||
if not isinstance(v, type) or v.__module__ == "builtins":
|
|
||||||
continue
|
|
||||||
|
|
||||||
module, _, name = v.__module__.rpartition(".")
|
|
||||||
if name[0].isdecimal():
|
|
||||||
stmt = f'''\
|
|
||||||
{module.partition(".")[0]} = __import__("{v.__module__}")
|
|
||||||
_{name} = getattr({module}, "{name}")'''
|
|
||||||
elif module:
|
|
||||||
stmt = f"from {module} import {name}"
|
|
||||||
else:
|
|
||||||
stmt = f"import {name}"
|
|
||||||
imports[v.__module__] = stmt
|
|
||||||
|
|
||||||
test = pyprint(head)
|
|
||||||
if instr:
|
|
||||||
test = f"{test[:-2]}{pyprint(instr)[1:]}"
|
|
||||||
if metadata:
|
|
||||||
for k, v in metadata.items():
|
|
||||||
if v == "type:datetime":
|
|
||||||
imports["datetime"] = "import datetime"
|
|
||||||
metadata[k] = "lit:datetime.datetime"
|
|
||||||
test = f"{test[:-1]}{pyprint(metadata, lmin=0)[1:]}"
|
|
||||||
|
|
||||||
tests.append(f"{test},\n\n")
|
|
||||||
|
|
||||||
return FORMAT.format(
|
|
||||||
imports="\n".join(imports.values()),
|
|
||||||
tests="".join(tests),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PATTERNS = None
|
|
||||||
DIRECTORY = "/tmp/_/results"
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument(
|
|
||||||
"-t", "--target",
|
|
||||||
help="target directory",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-c", "--category", action="append",
|
|
||||||
help="extractor categories to export",
|
|
||||||
)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if not args.target:
|
|
||||||
args.target = os.path.join(
|
|
||||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
|
||||||
"test", "results",
|
|
||||||
)
|
|
||||||
|
|
||||||
global PATTERNS
|
|
||||||
PATTERNS = collect_patterns()
|
|
||||||
|
|
||||||
os.makedirs(args.target, exist_ok=True)
|
|
||||||
for name, tests in collect_tests(args.category).items():
|
|
||||||
name = name.replace(".", "")
|
|
||||||
with util.lazy(f"{args.target}/{name}.py") as fp:
|
|
||||||
fp.write(export_tests(tests))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
Reference in New Issue
Block a user