allow filtering '--list-extractors' results

with blacklist/whitelist syntax, e.g.

--list-extractors pixiv
--list-extractors pixiv:user pixiv:work
--list-extractors :search
This commit is contained in:
Mike Fährmann
2024-09-07 22:41:02 +02:00
parent 0db3c11ab0
commit 4da3347d18
4 changed files with 14 additions and 6 deletions

View File

@@ -49,7 +49,8 @@
values for the given URLs values for the given URLs
-e, --error-file FILE Add input URLs which returned an error to FILE -e, --error-file FILE Add input URLs which returned an error to FILE
--list-modules Print a list of available extractor modules --list-modules Print a list of available extractor modules
--list-extractors Print a list of extractor classes with --list-extractors CATEGORIES
Print a list of extractor classes with
description, (sub)category and example URL description, (sub)category and example URL
--write-log FILE Write logging output to FILE --write-log FILE Write logging output to FILE
--write-unsupported FILE Write URLs, which get emitted by other --write-unsupported FILE Write URLs, which get emitted by other

View File

@@ -202,12 +202,18 @@ def main():
extractor.modules.append("") extractor.modules.append("")
sys.stdout.write("\n".join(extractor.modules)) sys.stdout.write("\n".join(extractor.modules))
elif args.list_extractors: elif args.list_extractors is not None:
write = sys.stdout.write write = sys.stdout.write
fmt = ("{}{}\nCategory: {} - Subcategory: {}" fmt = ("{}{}\nCategory: {} - Subcategory: {}"
"\nExample : {}\n\n").format "\nExample : {}\n\n").format
for extr in extractor.extractors(): extractors = extractor.extractors()
if args.list_extractors:
fltr = util.build_extractor_filter(
args.list_extractors, negate=False)
extractors = filter(fltr, extractors)
for extr in extractors:
write(fmt( write(fmt(
extr.__name__, extr.__name__,
"\n" + extr.__doc__ if extr.__doc__ else "", "\n" + extr.__doc__ if extr.__doc__ else "",

View File

@@ -344,7 +344,7 @@ def build_parser():
) )
output.add_argument( output.add_argument(
"--list-extractors", "--list-extractors",
dest="list_extractors", action="store_true", dest="list_extractors", metavar="CATEGORIES", nargs="*",
help=("Print a list of extractor classes " help=("Print a list of extractor classes "
"with description, (sub)category and example URL"), "with description, (sub)category and example URL"),
) )

View File

@@ -760,8 +760,9 @@ def build_extractor_filter(categories, negate=True, special=None):
if catsub: if catsub:
def test(extr): def test(extr):
for category, subcategory in catsub: for category, subcategory in catsub:
if category in (extr.category, extr.basecategory) and \ if subcategory == extr.subcategory and (
subcategory == extr.subcategory: category == extr.category or
category == extr.basecategory):
return not negate return not negate
return negate return negate
tests.append(test) tests.append(test)