From 62cc47755bfcfed5fdf73e7f27fc2674a9e74b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 13 Jul 2022 16:59:42 +0200 Subject: [PATCH] [nozomi] reduce memory consumption during searches (#2754) only load and use the entire 'index.nozomi' database if there are only negative search terms --- gallery_dl/extractor/nozomi.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/gallery_dl/extractor/nozomi.py b/gallery_dl/extractor/nozomi.py index 7d7c3f8d..d4348340 100644 --- a/gallery_dl/extractor/nozomi.py +++ b/gallery_dl/extractor/nozomi.py @@ -193,25 +193,27 @@ class NozomiSearchExtractor(NozomiExtractor): return {"search_tags": self.tags} def posts(self): - index = None - result = set() + result = None def nozomi(path): url = "https://j.nozomi.la/" + path + ".nozomi" return decode_nozomi(self.request(url).content) + positive, negative = [], [] for tag in self.tags: - tag = tag.replace("/", "") - if tag[0] == "-": - if not index: - index = set(nozomi("index")) - items = index.difference(nozomi("nozomi/" + tag[1:])) - else: - items = nozomi("nozomi/" + tag) + (negative if tag[0] == "-" else positive).append( + tag.replace("/", "")) - if result: - result.intersection_update(items) + for tag in positive: + ids = nozomi("nozomi/" + tag) + if result is None: + result = set(ids) else: - result.update(items) + result.intersection_update(ids) - return sorted(result, reverse=True) + for tag in negative: + if result is None: + result = set(nozomi("index")) + result.difference_update(nozomi("nozomi/" + tag[1:])) + + return sorted(result, reverse=True) if result else ()