[nozomi] reduce memory consumption during searches (#2754)
only load and use the entire 'index.nozomi' database if there are only negative search terms
This commit is contained in:
@@ -193,25 +193,27 @@ class NozomiSearchExtractor(NozomiExtractor):
|
|||||||
return {"search_tags": self.tags}
|
return {"search_tags": self.tags}
|
||||||
|
|
||||||
def posts(self):
|
def posts(self):
|
||||||
index = None
|
result = None
|
||||||
result = set()
|
|
||||||
|
|
||||||
def nozomi(path):
|
def nozomi(path):
|
||||||
url = "https://j.nozomi.la/" + path + ".nozomi"
|
url = "https://j.nozomi.la/" + path + ".nozomi"
|
||||||
return decode_nozomi(self.request(url).content)
|
return decode_nozomi(self.request(url).content)
|
||||||
|
|
||||||
|
positive, negative = [], []
|
||||||
for tag in self.tags:
|
for tag in self.tags:
|
||||||
tag = tag.replace("/", "")
|
(negative if tag[0] == "-" else positive).append(
|
||||||
if tag[0] == "-":
|
tag.replace("/", ""))
|
||||||
if not index:
|
|
||||||
index = set(nozomi("index"))
|
|
||||||
items = index.difference(nozomi("nozomi/" + tag[1:]))
|
|
||||||
else:
|
|
||||||
items = nozomi("nozomi/" + tag)
|
|
||||||
|
|
||||||
if result:
|
for tag in positive:
|
||||||
result.intersection_update(items)
|
ids = nozomi("nozomi/" + tag)
|
||||||
|
if result is None:
|
||||||
|
result = set(ids)
|
||||||
else:
|
else:
|
||||||
result.update(items)
|
result.intersection_update(ids)
|
||||||
|
|
||||||
return sorted(result, reverse=True)
|
for tag in negative:
|
||||||
|
if result is None:
|
||||||
|
result = set(nozomi("index"))
|
||||||
|
result.difference_update(nozomi("nozomi/" + tag[1:]))
|
||||||
|
|
||||||
|
return sorted(result, reverse=True) if result else ()
|
||||||
|
|||||||
Reference in New Issue
Block a user