[bellazon] add 'num_internal' & 'num_external' metadata fields (#8415)
This commit is contained in:
@@ -47,7 +47,7 @@ class BellazonExtractor(Extractor):
|
|||||||
post["count"] = data["count"] = len(urls)
|
post["count"] = data["count"] = len(urls)
|
||||||
|
|
||||||
yield Message.Directory, data
|
yield Message.Directory, data
|
||||||
data["num"] = 0
|
data["num"] = data["num_internal"] = data["num_external"] = 0
|
||||||
for info, url, url_img in urls:
|
for info, url, url_img in urls:
|
||||||
url = text.unescape(url or url_img)
|
url = text.unescape(url or url_img)
|
||||||
|
|
||||||
@@ -59,6 +59,7 @@ class BellazonExtractor(Extractor):
|
|||||||
):
|
):
|
||||||
continue
|
continue
|
||||||
data["num"] += 1
|
data["num"] += 1
|
||||||
|
data["num_internal"] += 1
|
||||||
if not (alt := text.extr(info, ' alt="', '"')) or (
|
if not (alt := text.extr(info, ' alt="', '"')) or (
|
||||||
alt.startswith("post-") and "_thumb." in alt):
|
alt.startswith("post-") and "_thumb." in alt):
|
||||||
name = url
|
name = url
|
||||||
@@ -80,6 +81,8 @@ class BellazonExtractor(Extractor):
|
|||||||
yield Message.Url, url, dc
|
yield Message.Url, url, dc
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
data["num"] += 1
|
||||||
|
data["num_external"] += 1
|
||||||
yield Message.Queue, url, data
|
yield Message.Queue, url, data
|
||||||
|
|
||||||
def _pagination(self, base, pnum=None):
|
def _pagination(self, base, pnum=None):
|
||||||
|
|||||||
@@ -23,7 +23,9 @@ __tests__ = (
|
|||||||
"filename" : str,
|
"filename" : str,
|
||||||
"extension": "jpg",
|
"extension": "jpg",
|
||||||
"count" : 5,
|
"count" : 5,
|
||||||
"num" : range(1, 5),
|
"num" : range(1, 5),
|
||||||
|
"num_internal": range(1, 5),
|
||||||
|
"num_external": 0,
|
||||||
"post" : {
|
"post" : {
|
||||||
"author_id" : "72476",
|
"author_id" : "72476",
|
||||||
"author_slug": "shepherd",
|
"author_slug": "shepherd",
|
||||||
@@ -134,7 +136,7 @@ __tests__ = (
|
|||||||
"extension": "mp4",
|
"extension": "mp4",
|
||||||
"filename" : r"re:^\d+$",
|
"filename" : r"re:^\d+$",
|
||||||
"id" : r"re:6361\d\d\d",
|
"id" : r"re:6361\d\d\d",
|
||||||
"num" : range(2, 11),
|
"num" : range(2, 12),
|
||||||
"post" : {
|
"post" : {
|
||||||
"author_id" : "101807",
|
"author_id" : "101807",
|
||||||
"author_slug": "rogerdanish",
|
"author_slug": "rogerdanish",
|
||||||
|
|||||||
Reference in New Issue
Block a user