[imgbb] fix & update (#7936)
This commit is contained in:
Mike Fährmann
@@ -16,63 +16,42 @@ from ..cache import cache
|
|||||||
class ImgbbExtractor(Extractor):
|
class ImgbbExtractor(Extractor):
|
||||||
"""Base class for imgbb extractors"""
|
"""Base class for imgbb extractors"""
|
||||||
category = "imgbb"
|
category = "imgbb"
|
||||||
directory_fmt = ("{category}", "{user}")
|
directory_fmt = ("{category}", "{user[name]:?//}{user[id]:? (/)/}",
|
||||||
filename_fmt = "{title} {id}.{extension}"
|
"{album[title]} ({album[id]})")
|
||||||
archive_fmt = "{id}"
|
filename_fmt = "{title} ({id}).{extension}"
|
||||||
root = "https://imgbb.com"
|
archive_fmt = "{user[id]} {id}"
|
||||||
|
cookies_domain = ".imgbb.com"
|
||||||
def __init__(self, match):
|
cookies_names = ("PHPSESSID", "LID")
|
||||||
Extractor.__init__(self, match)
|
root = "https://ibb.co"
|
||||||
self.page_url = self.sort = None
|
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
self.login()
|
self.login()
|
||||||
|
|
||||||
url = self.page_url
|
for image in self.posts():
|
||||||
params = {"sort": self.sort}
|
url = image["url"]
|
||||||
while True:
|
text.nameext_from_url(url, image)
|
||||||
response = self.request(url, params=params, allow_redirects=False)
|
yield Message.Directory, image
|
||||||
if response.status_code < 300:
|
yield Message.Url, url, image
|
||||||
break
|
|
||||||
url = response.headers["location"]
|
|
||||||
if url.startswith(self.root):
|
|
||||||
raise exception.NotFoundError(self.subcategory)
|
|
||||||
|
|
||||||
page = response.text
|
|
||||||
data = self.metadata(page)
|
|
||||||
first = True
|
|
||||||
|
|
||||||
for img in self.images(page):
|
|
||||||
image = {
|
|
||||||
"id" : img["url_viewer"].rpartition("/")[2],
|
|
||||||
"user" : img["user"]["username"] if "user" in img else "",
|
|
||||||
"title" : text.unescape(img["title"]),
|
|
||||||
"url" : img["image"]["url"],
|
|
||||||
"extension": img["image"]["extension"],
|
|
||||||
"size" : text.parse_int(img["image"]["size"]),
|
|
||||||
"width" : text.parse_int(img["width"]),
|
|
||||||
"height" : text.parse_int(img["height"]),
|
|
||||||
}
|
|
||||||
image.update(data)
|
|
||||||
if first:
|
|
||||||
first = False
|
|
||||||
yield Message.Directory, data
|
|
||||||
yield Message.Url, image["url"], image
|
|
||||||
|
|
||||||
def login(self):
|
def login(self):
|
||||||
|
if self.cookies_check(("LID",)):
|
||||||
|
return
|
||||||
|
|
||||||
username, password = self._get_auth_info()
|
username, password = self._get_auth_info()
|
||||||
if username:
|
if username:
|
||||||
self.cookies_update(self._login_impl(username, password))
|
return self.cookies_update(self._login_impl(username, password))
|
||||||
|
|
||||||
@cache(maxage=365*86400, keyarg=1)
|
@cache(maxage=365*86400, keyarg=1)
|
||||||
def _login_impl(self, username, password):
|
def _login_impl(self, username, password):
|
||||||
self.log.info("Logging in as %s", username)
|
self.log.info("Logging in as %s", username)
|
||||||
|
|
||||||
url = self.root + "/login"
|
url = "https://imgbb.com/login"
|
||||||
page = self.request(url).text
|
page = self.request(url).text
|
||||||
token = text.extr(page, 'PF.obj.config.auth_token="', '"')
|
token = text.extr(page, 'name="auth_token" value="', '"')
|
||||||
|
|
||||||
headers = {"Referer": url}
|
headers = {
|
||||||
|
"Referer": url,
|
||||||
|
}
|
||||||
data = {
|
data = {
|
||||||
"auth_token" : token,
|
"auth_token" : token,
|
||||||
"login-subject": username,
|
"login-subject": username,
|
||||||
@@ -84,27 +63,26 @@ class ImgbbExtractor(Extractor):
|
|||||||
raise exception.AuthenticationError()
|
raise exception.AuthenticationError()
|
||||||
return self.cookies
|
return self.cookies
|
||||||
|
|
||||||
def _extract_resource(self, page):
|
def _pagination(self, page, url, params):
|
||||||
return util.json_loads(text.extr(
|
|
||||||
page, "CHV.obj.resource=", "};") + "}")
|
|
||||||
|
|
||||||
def _extract_user(self, page):
|
|
||||||
return self._extract_resource(page).get("user") or {}
|
|
||||||
|
|
||||||
def _pagination(self, page, endpoint, params):
|
|
||||||
data = None
|
|
||||||
seek, pos = text.extract(page, 'data-seek="', '"')
|
seek, pos = text.extract(page, 'data-seek="', '"')
|
||||||
tokn, pos = text.extract(page, 'PF.obj.config.auth_token="', '"', pos)
|
tokn, pos = text.extract(page, 'PF.obj.config.auth_token="', '"', pos)
|
||||||
params["action"] = "list"
|
data, pos = text.extract(page, "CHV.obj.resource=", "};", pos)
|
||||||
params["list"] = "images"
|
self.kwdict["user"] = util.json_loads(data + "}").get("user")
|
||||||
params["sort"] = self.sort
|
|
||||||
params["seek"] = seek
|
|
||||||
params["page"] = 2
|
|
||||||
params["auth_token"] = tokn
|
|
||||||
|
|
||||||
|
data = None
|
||||||
while True:
|
while True:
|
||||||
for img in text.extract_iter(page, "data-object='", "'"):
|
for obj in text.extract_iter(page, "data-object='", "'"):
|
||||||
yield util.json_loads(text.unquote(img))
|
post = util.json_loads(text.unquote(obj))
|
||||||
|
image = post["image"]
|
||||||
|
image["filename"], image["name"] = \
|
||||||
|
image["name"], image["filename"]
|
||||||
|
image["id"] = post["id_encoded"]
|
||||||
|
image["title"] = post["title"]
|
||||||
|
image["width"] = text.parse_int(post["width"])
|
||||||
|
image["height"] = text.parse_int(post["height"])
|
||||||
|
image["size"] = text.parse_int(image["size"])
|
||||||
|
yield image
|
||||||
|
|
||||||
if data:
|
if data:
|
||||||
if not data["seekEnd"] or params["seek"] == data["seekEnd"]:
|
if not data["seekEnd"] or params["seek"] == data["seekEnd"]:
|
||||||
return
|
return
|
||||||
@@ -112,105 +90,111 @@ class ImgbbExtractor(Extractor):
|
|||||||
params["page"] += 1
|
params["page"] += 1
|
||||||
elif not seek or 'class="pagination-next"' not in page:
|
elif not seek or 'class="pagination-next"' not in page:
|
||||||
return
|
return
|
||||||
data = self.request_json(endpoint, method="POST", data=params)
|
else:
|
||||||
|
params["action"] = "list"
|
||||||
|
params["page"] = 2
|
||||||
|
params["seek"] = seek
|
||||||
|
params["auth_token"] = tokn
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||||
|
"X-Requested-With": "XMLHttpRequest",
|
||||||
|
"Origin": self.root,
|
||||||
|
"Sec-Fetch-Dest": "empty",
|
||||||
|
"Sec-Fetch-Mode": "cors",
|
||||||
|
"Sec-Fetch-Site": "same-origin",
|
||||||
|
}
|
||||||
|
|
||||||
|
data = self.request_json(
|
||||||
|
url, method="POST", headers=headers, data=params)
|
||||||
page = data["html"]
|
page = data["html"]
|
||||||
|
|
||||||
|
|
||||||
class ImgbbAlbumExtractor(ImgbbExtractor):
|
class ImgbbAlbumExtractor(ImgbbExtractor):
|
||||||
"""Extractor for albums on imgbb.com"""
|
"""Extractor for albums on imgbb.com"""
|
||||||
subcategory = "album"
|
subcategory = "album"
|
||||||
directory_fmt = ("{category}", "{user}", "{album_name} {album_id}")
|
|
||||||
pattern = r"(?:https?://)?ibb\.co/album/([^/?#]+)/?(?:\?([^#]+))?"
|
pattern = r"(?:https?://)?ibb\.co/album/([^/?#]+)/?(?:\?([^#]+))?"
|
||||||
example = "https://ibb.co/album/ID"
|
example = "https://ibb.co/album/ID"
|
||||||
|
|
||||||
def __init__(self, match):
|
def posts(self):
|
||||||
ImgbbExtractor.__init__(self, match)
|
album_id, qs = self.groups
|
||||||
self.album_name = None
|
url = f"{self.root}/album/{album_id}"
|
||||||
self.album_id = match[1]
|
params = text.parse_query(qs)
|
||||||
self.sort = text.parse_query(match[2]).get("sort", "date_desc")
|
params = {"sort": params["sort"]} if "sort" in params else {}
|
||||||
self.page_url = "https://ibb.co/album/" + self.album_id
|
page = self.request(url, params=params).text
|
||||||
|
extr = text.extract_from(page)
|
||||||
|
|
||||||
def metadata(self, page):
|
self.kwdict["album"] = album = {
|
||||||
album = text.extr(page, '"og:title" content="', '"')
|
"url": extr(
|
||||||
user = self._extract_user(page)
|
'property="og:url" content="', '"'),
|
||||||
return {
|
"title": text.unescape(extr(
|
||||||
"album_id" : self.album_id,
|
'property="og:title" content="', '"')),
|
||||||
"album_name" : text.unescape(album),
|
"description": text.unescape(extr(
|
||||||
"user" : user.get("username") or "",
|
'property="og:description" content="', '"')),
|
||||||
"user_id" : user.get("id") or "",
|
"id": extr(
|
||||||
"displayname": user.get("name") or "",
|
'data-text="album-name" href="https://ibb.co/album/', '"'),
|
||||||
|
"count": text.parse_int(extr(
|
||||||
|
'data-text="image-count">', "<")),
|
||||||
}
|
}
|
||||||
|
|
||||||
def images(self, page):
|
url = f"{self.root}/json"
|
||||||
url = text.extr(page, '"og:url" content="', '"')
|
params["pathname"] = f"/album/{album['id']}"
|
||||||
album_id = url.rpartition("/")[2].partition("?")[0]
|
return self._pagination(page, url, params)
|
||||||
|
|
||||||
return self._pagination(page, "https://ibb.co/json", {
|
|
||||||
"from" : "album",
|
|
||||||
"albumid" : album_id,
|
|
||||||
"params_hidden[list]" : "images",
|
|
||||||
"params_hidden[from]" : "album",
|
|
||||||
"params_hidden[albumid]": album_id,
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
class ImgbbUserExtractor(ImgbbExtractor):
|
|
||||||
"""Extractor for user profiles in imgbb.com"""
|
|
||||||
subcategory = "user"
|
|
||||||
pattern = r"(?:https?://)?([\w-]+)\.imgbb\.com/?(?:\?([^#]+))?$"
|
|
||||||
example = "https://USER.imgbb.com"
|
|
||||||
|
|
||||||
def __init__(self, match):
|
|
||||||
ImgbbExtractor.__init__(self, match)
|
|
||||||
self.user = match[1]
|
|
||||||
self.sort = text.parse_query(match[2]).get("sort", "date_desc")
|
|
||||||
self.page_url = f"https://{self.user}.imgbb.com/"
|
|
||||||
|
|
||||||
def metadata(self, page):
|
|
||||||
user = self._extract_user(page)
|
|
||||||
return {
|
|
||||||
"user" : user.get("username") or self.user,
|
|
||||||
"user_id" : user.get("id") or "",
|
|
||||||
"displayname": user.get("name") or "",
|
|
||||||
}
|
|
||||||
|
|
||||||
def images(self, page):
|
|
||||||
user = text.extr(page, '.obj.resource={"id":"', '"')
|
|
||||||
return self._pagination(page, self.page_url + "json", {
|
|
||||||
"from" : "user",
|
|
||||||
"userid" : user,
|
|
||||||
"params_hidden[userid]": user,
|
|
||||||
"params_hidden[from]" : "user",
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
class ImgbbImageExtractor(ImgbbExtractor):
|
class ImgbbImageExtractor(ImgbbExtractor):
|
||||||
subcategory = "image"
|
subcategory = "image"
|
||||||
pattern = r"(?:https?://)?ibb\.co/(?!album/)([^/?#]+)"
|
pattern = r"(?:https?://)?ibb\.co/([^/?#]+)"
|
||||||
example = "https://ibb.co/ID"
|
example = "https://ibb.co/ID"
|
||||||
|
|
||||||
def __init__(self, match):
|
def posts(self):
|
||||||
ImgbbExtractor.__init__(self, match)
|
url = f"{self.root}/{self.groups[0]}"
|
||||||
self.image_id = match[1]
|
|
||||||
|
|
||||||
def items(self):
|
|
||||||
url = "https://ibb.co/" + self.image_id
|
|
||||||
page = self.request(url).text
|
page = self.request(url).text
|
||||||
extr = text.extract_from(page)
|
extr = text.extract_from(page)
|
||||||
user = self._extract_user(page)
|
|
||||||
|
|
||||||
image = {
|
image = {
|
||||||
"id" : self.image_id,
|
"id" : extr('property="og:url" content="https://ibb.co/', '"'),
|
||||||
"title" : text.unescape(extr(
|
"title" : text.unescape(extr(
|
||||||
'"og:title" content="', ' hosted at ImgBB"')),
|
'"og:title" content="', ' hosted at ImgBB"')),
|
||||||
"url" : extr('"og:image" content="', '"'),
|
"url" : extr('"og:image" content="', '"'),
|
||||||
"width" : text.parse_int(extr('"og:image:width" content="', '"')),
|
"width" : text.parse_int(extr('"og:image:width" content="', '"')),
|
||||||
"height": text.parse_int(extr('"og:image:height" content="', '"')),
|
"height": text.parse_int(extr('"og:image:height" content="', '"')),
|
||||||
"user" : user.get("username") or "",
|
"album" : extr("Added to <a", "</a>"),
|
||||||
"user_id" : user.get("id") or "",
|
"date" : text.parse_datetime(extr(
|
||||||
"displayname": user.get("name") or "",
|
'<span title="', '"'), "%Y-%m-%d %H:%M:%S"),
|
||||||
|
"user" : util.json_loads(extr(
|
||||||
|
"CHV.obj.resource=", "};") + "}").get("user"),
|
||||||
}
|
}
|
||||||
image["extension"] = text.ext_from_url(image["url"])
|
|
||||||
|
|
||||||
yield Message.Directory, image
|
if album := image["album"]:
|
||||||
yield Message.Url, image["url"], image
|
image["album"] = {
|
||||||
|
"id" : text.extr(album, "/album/", '"'),
|
||||||
|
"title": text.unescape(album.rpartition(">")[2]),
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
image["album"] = None
|
||||||
|
|
||||||
|
return (image,)
|
||||||
|
|
||||||
|
|
||||||
|
class ImgbbUserExtractor(ImgbbExtractor):
|
||||||
|
"""Extractor for user profiles in imgbb.com"""
|
||||||
|
subcategory = "user"
|
||||||
|
directory_fmt = ("{category}", "{user[name]} ({user[id]})")
|
||||||
|
pattern = r"(?:https?://)?([\w-]+)\.imgbb\.com/?(?:\?([^#]+))?"
|
||||||
|
example = "https://USER.imgbb.com"
|
||||||
|
|
||||||
|
def posts(self):
|
||||||
|
user, qs = self.groups
|
||||||
|
url = f"https://{user}.imgbb.com/"
|
||||||
|
params = text.parse_query(qs)
|
||||||
|
params = {"sort": params["sort"]} if "sort" in params else {}
|
||||||
|
page = self.request(url, params=params).text
|
||||||
|
|
||||||
|
if "<h1>Sign in</h1>" in page:
|
||||||
|
raise exception.AuthRequired(
|
||||||
|
("username & password", "authenticated cookies"), "user")
|
||||||
|
|
||||||
|
url = f"{url}json"
|
||||||
|
params["pathname"] = "/"
|
||||||
|
return self._pagination(page, url, params)
|
||||||
|
|||||||
@@ -16,31 +16,39 @@ __tests__ = (
|
|||||||
"#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
"#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
||||||
"#count" : 91,
|
"#count" : 91,
|
||||||
"#sha1_url" : "efe7e5a76531436e3b82c87e4ebd34c4dfeb484c",
|
"#sha1_url" : "efe7e5a76531436e3b82c87e4ebd34c4dfeb484c",
|
||||||
"#sha1_metadata": "f1ab5492adb6333409f3367566a6dd7110537e21",
|
|
||||||
|
|
||||||
"album_id" : "i5PggF",
|
"extension": "jpg",
|
||||||
"album_name" : "British Scrap Book",
|
"filename" : str,
|
||||||
"extension" : "jpg",
|
"id" : r"re:^\w{7}$",
|
||||||
"id" : r"re:^\w{7}$",
|
"title" : str,
|
||||||
"title" : str,
|
"url" : r"re:https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
||||||
"url" : r"re:https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
"width" : range(501, 1034),
|
||||||
"user" : "folkie",
|
"height" : range(335, 768),
|
||||||
"user_id" : "GvFMGK",
|
"size" : range(74758, 439037),
|
||||||
"displayname": "Folkie",
|
"mime" : "image/jpeg",
|
||||||
"width" : range(501, 1034),
|
"album" : {
|
||||||
"height" : range(335, 768),
|
"count" : 91,
|
||||||
"size" : range(74758, 439037),
|
"description": "Brief description of this album",
|
||||||
|
"id" : "cgJrwc",
|
||||||
|
"title" : "British Scrap Book",
|
||||||
|
"url" : "https://ibb.co/album/cgJrwc",
|
||||||
|
},
|
||||||
|
"user" : {
|
||||||
|
"id" : "GvFMGK",
|
||||||
|
"name" : "Folkie",
|
||||||
|
"url" : "https://folkie.imgbb.com/",
|
||||||
|
"url_albums": "https://folkie.imgbb.com/albums",
|
||||||
|
"username" : "folkie",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"#url" : "https://ibb.co/album/i5PggF?sort=title_asc",
|
"#url" : "https://ibb.co/album/cgJrwc?sort=name_asc",
|
||||||
"#comment" : "'sort' query argument",
|
"#comment" : "'sort' query argument",
|
||||||
"#category": ("", "imgbb", "album"),
|
|
||||||
"#class" : imgbb.ImgbbAlbumExtractor,
|
"#class" : imgbb.ImgbbAlbumExtractor,
|
||||||
"#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
"#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
||||||
"#count" : 91,
|
"#count" : 91,
|
||||||
"#sha1_url" : "cde36552cc132a27178f22a1b9aceaa4df7e1575",
|
"#sha1_url": "a1bf67f74a6644b360a989a887ff413fd4eab2a6",
|
||||||
"#sha1_metadata": "b98bbb7671e31ebf9c7585fb9fc691b71bcdb546",
|
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -57,7 +65,7 @@ __tests__ = (
|
|||||||
|
|
||||||
{
|
{
|
||||||
"#url" : "https://ibb.co/album/hqgWrF",
|
"#url" : "https://ibb.co/album/hqgWrF",
|
||||||
"#comment" : "private",
|
"#comment" : "private / That page doesn't exist",
|
||||||
"#category": ("", "imgbb", "album"),
|
"#category": ("", "imgbb", "album"),
|
||||||
"#class" : imgbb.ImgbbAlbumExtractor,
|
"#class" : imgbb.ImgbbAlbumExtractor,
|
||||||
"#exception": exception.HttpError,
|
"#exception": exception.HttpError,
|
||||||
@@ -65,28 +73,46 @@ __tests__ = (
|
|||||||
|
|
||||||
{
|
{
|
||||||
"#url" : "https://folkie.imgbb.com",
|
"#url" : "https://folkie.imgbb.com",
|
||||||
"#category": ("", "imgbb", "user"),
|
|
||||||
"#class" : imgbb.ImgbbUserExtractor,
|
"#class" : imgbb.ImgbbUserExtractor,
|
||||||
|
"#auth" : True,
|
||||||
"#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
"#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
|
||||||
"#range" : "1-80",
|
"#range" : "1-80",
|
||||||
|
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"#url" : "https://folkie.imgbb.com",
|
||||||
|
"#class" : imgbb.ImgbbUserExtractor,
|
||||||
|
"#auth" : False,
|
||||||
|
"#exception": exception.AuthRequired,
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"#url" : "https://ibb.co/fUqh5b",
|
"#url" : "https://ibb.co/fUqh5b",
|
||||||
"#category": ("", "imgbb", "image"),
|
|
||||||
"#class" : imgbb.ImgbbImageExtractor,
|
"#class" : imgbb.ImgbbImageExtractor,
|
||||||
"#pattern" : r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg",
|
"#auth" : False,
|
||||||
|
"#results" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg",
|
||||||
"#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e",
|
"#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e",
|
||||||
|
|
||||||
"id" : "fUqh5b",
|
"id" : "qdT0v8H",
|
||||||
"title" : "Arundel Ireeman 5",
|
"title" : "Arundel Ireeman 5",
|
||||||
"url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg",
|
"url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg",
|
||||||
"width" : 960,
|
"width" : 960,
|
||||||
"height" : 719,
|
"height" : 719,
|
||||||
"user" : "folkie",
|
"filename" : "Arundel-Ireeman-5",
|
||||||
"user_id" : "GvFMGK",
|
"extension": "jpg",
|
||||||
"displayname": "Folkie",
|
"date" : "dt:2017-09-29 13:50:25",
|
||||||
"extension" : "jpg",
|
"album" : {
|
||||||
|
"id" : "cgJrwc",
|
||||||
|
"title": "British Scrap Book",
|
||||||
|
},
|
||||||
|
"user" : {
|
||||||
|
"id" : "GvFMGK",
|
||||||
|
"name" : "Folkie",
|
||||||
|
"url" : "https://folkie.imgbb.com/",
|
||||||
|
"url_albums": "https://folkie.imgbb.com/albums",
|
||||||
|
"username" : "folkie",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user