github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[urlgalleries] fix extraction

Browse Source 
accept HTTP status 500 as valid pages
This commit is contained in:
Mike Fährmann
2024-10-11 20:12:11 +02:00
parent 1023b2d85d
commit 39b7d748f6
1 changed files with 18 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
gallery_dl/extractor/urlgalleries.py
View File

@@ -7,7 +7,7 @@
"""Extractors for https://urlgalleries.net/""" """Extractors for https://urlgalleries.net/"""
from .common import GalleryExtractor, Message from .common import GalleryExtractor, Message
from .. import text from .. import text, exception
class UrlgalleriesGalleryExtractor(GalleryExtractor): class UrlgalleriesGalleryExtractor(GalleryExtractor):
@@ -16,27 +16,31 @@ class UrlgalleriesGalleryExtractor(GalleryExtractor):
root = "urlgalleries.net" root = "urlgalleries.net"
request_interval = (0.5, 1.0) request_interval = (0.5, 1.0)
pattern = r"(?:https?://)(?:(\w+)\.)?urlgalleries\.net/(?:[\w-]+-)?(\d+)" pattern = r"(?:https?://)(?:(\w+)\.)?urlgalleries\.net/(?:[\w-]+-)?(\d+)"
example = "https://blog.urlgalleries.net/gallery-12345/TITLE" example = "https://BLOG.urlgalleries.net/gallery-12345/TITLE"
def __init__(self, match):
self.blog, self.gallery_id = match.groups()
url = "https://{}.urlgalleries.net/porn-gallery-{}/?a=10000".format(
self.blog, self.gallery_id)
GalleryExtractor.__init__(self, match, url)
def items(self): def items(self):
page = self.request(self.gallery_url).text blog, self.gallery_id = self.groups
url = "https://{}.urlgalleries.net/porn-gallery-{}/?a=10000".format(
blog, self.gallery_id)
with self.request(url, allow_redirects=False, fatal=...) as response:
if 300 <= response.status_code < 500:
if response.headers.get("location", "").endswith(
"/not_found_adult.php"):
raise exception.NotFoundError("gallery")
raise exception.HttpError(None, response)
page = response.text
imgs = self.images(page) imgs = self.images(page)
data = self.metadata(page) data = self.metadata(page)
data["count"] = len(imgs) data["count"] = len(imgs)
del page
root = "https://{}.urlgalleries.net".format(self.blog) root = "https://{}.urlgalleries.net".format(blog)
yield Message.Directory, data yield Message.Directory, data
for data["num"], img in enumerate(imgs, 1): for data["num"], img in enumerate(imgs, 1):
response = self.request( page = self.request(root + img).text
root + img, method="HEAD", allow_redirects=False) url = text.extr(page, "window.location.href = '", "'")
yield Message.Queue, response.headers["Location"], data yield Message.Queue, url, data
def metadata(self, page): def metadata(self, page):
extr = text.extract_from(page) extr = text.extract_from(page)