github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[pictoa] update

Browse Source 
- simplify code
- update URL patterns
- update tests
- update docs/supportedsites
This commit is contained in:
Mike Fährmann
2025-04-24 17:57:39 +02:00
parent f342108280
commit 29a4444b21
4 changed files with 94 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/supportedsites.md
View File

@@ -694,7 +694,7 @@ Consider all listed sites to potentially be NSFW.
<tr>
<td>Pictoa</td>
<td>https://pictoa.com/</td>
<td>Galleries, individual Images</td>
<td>Albums, individual Images</td>
<td></td>
</tr>
<tr>

2
gallery_dl/extractor/__init__.py
View File

@@ -129,8 +129,8 @@ modules = [
"pexels",
"philomena",
"photovogue",
"pictoa",
"picarto",
"pictoa",
"piczel",
"pillowfort",
"pinterest",

108
gallery_dl/extractor/pictoa.py
View File

@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
@@ -7,104 +8,71 @@
from .common import Extractor, Message
from .. import text
import re
BASE_PATTERN = r"(?:https?://)?(?:[\w]+\.)?pictoa\.com(?:\.de)?"
class PictoaExtractor(Extractor):
"""Base class for pictoa extractors"""
category = "pictoa"
root = "https://pictoa.com"
directory_fmt = ("{category}", "{album_id} {album_title}")
filename_fmt = "{id}.{extension}"
archive_fmt = "{id}"
class PictoaImageExtractor(PictoaExtractor):
"""Extractor for single images from pictoa.com"""
subcategory = "image"
pattern = BASE_PATTERN + r"/albums/([^/\.?#]+)/([^/\.?#]+).html"
filename_fmt = "{id}.{extension}"
directory_fmt = ("{category}", "{album[id]}")
archive_fmt = "{image_id}"
example = "https://www.pictoa.com/albums/name-2693203/12345.html"
def __init__(self, match):
PictoaExtractor.__init__(self, match)
self.album_id = match.group(1)
self.image_id = match.group(2)
pattern = BASE_PATTERN + r"/albums/(?:[\w-]+-)?(\d+)/(\d+)"
example = "https://www.pictoa.com/albums/NAME-12345/12345.html"
def items(self):
url = f"{self.root}/albums/{self.album_id}/{self.image_id}.html"
album_id, image_id = self.groups
url = "{}/albums/{}/{}.html".format(self.root, album_id, image_id)
page = self.request(url).text
container = text.extract(page, '<div id="player"', "</div>")[0]
album_title = text.extract(page, '<meta name="keywords" content="', '"')[0]
image_url = text.extract(container, 'src="', '"')[0]
album_title = text.extr(page, 'property="og:title" content="', '"')
image_url = text.extr(page, 'property="og:image" content="', '"')
data = {
"album": {
"id": self.album_id,
"title": album_title,
},
"image_id": self.image_id,
"id": self.image_id,
"url": image_url,
"album_id" : album_id,
"album_title": album_title.rpartition(" #")[0],
"id" : image_id,
"url" : image_url,
}
text.nameext_from_url(image_url, data)
yield Message.Directory, data
yield Message.Url, image_url, data
class PictoaAlbumExtractor(PictoaExtractor):
"""Extractor for image albums from pictoa.com"""
subcategory = "album"
directory_fmt = ("{category}", "{album[id]} {album[title]}")
archive_fmt = "{album[id]}_{id}"
pattern = BASE_PATTERN + r"/albums/([^/\.?#]+).html"
example = "https://www.pictoa.com/albums/name-2693203.html"
def __init__(self, match):
PictoaExtractor.__init__(self, match)
self.album_id = match.group(1)
pattern = BASE_PATTERN + r"/albums/(?:[\w-]+-)?(\d+).html"
example = "https://www.pictoa.com/albums/NAME-12345.html"
def items(self):
url = f"{self.root}/albums/{self.album_id}.html"
album_id = self.groups[0]
url = "{}/albums/{}.html".format(self.root, album_id)
page = self.request(url).text
title = text.extract(page, '<h1>', '</h1>')[0]
# grab the id out of the title (handiest place to get it)
htmltitle = text.extract(page, '<title>', '</title>')[0]
album_id = text.extract(htmltitle, '#', ' ')[0]
# tags
taghunk = text.extract(page, '<ol class="related-categories bt"', '</ol>')
tags = re.compile(r"\s<li><a href=\".*\">([\d\w ]+)</a>").findall(taghunk[0])
album_data = {
"album": {
"id": album_id,
"title": title
},
"date": None,
"title": title,
"tags": tags,
"album_id" : album_id,
"album_title": text.extr(page, "<h1>", "<"),
"tags" : text.split_html(text.extr(
page, '<ol class="related-categories', '</ol>'))[1:],
"_extractor" : PictoaImageExtractor,
}
yield Message.Directory, album_data
# paginate through the pages
pagination = text.extract(page, '<ul class="pagination"', '</ul>')[0]
findall = re.compile(self.pattern).findall
pages = findall(pagination)
while True:
container = text.extr(page, '<main>', '<span id="flag" >')
for url in text.extract_iter(
container, '<a rel="nofollow" href="', '"'):
yield Message.Queue, url, album_data
# run a quick dedupe
page_set = {self.album_id: page}
for i in pages:
if i not in page_set:
page_set[i] = self.request(f"{self.root}/albums/{i}.html").text
for id, page in page_set.items():
# we'll use the span#flag as the ending token for the
# galleries portion of the page.
image_container = text.extract(page, '<main>', '<span id="flag" >')[0]
for url in text.extract_iter(image_container, '<a rel="nofollow" href="', '"'):
data = {
"url": url,
"album_id": album_id,
"_extractor": PictoaImageExtractor
}
data.update(album_data)
yield Message.Queue, url, data
url = text.extr(page, '<link rel="next" href="', '"')
if not url:
break
page = self.request(url).text

85
test/results/pictoa.py
View File

@@ -6,44 +6,67 @@
from gallery_dl.extractor import pictoa
__tests__ = (
{
"#url" : "https://www.pictoa.com/albums/anna-kendrick-busty-in-a-strapless-red-dress-out-in-nyc-3229225.html",
"#category": ("", "pictoa", "album"),
"#class" : pictoa.PictoaAlbumExtractor,
"#pattern" : r"https://?(?:[\w]+\.)?pictoa\.com/albums/[\w-]+/[\d]+.html",
"#count" : 16,
"album_id" : "3229225",
"date" : None,
"title" : "Anna Kendrick busty in a strapless red dress out in NYC",
"tags" : ["Anna Kendrick", "Celebrity"]
"#url" : "https://www.pictoa.com/albums/anna-kendrick-busty-in-a-strapless-red-dress-out-in-nyc-3229225.html",
"#class" : pictoa.PictoaAlbumExtractor,
"#pattern": pictoa.PictoaImageExtractor.pattern,
"#count" : 16,
"album_id" : "3229225",
"album_title": "Anna Kendrick busty in a strapless red dress out in NYC",
"tags" : ["Anna Kendrick", "Celebrity"],
},
{
# verify pagination works
"#url" : "https://www.pictoa.com/albums/oscars-2020-red-carpet-4010403.html",
"#category": ("", "pictoa", "album"),
"#class" : pictoa.PictoaAlbumExtractor,
"#pattern" : r"https://?(?:[\w]+\.)?pictoa\.com/albums/[\w-]+/[\d]+.html",
"#count" : 182,
"album_id" : "4010403",
"date" : None,
"title" : "Oscars 2020 Red Carpet",
"tags" : ['Celebrity', 'Red']
"#url" : "https://www.pictoa.com.de/albums/oscars-2020-red-carpet-4010403.html",
"#comment": "verify pagination works",
"#class" : pictoa.PictoaAlbumExtractor,
"#pattern": pictoa.PictoaImageExtractor.pattern,
"#count" : 182,
"album_id" : "4010403",
"album_title": "Oscars 2020 Red Carpet",
"tags" : ['Celebrity', 'Red'],
},
{
# null tags
"#url" : "https://www.pictoa.com/albums/carl-virkus-149024.html",
"#category": ("", "pictoa", "album"),
"#class" : pictoa.PictoaAlbumExtractor,
"#count" : 1,
"album_id" : "149024",
"tags" : []
"#url" : "https://it.pictoa.com/albums/carl-virkus-149024.html",
"#comment": "null tags",
"#class" : pictoa.PictoaAlbumExtractor,
"#urls" : "https://www.pictoa.com/albums/carl-virkus-149024/2221031.html",
"album_id" : "149024",
"album_title": "Carl Virkus",
"tags" : [],
},
{
"#url" : "https://www.pictoa.com/albums/anna-kendrick-showing-cleavage-at-the-56th-annual-grammy-awards-3233172/75206264.html",
"#category": ("", "pictoa", "image"),
"#class" : pictoa.PictoaImageExtractor,
"#pattern" : r"https://s1.pictoa.com/media/galleries/168/930/[\w\d]+/[\w\d]+.jpg",
"id" : "75206264",
"#url" : "https://www.pictoa.com/albums/anna-kendrick-showing-cleavage-at-the-56th-annual-grammy-awards-3233172/75206264.html",
"#class": pictoa.PictoaImageExtractor,
"#urls" : "https://s1.pictoa.com/media/galleries/168/930/168930594a8750dfd3e/3233172594a8759dcc3a.jpg",
"album_id" : "3233172",
"album_title": "Anna Kendrick showing cleavage at the 56th Annual GRAMMY Awards",
"extension" : "jpg",
"filename" : "3233172594a8759dcc3a",
"id" : "75206264",
"url" : "https://s1.pictoa.com/media/galleries/168/930/168930594a8750dfd3e/3233172594a8759dcc3a.jpg"
},
{
"#url" : "https://nl.pictoa.com/albums/kandi-barbour-3840809/94038192.html",
"#class": pictoa.PictoaImageExtractor,
"#urls" : "https://s2.pictoa.com/media/galleries/294/452/29445260009fa5b68e4/384080960009fb51e389.jpg",
"#sha1_content": "152595069016da89565eb3d8e73df835afd22e2c",
"album_id" : "3840809",
"album_title": "Kandi Barbour",
"extension" : "jpg",
"filename" : "384080960009fb51e389",
"id" : "94038192",
"url" : "https://s2.pictoa.com/media/galleries/294/452/29445260009fa5b68e4/384080960009fb51e389.jpg",
},
)