[behance] add user extractor
This commit is contained in:
@@ -12,7 +12,7 @@ arch.b4k.co https://arch.b4k.co/ Threads
|
|||||||
Archive of Sins https://archiveofsins.com/ Threads
|
Archive of Sins https://archiveofsins.com/ Threads
|
||||||
Archived.Moe https://archived.moe/ Threads
|
Archived.Moe https://archived.moe/ Threads
|
||||||
ArtStation https://www.artstation.com/ |Images from Use-0|
|
ArtStation https://www.artstation.com/ |Images from Use-0|
|
||||||
Behance https://www.behance.net/ Galleries
|
Behance https://www.behance.net/ Images from Users, Galleries
|
||||||
Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches
|
Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches
|
||||||
Desuarchive https://desuarchive.org/ Threads
|
Desuarchive https://desuarchive.org/ Threads
|
||||||
DeviantArt https://www.deviantart.com/ |Collections, De-1| Optional (OAuth)
|
DeviantArt https://www.deviantart.com/ |Collections, De-1| Optional (OAuth)
|
||||||
@@ -76,7 +76,7 @@ SlideShare https://www.slideshare.net/ Presentations
|
|||||||
SmugMug https://www.smugmug.com/ |Albums, individ-5| Optional (OAuth)
|
SmugMug https://www.smugmug.com/ |Albums, individ-5| Optional (OAuth)
|
||||||
The /b/ Archive https://thebarchive.com/ Threads
|
The /b/ Archive https://thebarchive.com/ Threads
|
||||||
Tumblr https://www.tumblr.com/ Images from Users, Likes, Posts, Tag-Searches Optional (OAuth)
|
Tumblr https://www.tumblr.com/ Images from Users, Likes, Posts, Tag-Searches Optional (OAuth)
|
||||||
Twitter https://twitter.com/ Timelines, Tweets, Media Tweets
|
Twitter https://twitter.com/ Media Timelines, Timelines, Tweets
|
||||||
Warosu https://warosu.org/ Threads
|
Warosu https://warosu.org/ Threads
|
||||||
World Three http://www.slide.world-three.org/ Chapters, Manga
|
World Three http://www.slide.world-three.org/ Chapters, Manga
|
||||||
XVideos https://www.xvideos.com/ Images from Users, Galleries
|
XVideos https://www.xvideos.com/ Images from Users, Galleries
|
||||||
|
|||||||
@@ -12,14 +12,18 @@ from .common import Extractor, Message
|
|||||||
from .. import text
|
from .. import text
|
||||||
|
|
||||||
|
|
||||||
class BehanceGalleryExtractor(Extractor):
|
class BehanceExtractor(Extractor):
|
||||||
"""Extractor for image galleries from www.behance.net"""
|
"""Base class for behance extractors"""
|
||||||
category = "behance"
|
category = "behance"
|
||||||
|
root = "https://www.behance.net"
|
||||||
|
|
||||||
|
|
||||||
|
class BehanceGalleryExtractor(BehanceExtractor):
|
||||||
|
"""Extractor for image galleries from www.behance.net"""
|
||||||
subcategory = "gallery"
|
subcategory = "gallery"
|
||||||
directory_fmt = ["{category}", "{user}", "{gallery_id} {title}"]
|
directory_fmt = ["{category}", "{user}", "{gallery_id} {title}"]
|
||||||
filename_fmt = "{category}_{gallery_id}_{num:>02}.{extension}"
|
filename_fmt = "{category}_{gallery_id}_{num:>02}.{extension}"
|
||||||
archive_fmt = "{gallery_id}_{num}"
|
archive_fmt = "{gallery_id}_{num}"
|
||||||
root = "https://www.behance.net"
|
|
||||||
pattern = [r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)"]
|
pattern = [r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)"]
|
||||||
test = [
|
test = [
|
||||||
("https://www.behance.net/gallery/17386197", {
|
("https://www.behance.net/gallery/17386197", {
|
||||||
@@ -43,7 +47,7 @@ class BehanceGalleryExtractor(Extractor):
|
|||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self, match):
|
def __init__(self, match):
|
||||||
Extractor.__init__(self)
|
BehanceExtractor.__init__(self)
|
||||||
self.gallery_id = match.group(1)
|
self.gallery_id = match.group(1)
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
@@ -115,3 +119,34 @@ class BehanceGalleryExtractor(Extractor):
|
|||||||
|
|
||||||
user = text.extract(users, ' class="profile-list-name"', '</a>')[0]
|
user = text.extract(users, ' class="profile-list-name"', '</a>')[0]
|
||||||
return (user.rpartition(">")[2],)
|
return (user.rpartition(">")[2],)
|
||||||
|
|
||||||
|
|
||||||
|
class BehanceUserExtractor(BehanceExtractor):
|
||||||
|
"""Extractor for a user's galleries from www.behance.net"""
|
||||||
|
subcategory = "user"
|
||||||
|
pattern = [r"(?:https?://)?(?:www\.)?behance\.net"
|
||||||
|
r"/(?!gallery/)([^/?&#]+)/?$"]
|
||||||
|
test = [("https://www.behance.net/alexstrohl", {
|
||||||
|
"count": ">= 8",
|
||||||
|
"pattern": BehanceGalleryExtractor.pattern[0],
|
||||||
|
})]
|
||||||
|
|
||||||
|
def __init__(self, match):
|
||||||
|
BehanceExtractor.__init__(self)
|
||||||
|
self.user = match.group(1)
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
url = "{}/{}".format(self.root, self.user)
|
||||||
|
headers = {"X-Requested-With": "XMLHttpRequest"}
|
||||||
|
params = {"offset": None}
|
||||||
|
|
||||||
|
yield Message.Version, 1
|
||||||
|
while True:
|
||||||
|
data = self.request(url, headers=headers, params=params).json()
|
||||||
|
|
||||||
|
for gallery in data["section_content"]:
|
||||||
|
yield Message.Queue, gallery["url"], gallery
|
||||||
|
|
||||||
|
if "offset" not in data:
|
||||||
|
return
|
||||||
|
params["offset"] = data["offset"]
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ SUBCATEGORY_MAP = {
|
|||||||
"issue" : "Comic-Issues",
|
"issue" : "Comic-Issues",
|
||||||
"manga" : "Manga",
|
"manga" : "Manga",
|
||||||
"me" : "pixiv.me Links",
|
"me" : "pixiv.me Links",
|
||||||
"media" : "Media Tweets",
|
"media" : "Media Timelines",
|
||||||
"path" : "Images from Users and Folders",
|
"path" : "Images from Users and Folders",
|
||||||
"pinit" : "pin.it Links",
|
"pinit" : "pin.it Links",
|
||||||
"popular": "Popular Images",
|
"popular": "Popular Images",
|
||||||
@@ -227,8 +227,6 @@ def category_key(extrlist):
|
|||||||
def subcategory_key(cls):
|
def subcategory_key(cls):
|
||||||
if cls.subcategory in ("user", "issue"):
|
if cls.subcategory in ("user", "issue"):
|
||||||
return "A"
|
return "A"
|
||||||
if cls.subcategory in ("media",):
|
|
||||||
return "z"
|
|
||||||
return cls.subcategory
|
return cls.subcategory
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user