[behance] add user extractor

This commit is contained in:
Mike Fährmann
2018-08-31 17:40:44 +02:00
parent a493fed376
commit 75862715ac
3 changed files with 42 additions and 9 deletions

View File

@@ -12,7 +12,7 @@ arch.b4k.co https://arch.b4k.co/ Threads
Archive of Sins https://archiveofsins.com/ Threads Archive of Sins https://archiveofsins.com/ Threads
Archived.Moe https://archived.moe/ Threads Archived.Moe https://archived.moe/ Threads
ArtStation https://www.artstation.com/ |Images from Use-0| ArtStation https://www.artstation.com/ |Images from Use-0|
Behance https://www.behance.net/ Galleries Behance https://www.behance.net/ Images from Users, Galleries
Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches
Desuarchive https://desuarchive.org/ Threads Desuarchive https://desuarchive.org/ Threads
DeviantArt https://www.deviantart.com/ |Collections, De-1| Optional (OAuth) DeviantArt https://www.deviantart.com/ |Collections, De-1| Optional (OAuth)
@@ -76,7 +76,7 @@ SlideShare https://www.slideshare.net/ Presentations
SmugMug https://www.smugmug.com/ |Albums, individ-5| Optional (OAuth) SmugMug https://www.smugmug.com/ |Albums, individ-5| Optional (OAuth)
The /b/ Archive https://thebarchive.com/ Threads The /b/ Archive https://thebarchive.com/ Threads
Tumblr https://www.tumblr.com/ Images from Users, Likes, Posts, Tag-Searches Optional (OAuth) Tumblr https://www.tumblr.com/ Images from Users, Likes, Posts, Tag-Searches Optional (OAuth)
Twitter https://twitter.com/ Timelines, Tweets, Media Tweets Twitter https://twitter.com/ Media Timelines, Timelines, Tweets
Warosu https://warosu.org/ Threads Warosu https://warosu.org/ Threads
World Three http://www.slide.world-three.org/ Chapters, Manga World Three http://www.slide.world-three.org/ Chapters, Manga
XVideos https://www.xvideos.com/ Images from Users, Galleries XVideos https://www.xvideos.com/ Images from Users, Galleries

View File

@@ -12,14 +12,18 @@ from .common import Extractor, Message
from .. import text from .. import text
class BehanceGalleryExtractor(Extractor): class BehanceExtractor(Extractor):
"""Extractor for image galleries from www.behance.net""" """Base class for behance extractors"""
category = "behance" category = "behance"
root = "https://www.behance.net"
class BehanceGalleryExtractor(BehanceExtractor):
"""Extractor for image galleries from www.behance.net"""
subcategory = "gallery" subcategory = "gallery"
directory_fmt = ["{category}", "{user}", "{gallery_id} {title}"] directory_fmt = ["{category}", "{user}", "{gallery_id} {title}"]
filename_fmt = "{category}_{gallery_id}_{num:>02}.{extension}" filename_fmt = "{category}_{gallery_id}_{num:>02}.{extension}"
archive_fmt = "{gallery_id}_{num}" archive_fmt = "{gallery_id}_{num}"
root = "https://www.behance.net"
pattern = [r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)"] pattern = [r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)"]
test = [ test = [
("https://www.behance.net/gallery/17386197", { ("https://www.behance.net/gallery/17386197", {
@@ -43,7 +47,7 @@ class BehanceGalleryExtractor(Extractor):
] ]
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self) BehanceExtractor.__init__(self)
self.gallery_id = match.group(1) self.gallery_id = match.group(1)
def items(self): def items(self):
@@ -115,3 +119,34 @@ class BehanceGalleryExtractor(Extractor):
user = text.extract(users, ' class="profile-list-name"', '</a>')[0] user = text.extract(users, ' class="profile-list-name"', '</a>')[0]
return (user.rpartition(">")[2],) return (user.rpartition(">")[2],)
class BehanceUserExtractor(BehanceExtractor):
"""Extractor for a user's galleries from www.behance.net"""
subcategory = "user"
pattern = [r"(?:https?://)?(?:www\.)?behance\.net"
r"/(?!gallery/)([^/?&#]+)/?$"]
test = [("https://www.behance.net/alexstrohl", {
"count": ">= 8",
"pattern": BehanceGalleryExtractor.pattern[0],
})]
def __init__(self, match):
BehanceExtractor.__init__(self)
self.user = match.group(1)
def items(self):
url = "{}/{}".format(self.root, self.user)
headers = {"X-Requested-With": "XMLHttpRequest"}
params = {"offset": None}
yield Message.Version, 1
while True:
data = self.request(url, headers=headers, params=params).json()
for gallery in data["section_content"]:
yield Message.Queue, gallery["url"], gallery
if "offset" not in data:
return
params["offset"] = data["offset"]

View File

@@ -68,7 +68,7 @@ SUBCATEGORY_MAP = {
"issue" : "Comic-Issues", "issue" : "Comic-Issues",
"manga" : "Manga", "manga" : "Manga",
"me" : "pixiv.me Links", "me" : "pixiv.me Links",
"media" : "Media Tweets", "media" : "Media Timelines",
"path" : "Images from Users and Folders", "path" : "Images from Users and Folders",
"pinit" : "pin.it Links", "pinit" : "pin.it Links",
"popular": "Popular Images", "popular": "Popular Images",
@@ -227,8 +227,6 @@ def category_key(extrlist):
def subcategory_key(cls): def subcategory_key(cls):
if cls.subcategory in ("user", "issue"): if cls.subcategory in ("user", "issue"):
return "A" return "A"
if cls.subcategory in ("media",):
return "z"
return cls.subcategory return cls.subcategory