From 82891b4d0cc297e8476d39b56d9a1979676a80b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 4 Jul 2025 20:13:19 +0200 Subject: [PATCH] [pixiv] move 'novel' extractors to a 'pixiv-novel' category (#7746) TODO: - restore full 'include' functionality - allow remapping category:subcategory pairs --- docs/configuration.rst | 147 +++++++++++++++++++++++------- docs/gallery-dl.conf | 15 +++- docs/supportedsites.md | 10 ++- gallery_dl/__init__.py | 1 + gallery_dl/config.py | 1 + gallery_dl/extractor/pixiv.py | 158 ++++++++++++++++---------------- scripts/supportedsites.py | 6 +- test/results/pixiv.py | 165 ---------------------------------- test/results/pixivnovel.py | 113 +++++++++++++++++++++++ 9 files changed, 333 insertions(+), 283 deletions(-) create mode 100644 test/results/pixivnovel.py diff --git a/docs/configuration.rst b/docs/configuration.rst index 1ab6d687..ebd40cfc 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -393,7 +393,7 @@ Default ``itaku``, ``newgrounds``, ``[philomena]``, - ``pixiv:novel``, + ``pixiv-novel``, ``plurk``, ``poipiku`` , ``pornpics``, @@ -4022,37 +4022,6 @@ Description `gppt `__. -extractor.pixiv.novel.covers ----------------------------- -Type - ``bool`` -Default - ``false`` -Description - Download cover images. - - -extractor.pixiv.novel.embeds ----------------------------- -Type - ``bool`` -Default - ``false`` -Description - Download embedded images. - - -extractor.pixiv.novel.full-series ---------------------------------- -Type - ``bool`` -Default - ``false`` -Description - When downloading a novel being part of a series, - download all novels of that series. - - extractor.pixiv.metadata ------------------------ Type @@ -4168,6 +4137,114 @@ Description Try to fetch ``limit_sanity_level`` works via web API. +extractor.pixiv-novel.comments +------------------------------ +Type + ``bool`` +Default + ``false`` +Description + Fetch ``comments`` metadata. + + Note: This requires 1 or more additional API requests per novel, + depending on the number of comments. + + +extractor.pixiv-novel.covers +---------------------------- +Type + ``bool`` +Default + ``false`` +Description + Download cover images. + + +extractor.pixiv-novel.embeds +---------------------------- +Type + ``bool`` +Default + ``false`` +Description + Download embedded images. + + +extractor.pixiv-novel.full-series +--------------------------------- +Type + ``bool`` +Default + ``false`` +Description + When downloading a novel being part of a series, + download all novels of that series. + + +extractor.pixiv-novel.max-posts +------------------------------- +Type + ``integer`` +Default + ``0`` +Description + When downloading multiple novels, + this sets the maximum number of novels to get. + + A value of ``0`` means no limit. + + +extractor.pixiv-novel.metadata +------------------------------ +Type + ``bool`` +Default + ``false`` +Description + Fetch extended ``user`` metadata. + + +extractor.pixiv-novel.metadata-bookmark +--------------------------------------- +Type + ``bool`` +Default + ``false`` +Description + For novels bookmarked by + `your own account `__, + fetch bookmark tags as ``tags_bookmark`` metadata. + + Note: This requires 1 additional API request per bookmarked post. + + +extractor.pixiv-novel.refresh-token +----------------------------------- +Type + ``string`` +Description + The ``refresh-token`` value you get + from running ``gallery-dl oauth:pixiv`` (see OAuth_) or + by using a third-party tool like + `gppt `__. + + This can be the same value as `extractor.pixiv.refresh-token`_ + + +extractor.pixiv-novel.tags +-------------------------- +Type + ``string`` +Default + ``"japanese"`` +Description + Controls the ``tags`` metadata field. + + * `"japanese"`: List of Japanese tags + * `"translated"`: List of translated tags + * `"original"`: Unmodified list with both Japanese and translated tags + + extractor.plurk.comments ------------------------ Type @@ -7695,7 +7772,8 @@ Description "schalenetwork": "koharu", "naver-chzzk" : "chzzk", "naver-blog" : "naver", - "naver-webtoon": "naverwebtoon" + "naver-webtoon": "naverwebtoon", + "pixiv-novel" : "pixiv" } @@ -7712,7 +7790,8 @@ Default "koharu" : "schalenetwork", "chzzk" : "naver-chzzk", "naver" : "naver-blog", - "naverwebtoon": "naver-webtoon" + "naverwebtoon": "naver-webtoon", + "pixiv" : "pixiv-novel" } Description Duplicate the configuration settings of extractor `categories` diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index b96eae27..095f56e5 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -95,7 +95,8 @@ "koharu" : "schalenetwork", "chzzk" : "naver-chzzk", "naver" : "naver-blog", - "naverwebtoon": "naver-webtoon" + "naverwebtoon": "naver-webtoon", + "pixiv" : "pixiv-novel" }, @@ -533,7 +534,17 @@ "metadata-bookmark": false, "sanity" : true, "tags" : "japanese", - "ugoira" : true, + "ugoira" : true + }, + "pixiv-novel": + { + "refresh-token": null, + + "comments" : false, + "max-posts": null, + "metadata" : false, + "metadata-bookmark": false, + "tags" : "japanese", "covers" : false, "embeds" : false, diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 2c666543..8c21c670 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -740,11 +740,17 @@ Consider all listed sites to potentially be NSFW. - Pixiv + [pixiv] https://www.pixiv.net/ - Artworks, Avatars, Backgrounds, Favorites, Follows, pixiv.me Links, Novels, Novel Bookmarks, Novel Series, pixivision, Rankings, Search Results, Series, Sketch, Unlisted Works, User Profiles, individual Images + Artworks, Avatars, Backgrounds, Favorites, Follows, pixiv.me Links, pixivision, Rankings, Search Results, Series, Sketch, Unlisted Works, User Profiles, individual Images OAuth + + [pixiv] Novels + https://www.pixiv.net/novel + Bookmarks, Novels, Series, User Profiles + + pixivFANBOX https://www.fanbox.cc/ diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index be0f6dee..9a6449c0 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -326,6 +326,7 @@ def main(): "naver-blog" : "naver", "naver-chzzk" : "chzzk", "naver-webtoon": "naverwebtoon", + "pixiv-novel" : "pixiv", } from .extractor import common common.CATEGORY_MAP = catmap diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 1eeda8c4..4f686821 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -175,6 +175,7 @@ def remap_categories(): ("naver" , "naver-blog"), ("chzzk" , "naver-chzzk"), ("naverwebtoon", "naver-webtoon"), + ("pixiv" , "pixiv-novel"), ) elif not cmap: return diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index af34bdd5..977ba005 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -861,18 +861,71 @@ class PixivSeriesExtractor(PixivExtractor): yield work -class PixivNovelExtractor(PixivExtractor): - """Extractor for pixiv novels""" - subcategory = "novel" - request_interval = (0.5, 1.5) - pattern = BASE_PATTERN + r"/n(?:ovel/show\.php\?id=|/)(\d+)" - example = "https://www.pixiv.net/novel/show.php?id=12345" - - def __init__(self, match): - PixivExtractor.__init__(self, match) - self.novel_id = match[1] +class PixivSketchExtractor(Extractor): + """Extractor for user pages on sketch.pixiv.net""" + category = "pixiv" + subcategory = "sketch" + directory_fmt = ("{category}", "sketch", "{user[unique_name]}") + filename_fmt = "{post_id} {id}.{extension}" + archive_fmt = "S{user[id]}_{id}" + root = "https://sketch.pixiv.net" + cookies_domain = ".pixiv.net" + pattern = r"(?:https?://)?sketch\.pixiv\.net/@([^/?#]+)" + example = "https://sketch.pixiv.net/@USER" def items(self): + self.username = self.groups[0] + headers = {"Referer": f"{self.root}/@{self.username}"} + + for post in self.posts(): + media = post["media"] + post["post_id"] = post["id"] + post["date"] = text.parse_datetime( + post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + util.delete_items(post, ("id", "media", "_links")) + + yield Message.Directory, post + post["_http_headers"] = headers + + for photo in media: + original = photo["photo"]["original"] + post["id"] = photo["id"] + post["width"] = original["width"] + post["height"] = original["height"] + + url = original["url"] + text.nameext_from_url(url, post) + yield Message.Url, url, post + + def posts(self): + url = f"{self.root}/api/walls/@{self.username}/posts/public.json" + headers = { + "Accept": "application/vnd.sketch-v4+json", + "Referer": self.root + "/", + "X-Requested-With": f"{self.root}/@{self.username}", + } + + while True: + data = self.request_json(url, headers=headers) + yield from data["data"]["items"] + + next_url = data["_links"].get("next") + if not next_url: + return + url = self.root + next_url["href"] + + +############################################################################### +# Novels ###################################################################### + +class PixivNovelExtractor(PixivExtractor): + """Base class for pixiv novel extractors""" + category = "pixiv-novel" + request_interval = (0.5, 1.5) + + def items(self): + self.novel_id = self.groups[0] + tags = self.config("tags", "japanese") if tags == "original": transform_tags = None @@ -974,6 +1027,13 @@ class PixivNovelExtractor(PixivExtractor): url = f"{self.root}/artworks/{illust_id}" yield Message.Queue, url, novel + +class PixivNovelNovelExtractor(PixivNovelExtractor): + """Extractor for pixiv novels""" + subcategory = "novel" + pattern = BASE_PATTERN + r"/n(?:ovel/show\.php\?id=|/)(\d+)" + example = "https://www.pixiv.net/novel/show.php?id=12345" + def novels(self): novel = self.api.novel_detail(self.novel_id) if self.config("full-series") and novel["series"]: @@ -984,7 +1044,7 @@ class PixivNovelExtractor(PixivExtractor): class PixivNovelUserExtractor(PixivNovelExtractor): """Extractor for pixiv users' novels""" - subcategory = "novel-user" + subcategory = "user" pattern = USER_PATTERN + r"/novels" example = "https://www.pixiv.net/en/users/12345/novels" @@ -994,7 +1054,7 @@ class PixivNovelUserExtractor(PixivNovelExtractor): class PixivNovelSeriesExtractor(PixivNovelExtractor): """Extractor for pixiv novel series""" - subcategory = "novel-series" + subcategory = "series" pattern = BASE_PATTERN + r"/novel/series/(\d+)" example = "https://www.pixiv.net/novel/series/12345" @@ -1004,85 +1064,25 @@ class PixivNovelSeriesExtractor(PixivNovelExtractor): class PixivNovelBookmarkExtractor(PixivNovelExtractor): """Extractor for bookmarked pixiv novels""" - subcategory = "novel-bookmark" + subcategory = "bookmark" pattern = (USER_PATTERN + r"/bookmarks/novels" r"(?:/([^/?#]+))?(?:/?\?([^#]+))?") example = "https://www.pixiv.net/en/users/12345/bookmarks/novels" - def __init__(self, match): - PixivNovelExtractor.__init__(self, match) - self.user_id, self.tag, self.query = match.groups() - def novels(self): - if self.tag: - tag = text.unquote(self.tag) - else: - tag = None + user_id, tag, query = self.groups + tag = text.unquote(tag) if tag else None - if text.parse_query(self.query).get("rest") == "hide": + if text.parse_query(query).get("rest") == "hide": restrict = "private" else: restrict = "public" - return self.api.user_bookmarks_novel(self.user_id, tag, restrict) + return self.api.user_bookmarks_novel(user_id, tag, restrict) -class PixivSketchExtractor(Extractor): - """Extractor for user pages on sketch.pixiv.net""" - category = "pixiv" - subcategory = "sketch" - directory_fmt = ("{category}", "sketch", "{user[unique_name]}") - filename_fmt = "{post_id} {id}.{extension}" - archive_fmt = "S{user[id]}_{id}" - root = "https://sketch.pixiv.net" - cookies_domain = ".pixiv.net" - pattern = r"(?:https?://)?sketch\.pixiv\.net/@([^/?#]+)" - example = "https://sketch.pixiv.net/@USER" - - def __init__(self, match): - Extractor.__init__(self, match) - self.username = match[1] - - def items(self): - headers = {"Referer": f"{self.root}/@{self.username}"} - - for post in self.posts(): - media = post["media"] - post["post_id"] = post["id"] - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") - util.delete_items(post, ("id", "media", "_links")) - - yield Message.Directory, post - post["_http_headers"] = headers - - for photo in media: - original = photo["photo"]["original"] - post["id"] = photo["id"] - post["width"] = original["width"] - post["height"] = original["height"] - - url = original["url"] - text.nameext_from_url(url, post) - yield Message.Url, url, post - - def posts(self): - url = f"{self.root}/api/walls/@{self.username}/posts/public.json" - headers = { - "Accept": "application/vnd.sketch-v4+json", - "X-Requested-With": f"{self.root}/@{self.username}", - "Referer": self.root + "/", - } - - while True: - data = self.request_json(url, headers=headers) - yield from data["data"]["items"] - - next_url = data["_links"].get("next") - if not next_url: - return - url = self.root + next_url["href"] - +############################################################################### +# API ######################################################################### class PixivAppAPI(): """Minimal interface for the Pixiv App API for mobile devices diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index b4f00d3d..172d10f0 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -122,6 +122,8 @@ CATEGORY_MAP = { "photovogue" : "PhotoVogue", "pidgiwiki" : "PidgiWiki", "pixeldrain" : "pixeldrain", + "pixiv" : "[pixiv]", + "pixiv-novel" : "[pixiv] Novels", "pornimage" : "Porn Image", "pornpics" : "PornPics.com", "pornreactor" : "PornReactor", @@ -566,7 +568,7 @@ def subcategory_text(bc, c, sc): def category_key(c): """Generate sorting keys by category""" - return category_text(c[0]).lower() + return category_text(c[0]).lower().lstrip("[") def subcategory_key(sc): @@ -610,6 +612,8 @@ def build_extractor_list(): for subcategories in base.values(): subcategories.sort(key=subcategory_key) + domains["pixiv-novel"] += "novel" + # add e-hentai.org default["e-hentai"] = default["exhentai"] domains["e-hentai"] = domains["exhentai"].replace("x", "-") diff --git a/test/results/pixiv.py b/test/results/pixiv.py index b3aa2917..3b27c49a 100644 --- a/test/results/pixiv.py +++ b/test/results/pixiv.py @@ -11,37 +11,31 @@ from gallery_dl import exception __tests__ = ( { "#url" : "https://www.pixiv.net/en/users/173530", - "#category": ("", "pixiv", "user"), "#class" : pixiv.PixivUserExtractor, }, { "#url" : "https://www.pixiv.net/u/173530", - "#category": ("", "pixiv", "user"), "#class" : pixiv.PixivUserExtractor, }, { "#url" : "https://www.pixiv.net/member.php?id=173530", - "#category": ("", "pixiv", "user"), "#class" : pixiv.PixivUserExtractor, }, { "#url" : "https://www.pixiv.net/mypage.php#id=173530", - "#category": ("", "pixiv", "user"), "#class" : pixiv.PixivUserExtractor, }, { "#url" : "https://www.pixiv.net/#id=173530", - "#category": ("", "pixiv", "user"), "#class" : pixiv.PixivUserExtractor, }, { "#url" : "https://www.pixiv.net/en/users/173530/artworks", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7", }, @@ -49,14 +43,12 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D", "#comment" : "illusts with specific tag", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", }, { "#url" : "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", }, @@ -64,7 +56,6 @@ __tests__ = ( { "#url" : "http://www.pixiv.net/member_illust.php?id=173531", "#comment" : "deleted account", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, "#options" : {"metadata": True}, "#exception": exception.NotFoundError, @@ -79,64 +70,54 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/en/users/173530/manga", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, }, { "#url" : "https://www.pixiv.net/en/users/173530/illustrations", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, }, { "#url" : "https://www.pixiv.net/member_illust.php?id=173530", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, }, { "#url" : "https://touch.pixiv.net/member_illust.php?id=173530", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, }, { "#url" : "https://www.phixiv.net/member_illust.php?id=173530", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, }, { "#url" : "https://phixiv.net/en/users/56514424/artworks", - "#category": ("", "pixiv", "artworks"), "#class" : pixiv.PixivArtworksExtractor, }, { "#url" : "https://www.pixiv.net/en/users/173530/avatar", - "#category": ("", "pixiv", "avatar"), "#class" : pixiv.PixivAvatarExtractor, "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66", }, { "#url" : "https://www.pixiv.net/en/users/194921/background", - "#category": ("", "pixiv", "background"), "#class" : pixiv.PixivBackgroundExtractor, "#pattern" : r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", }, { "#url" : "https://pixiv.me/del_shannon", - "#category": ("", "pixiv", "me"), "#class" : pixiv.PixivMeExtractor, "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf", }, { "#url" : "https://pixiv.me/del_shanno", - "#category": ("", "pixiv", "me"), "#class" : pixiv.PixivMeExtractor, "#exception": exception.NotFoundError, }, @@ -144,7 +125,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/artworks/966412", "#comment" : "related works (#1237)", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, "#sha1_url" : "90c1715b07b0d1aad300bce256a0bc71f42540ba", "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", @@ -155,7 +135,6 @@ __tests__ = ( { "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, "#exception": exception.NotFoundError, }, @@ -163,7 +142,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629", "#comment" : "ugoira", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, "#results" : "https://i.pximg.net/img-zip-ugoira/img/2018/01/15/13/24/48/66806629_ugoira1920x1080.zip", @@ -175,7 +153,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/artworks/101003492", "#comment" : "original ugoira frames (#6056)", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, "#options" : {"ugoira": "original"}, "#results" : ( @@ -193,7 +170,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/artworks/966412", "#comment" : "related works (#1237)", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, "#options" : {"related": True}, "#range" : "1-10", @@ -321,55 +297,46 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/en/artworks/966412", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "https://www.pixiv.net/i/966412", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "http://img.pixiv.net/img/soundcross/42626136.jpg", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "https://www.phixiv.net/en/artworks/966412", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, { "#url" : "https://phixiv.net/member_illust.php?mode=medium&illust_id=966412", - "#category": ("", "pixiv", "work"), "#class" : pixiv.PixivWorkExtractor, }, @@ -384,7 +351,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/en/users/173530/bookmarks/artworks", - "#category": ("", "pixiv", "favorite"), "#class" : pixiv.PixivFavoriteExtractor, "#results" : ( "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", @@ -397,7 +363,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/bookmark.php?id=173530", - "#category": ("", "pixiv", "favorite"), "#class" : pixiv.PixivFavoriteExtractor, "#results" : ( "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", @@ -411,7 +376,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93", "#comment" : "bookmarks with specific tag", - "#category": ("", "pixiv", "favorite"), "#class" : pixiv.PixivFavoriteExtractor, "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", }, @@ -419,7 +383,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1", "#comment" : "bookmarks with specific tag (legacy url)", - "#category": ("", "pixiv", "favorite"), "#class" : pixiv.PixivFavoriteExtractor, "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", }, @@ -467,7 +430,6 @@ __tests__ = ( { "#url" : "https://touch.pixiv.net/bookmark.php?id=173530", "#comment" : "touch URLs", - "#category": ("", "pixiv", "favorite"), "#class" : pixiv.PixivFavoriteExtractor, }, @@ -479,13 +441,11 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/ranking.php?mode=daily&date=20170818", - "#category": ("", "pixiv", "ranking"), "#class" : pixiv.PixivRankingExtractor, }, { "#url" : "https://www.pixiv.net/ranking.php", - "#category": ("", "pixiv", "ranking"), "#class" : pixiv.PixivRankingExtractor, "#options" : {"max-posts": 10}, @@ -498,20 +458,17 @@ __tests__ = ( { "#url" : "https://touch.pixiv.net/ranking.php", - "#category": ("", "pixiv", "ranking"), "#class" : pixiv.PixivRankingExtractor, }, { "#url" : "https://www.pixiv.net/ranking.php?mode=unknown", - "#category": ("", "pixiv", "ranking"), "#class" : pixiv.PixivRankingExtractor, "#exception": exception.StopExtraction, }, { "#url" : "https://www.pixiv.net/en/tags/Original", - "#category": ("", "pixiv", "search"), "#class" : pixiv.PixivSearchExtractor, "#range" : "1-10", "#count" : 10, @@ -519,64 +476,54 @@ __tests__ = ( { "#url" : "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", - "#category": ("", "pixiv", "search"), "#class" : pixiv.PixivSearchExtractor, "#exception": exception.StopExtraction, }, { "#url" : "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", - "#category": ("", "pixiv", "search"), "#class" : pixiv.PixivSearchExtractor, "#exception": exception.StopExtraction, }, { "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", - "#category": ("", "pixiv", "search"), "#class" : pixiv.PixivSearchExtractor, "#exception": exception.StopExtraction, }, { "#url" : "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag", - "#category": ("", "pixiv", "search"), "#class" : pixiv.PixivSearchExtractor, }, { "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", - "#category": ("", "pixiv", "search"), "#class" : pixiv.PixivSearchExtractor, }, { "#url" : "https://touch.pixiv.net/search.php?word=Original", - "#category": ("", "pixiv", "search"), "#class" : pixiv.PixivSearchExtractor, }, { "#url" : "https://www.pixiv.net/bookmark_new_illust.php", - "#category": ("", "pixiv", "follow"), "#class" : pixiv.PixivFollowExtractor, }, { "#url" : "https://touch.pixiv.net/bookmark_new_illust.php", - "#category": ("", "pixiv", "follow"), "#class" : pixiv.PixivFollowExtractor, }, { "#url" : "https://www.pixivision.net/en/a/2791", - "#category": ("", "pixiv", "pixivision"), "#class" : pixiv.PixivPixivisionExtractor, }, { "#url" : "https://pixivision.net/a/2791", - "#category": ("", "pixiv", "pixivision"), "#class" : pixiv.PixivPixivisionExtractor, "#count" : 7, @@ -586,7 +533,6 @@ __tests__ = ( { "#url" : "https://www.pixiv.net/user/10509347/series/21859", - "#category": ("", "pixiv", "series"), "#class" : pixiv.PixivSeriesExtractor, "#range" : "1-10", "#count" : 10, @@ -604,119 +550,8 @@ __tests__ = ( }, }, -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#count" : 1, - "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93", - - "caption" : "
第一印象から決めてました！

素敵な表紙はいもこは妹さん(illust/53802907)からお借りしました。

たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！

■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　

■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！", - "create_date" : "2019-12-19T23:14:36+09:00", - "date" : "dt:2019-12-19 14:14:36", - "extension" : "txt", - "id" : 12101012, - "image_urls" : dict, - "is_bookmarked" : False, - "is_muted" : False, - "is_mypixiv_only": False, - "is_original" : False, - "is_x_restricted": False, - "novel_ai_type" : 0, - "page_count" : 1, - "rating" : "General", - "restrict" : 0, - "series" : { - "id" : 1479656, - "title": "一目惚れした彼らの話", - }, - "tags" : [ - "鬼滅の夢", - "女主人公", - "煉獄杏寿郎", - "涙腺崩壊", - "なにこれすごい", - "来世で幸せになって欲しい", - "キメ学世界線できっと幸せになってる!!", - "あなたが神か!!", - "キメ学編を·····", - "鬼滅の夢小説10000users入り", - ], - "text_length" : 9569, - "title" : "本当は、一目惚れだった", - "total_bookmarks": range(17900, 20000), - "total_comments" : range(200, 400), - "total_view" : range(158000, 300000), - "user" : { - "account": "46_maru", - "id" : 888268, - }, - "visible" : True, - "x_restrict" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=16422450", - "#comment" : "embeds // covers (#5373)", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#options" : { - "embeds": True, - "covers": True, - }, - "#count" : 4, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", - "#comment" : "full series", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#options" : {"full-series": True}, - "#count" : 2, -}, - -{ - "#url" : "https://www.pixiv.net/n/19612040", - "#comment" : "short URL", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/77055466/novels", - "#category": ("", "pixiv", "novel-user"), - "#class" : pixiv.PixivNovelUserExtractor, - "#pattern" : "^text:", - "#range" : "1-5", - "#count" : 5, -}, - -{ - "#url" : "https://www.pixiv.net/novel/series/1479656", - "#category": ("", "pixiv", "novel-series"), - "#class" : pixiv.PixivNovelSeriesExtractor, - "#count" : 2, - "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/77055466/bookmarks/novels", - "#category": ("", "pixiv", "novel-bookmark"), - "#class" : pixiv.PixivNovelBookmarkExtractor, - "#count" : 1, - "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", - "#category": ("", "pixiv", "novel-bookmark"), - "#class" : pixiv.PixivNovelBookmarkExtractor, -}, - { "#url" : "https://sketch.pixiv.net/@nicoby", - "#category": ("", "pixiv", "sketch"), "#class" : pixiv.PixivSketchExtractor, "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", "#count" : ">= 35", diff --git a/test/results/pixivnovel.py b/test/results/pixivnovel.py new file mode 100644 index 00000000..286d53f4 --- /dev/null +++ b/test/results/pixivnovel.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pixiv + + +__tests__ = ( +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", + "#class" : pixiv.PixivNovelNovelExtractor, + "#count" : 1, + "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93", + + "caption" : "
第一印象から決めてました！

素敵な表紙はいもこは妹さん(illust/53802907)からお借りしました。

たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！

■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　

■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！", + "create_date" : "2019-12-19T23:14:36+09:00", + "date" : "dt:2019-12-19 14:14:36", + "extension" : "txt", + "id" : 12101012, + "image_urls" : dict, + "is_bookmarked" : False, + "is_muted" : False, + "is_mypixiv_only": False, + "is_original" : False, + "is_x_restricted": False, + "novel_ai_type" : 0, + "page_count" : 1, + "rating" : "General", + "restrict" : 0, + "series" : { + "id" : 1479656, + "title": "一目惚れした彼らの話", + }, + "tags" : [ + "鬼滅の夢", + "女主人公", + "煉獄杏寿郎", + "涙腺崩壊", + "なにこれすごい", + "来世で幸せになって欲しい", + "キメ学世界線できっと幸せになってる!!", + "あなたが神か!!", + "キメ学編を·····", + "鬼滅の夢小説10000users入り", + ], + "text_length" : 9569, + "title" : "本当は、一目惚れだった", + "total_bookmarks": range(17900, 20000), + "total_comments" : range(200, 400), + "total_view" : range(158000, 300000), + "user" : { + "account": "46_maru", + "id" : 888268, + }, + "visible" : True, + "x_restrict" : 0, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=16422450", + "#comment" : "embeds // covers (#5373)", + "#class" : pixiv.PixivNovelNovelExtractor, + "#options" : { + "embeds": True, + "covers": True, + }, + "#count" : 4, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", + "#comment" : "full series", + "#class" : pixiv.PixivNovelNovelExtractor, + "#options" : {"full-series": True}, + "#count" : 2, +}, + +{ + "#url" : "https://www.pixiv.net/n/19612040", + "#comment" : "short URL", + "#class" : pixiv.PixivNovelNovelExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/77055466/novels", + "#class" : pixiv.PixivNovelUserExtractor, + "#pattern" : "^text:", + "#range" : "1-5", + "#count" : 5, +}, + +{ + "#url" : "https://www.pixiv.net/novel/series/1479656", + "#class" : pixiv.PixivNovelSeriesExtractor, + "#count" : 2, + "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/77055466/bookmarks/novels", + "#class" : pixiv.PixivNovelBookmarkExtractor, + "#count" : 1, + "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", + "#class" : pixiv.PixivNovelBookmarkExtractor, +}, + +)