[pixiv] move 'novel' extractors to a 'pixiv-novel' category (#7746)

TODO:
- restore full 'include' functionality
- allow remapping category:subcategory pairs

docs/configuration.rst

@@ -393,7 +393,7 @@ Default
         ``itaku``,
         ``newgrounds``,
         ``[philomena]``,
-        ``pixiv:novel``,
+        ``pixiv-novel``,
         ``plurk``,
         ``poipiku`` ,
         ``pornpics``,
@@ -4022,37 +4022,6 @@ Description
     `gppt <https://github.com/eggplants/get-pixivpy-token>`__.
 
 
-extractor.pixiv.novel.covers
-----------------------------
-Type
-    ``bool``
-Default
-    ``false``
-Description
-    Download cover images.
-
-
-extractor.pixiv.novel.embeds
-----------------------------
-Type
-    ``bool``
-Default
-    ``false``
-Description
-    Download embedded images.
-
-
-extractor.pixiv.novel.full-series
----------------------------------
-Type
-    ``bool``
-Default
-    ``false``
-Description
-    When downloading a novel being part of a series,
-    download all novels of that series.
-
-
 extractor.pixiv.metadata
 ------------------------
 Type
@@ -4168,6 +4137,114 @@ Description
     Try to fetch ``limit_sanity_level`` works via web API.
 
 
+extractor.pixiv-novel.comments
+------------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Fetch ``comments`` metadata.
+
+    Note: This requires 1 or more additional API requests per novel,
+    depending on the number of comments.
+
+
+extractor.pixiv-novel.covers
+----------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Download cover images.
+
+
+extractor.pixiv-novel.embeds
+----------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Download embedded images.
+
+
+extractor.pixiv-novel.full-series
+---------------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    When downloading a novel being part of a series,
+    download all novels of that series.
+
+
+extractor.pixiv-novel.max-posts
+-------------------------------
+Type
+    ``integer``
+Default
+    ``0``
+Description
+    When downloading multiple novels,
+    this sets the maximum number of novels to get.
+
+    A value of ``0`` means no limit.
+
+
+extractor.pixiv-novel.metadata
+------------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Fetch extended ``user`` metadata.
+
+
+extractor.pixiv-novel.metadata-bookmark
+---------------------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    For novels bookmarked by
+    `your own account <extractor.pixiv-novel.refresh-token_>`__,
+    fetch bookmark tags as ``tags_bookmark`` metadata.
+
+    Note: This requires 1 additional API request per bookmarked post.
+
+
+extractor.pixiv-novel.refresh-token
+-----------------------------------
+Type
+    ``string``
+Description
+    The ``refresh-token`` value you get
+    from running ``gallery-dl oauth:pixiv`` (see OAuth_) or
+    by using a third-party tool like
+    `gppt <https://github.com/eggplants/get-pixivpy-token>`__.
+
+    This can be the same value as `extractor.pixiv.refresh-token`_
+
+
+extractor.pixiv-novel.tags
+--------------------------
+Type
+    ``string``
+Default
+    ``"japanese"``
+Description
+    Controls the ``tags`` metadata field.
+
+    * `"japanese"`: List of Japanese tags
+    * `"translated"`: List of translated tags
+    * `"original"`: Unmodified list with both Japanese and translated tags
+
+
 extractor.plurk.comments
 ------------------------
 Type
@@ -7695,7 +7772,8 @@ Description
                 "schalenetwork": "koharu",
                 "naver-chzzk"  : "chzzk",
                 "naver-blog"   : "naver",
-                "naver-webtoon": "naverwebtoon"
+                "naver-webtoon": "naverwebtoon",
+                "pixiv-novel"  : "pixiv"
             }
 
 
@@ -7712,7 +7790,8 @@ Default
             "koharu"      : "schalenetwork",
             "chzzk"       : "naver-chzzk",
             "naver"       : "naver-blog",
-            "naverwebtoon": "naver-webtoon"
+            "naverwebtoon": "naver-webtoon",
+            "pixiv"       : "pixiv-novel"
         }
 Description
     Duplicate the configuration settings of extractor `categories`

docs/gallery-dl.conf

@@ -95,7 +95,8 @@
             "koharu"      : "schalenetwork",
             "chzzk"       : "naver-chzzk",
             "naver"       : "naver-blog",
-            "naverwebtoon": "naver-webtoon"
+            "naverwebtoon": "naver-webtoon",
+            "pixiv"       : "pixiv-novel"
         },
 
 
@@ -533,7 +534,17 @@
             "metadata-bookmark": false,
             "sanity"   : true,
             "tags"     : "japanese",
-            "ugoira"   : true,
+            "ugoira"   : true
+        },
+        "pixiv-novel":
+        {
+            "refresh-token": null,
+
+            "comments" : false,
+            "max-posts": null,
+            "metadata" : false,
+            "metadata-bookmark": false,
+            "tags"     : "japanese",
 
             "covers"     : false,
             "embeds"     : false,

docs/supportedsites.md

@@ -740,11 +740,17 @@ Consider all listed sites to potentially be NSFW.
     <td></td>
 </tr>
 <tr>
-    <td>Pixiv</td>
+    <td>[pixiv]</td>
     <td>https://www.pixiv.net/</td>
-    <td>Artworks, Avatars, Backgrounds, Favorites, Follows, pixiv.me Links, Novels, Novel Bookmarks, Novel Series, pixivision, Rankings, Search Results, Series, Sketch, Unlisted Works, User Profiles, individual Images</td>
+    <td>Artworks, Avatars, Backgrounds, Favorites, Follows, pixiv.me Links, pixivision, Rankings, Search Results, Series, Sketch, Unlisted Works, User Profiles, individual Images</td>
     <td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
 </tr>
+<tr>
+    <td>[pixiv] Novels</td>
+    <td>https://www.pixiv.net/novel</td>
+    <td>Bookmarks, Novels, Series, User Profiles</td>
+    <td></td>
+</tr>
 <tr>
     <td>pixivFANBOX</td>
     <td>https://www.fanbox.cc/</td>

gallery_dl/__init__.py

@@ -326,6 +326,7 @@ def main():
                         "naver-blog"   : "naver",
                         "naver-chzzk"  : "chzzk",
                         "naver-webtoon": "naverwebtoon",
+                        "pixiv-novel"  : "pixiv",
                     }
                 from .extractor import common
                 common.CATEGORY_MAP = catmap

gallery_dl/config.py

@@ -175,6 +175,7 @@ def remap_categories():
             ("naver"       , "naver-blog"),
             ("chzzk"       , "naver-chzzk"),
             ("naverwebtoon", "naver-webtoon"),
+            ("pixiv"       , "pixiv-novel"),
         )
     elif not cmap:
         return

gallery_dl/extractor/pixiv.py

@@ -861,18 +861,71 @@ class PixivSeriesExtractor(PixivExtractor):
             yield work
 
 
-class PixivNovelExtractor(PixivExtractor):
+class PixivSketchExtractor(Extractor):
-    """Extractor for pixiv novels"""
+    """Extractor for user pages on sketch.pixiv.net"""
-    subcategory = "novel"
+    category = "pixiv"
-    request_interval = (0.5, 1.5)
+    subcategory = "sketch"
-    pattern = BASE_PATTERN + r"/n(?:ovel/show\.php\?id=|/)(\d+)"
+    directory_fmt = ("{category}", "sketch", "{user[unique_name]}")
-    example = "https://www.pixiv.net/novel/show.php?id=12345"
+    filename_fmt = "{post_id} {id}.{extension}"
-
+    archive_fmt = "S{user[id]}_{id}"
-    def __init__(self, match):
+    root = "https://sketch.pixiv.net"
-        PixivExtractor.__init__(self, match)
+    cookies_domain = ".pixiv.net"
-        self.novel_id = match[1]
+    pattern = r"(?:https?://)?sketch\.pixiv\.net/@([^/?#]+)"
+    example = "https://sketch.pixiv.net/@USER"
 
     def items(self):
+        self.username = self.groups[0]
+        headers = {"Referer": f"{self.root}/@{self.username}"}
+
+        for post in self.posts():
+            media = post["media"]
+            post["post_id"] = post["id"]
+            post["date"] = text.parse_datetime(
+                post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
+            util.delete_items(post, ("id", "media", "_links"))
+
+            yield Message.Directory, post
+            post["_http_headers"] = headers
+
+            for photo in media:
+                original = photo["photo"]["original"]
+                post["id"] = photo["id"]
+                post["width"] = original["width"]
+                post["height"] = original["height"]
+
+                url = original["url"]
+                text.nameext_from_url(url, post)
+                yield Message.Url, url, post
+
+    def posts(self):
+        url = f"{self.root}/api/walls/@{self.username}/posts/public.json"
+        headers = {
+            "Accept": "application/vnd.sketch-v4+json",
+            "Referer": self.root + "/",
+            "X-Requested-With": f"{self.root}/@{self.username}",
+        }
+
+        while True:
+            data = self.request_json(url, headers=headers)
+            yield from data["data"]["items"]
+
+            next_url = data["_links"].get("next")
+            if not next_url:
+                return
+            url = self.root + next_url["href"]
+
+
+###############################################################################
+# Novels ######################################################################
+
+class PixivNovelExtractor(PixivExtractor):
+    """Base class for pixiv novel extractors"""
+    category = "pixiv-novel"
+    request_interval = (0.5, 1.5)
+
+    def items(self):
+        self.novel_id = self.groups[0]
+
         tags = self.config("tags", "japanese")
         if tags == "original":
             transform_tags = None
@@ -974,6 +1027,13 @@ class PixivNovelExtractor(PixivExtractor):
                         url = f"{self.root}/artworks/{illust_id}"
                         yield Message.Queue, url, novel
 
+
+class PixivNovelNovelExtractor(PixivNovelExtractor):
+    """Extractor for pixiv novels"""
+    subcategory = "novel"
+    pattern = BASE_PATTERN + r"/n(?:ovel/show\.php\?id=|/)(\d+)"
+    example = "https://www.pixiv.net/novel/show.php?id=12345"
+
     def novels(self):
         novel = self.api.novel_detail(self.novel_id)
         if self.config("full-series") and novel["series"]:
@@ -984,7 +1044,7 @@ class PixivNovelExtractor(PixivExtractor):
 
 class PixivNovelUserExtractor(PixivNovelExtractor):
     """Extractor for pixiv users' novels"""
-    subcategory = "novel-user"
+    subcategory = "user"
     pattern = USER_PATTERN + r"/novels"
     example = "https://www.pixiv.net/en/users/12345/novels"
 
@@ -994,7 +1054,7 @@ class PixivNovelUserExtractor(PixivNovelExtractor):
 
 class PixivNovelSeriesExtractor(PixivNovelExtractor):
     """Extractor for pixiv novel series"""
-    subcategory = "novel-series"
+    subcategory = "series"
     pattern = BASE_PATTERN + r"/novel/series/(\d+)"
     example = "https://www.pixiv.net/novel/series/12345"
 
@@ -1004,85 +1064,25 @@ class PixivNovelSeriesExtractor(PixivNovelExtractor):
 
 class PixivNovelBookmarkExtractor(PixivNovelExtractor):
     """Extractor for bookmarked pixiv novels"""
-    subcategory = "novel-bookmark"
+    subcategory = "bookmark"
     pattern = (USER_PATTERN + r"/bookmarks/novels"
                r"(?:/([^/?#]+))?(?:/?\?([^#]+))?")
     example = "https://www.pixiv.net/en/users/12345/bookmarks/novels"
 
-    def __init__(self, match):
-        PixivNovelExtractor.__init__(self, match)
-        self.user_id, self.tag, self.query = match.groups()
-
     def novels(self):
-        if self.tag:
+        user_id, tag, query = self.groups
-            tag = text.unquote(self.tag)
+        tag = text.unquote(tag) if tag else None
-        else:
-            tag = None
 
-        if text.parse_query(self.query).get("rest") == "hide":
+        if text.parse_query(query).get("rest") == "hide":
             restrict = "private"
         else:
             restrict = "public"
 
-        return self.api.user_bookmarks_novel(self.user_id, tag, restrict)
+        return self.api.user_bookmarks_novel(user_id, tag, restrict)
 
 
-class PixivSketchExtractor(Extractor):
+###############################################################################
-    """Extractor for user pages on sketch.pixiv.net"""
+# API #########################################################################
-    category = "pixiv"
-    subcategory = "sketch"
-    directory_fmt = ("{category}", "sketch", "{user[unique_name]}")
-    filename_fmt = "{post_id} {id}.{extension}"
-    archive_fmt = "S{user[id]}_{id}"
-    root = "https://sketch.pixiv.net"
-    cookies_domain = ".pixiv.net"
-    pattern = r"(?:https?://)?sketch\.pixiv\.net/@([^/?#]+)"
-    example = "https://sketch.pixiv.net/@USER"
-
-    def __init__(self, match):
-        Extractor.__init__(self, match)
-        self.username = match[1]
-
-    def items(self):
-        headers = {"Referer": f"{self.root}/@{self.username}"}
-
-        for post in self.posts():
-            media = post["media"]
-            post["post_id"] = post["id"]
-            post["date"] = text.parse_datetime(
-                post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
-            util.delete_items(post, ("id", "media", "_links"))
-
-            yield Message.Directory, post
-            post["_http_headers"] = headers
-
-            for photo in media:
-                original = photo["photo"]["original"]
-                post["id"] = photo["id"]
-                post["width"] = original["width"]
-                post["height"] = original["height"]
-
-                url = original["url"]
-                text.nameext_from_url(url, post)
-                yield Message.Url, url, post
-
-    def posts(self):
-        url = f"{self.root}/api/walls/@{self.username}/posts/public.json"
-        headers = {
-            "Accept": "application/vnd.sketch-v4+json",
-            "X-Requested-With": f"{self.root}/@{self.username}",
-            "Referer": self.root + "/",
-        }
-
-        while True:
-            data = self.request_json(url, headers=headers)
-            yield from data["data"]["items"]
-
-            next_url = data["_links"].get("next")
-            if not next_url:
-                return
-            url = self.root + next_url["href"]
-
 
 class PixivAppAPI():
     """Minimal interface for the Pixiv App API for mobile devices

scripts/supportedsites.py

@@ -122,6 +122,8 @@ CATEGORY_MAP = {
     "photovogue"     : "PhotoVogue",
     "pidgiwiki"      : "PidgiWiki",
     "pixeldrain"     : "pixeldrain",
+    "pixiv"          : "[pixiv]",
+    "pixiv-novel"    : "[pixiv] Novels",
     "pornimage"      : "Porn Image",
     "pornpics"       : "PornPics.com",
     "pornreactor"    : "PornReactor",
@@ -566,7 +568,7 @@ def subcategory_text(bc, c, sc):
 
 def category_key(c):
     """Generate sorting keys by category"""
-    return category_text(c[0]).lower()
+    return category_text(c[0]).lower().lstrip("[")
 
 
 def subcategory_key(sc):
@@ -610,6 +612,8 @@ def build_extractor_list():
         for subcategories in base.values():
             subcategories.sort(key=subcategory_key)
 
+    domains["pixiv-novel"] += "novel"
+
     # add e-hentai.org
     default["e-hentai"] = default["exhentai"]
     domains["e-hentai"] = domains["exhentai"].replace("x", "-")

test/results/pixiv.py

@@ -11,37 +11,31 @@ from gallery_dl import exception
 __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/en/users/173530",
-    "#category": ("", "pixiv", "user"),
     "#class"   : pixiv.PixivUserExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/u/173530",
-    "#category": ("", "pixiv", "user"),
     "#class"   : pixiv.PixivUserExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/member.php?id=173530",
-    "#category": ("", "pixiv", "user"),
     "#class"   : pixiv.PixivUserExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/mypage.php#id=173530",
-    "#category": ("", "pixiv", "user"),
     "#class"   : pixiv.PixivUserExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/#id=173530",
-    "#category": ("", "pixiv", "user"),
     "#class"   : pixiv.PixivUserExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/en/users/173530/artworks",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
     "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7",
 },
@@ -49,14 +43,12 @@ __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D",
     "#comment" : "illusts with specific tag",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
     "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658",
 },
 
 {
     "#url"     : "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
     "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658",
 },
@@ -64,7 +56,6 @@ __tests__ = (
 {
     "#url"     : "http://www.pixiv.net/member_illust.php?id=173531",
     "#comment" : "deleted account",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
     "#options"  : {"metadata": True},
     "#exception": exception.NotFoundError,
@@ -79,64 +70,54 @@ __tests__ = (
 
 {
     "#url"     : "https://www.pixiv.net/en/users/173530/manga",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/en/users/173530/illustrations",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/member_illust.php?id=173530",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
 },
 
 {
     "#url"     : "https://touch.pixiv.net/member_illust.php?id=173530",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
 },
 
 {
     "#url"     : "https://www.phixiv.net/member_illust.php?id=173530",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
 },
 
 {
     "#url"     : "https://phixiv.net/en/users/56514424/artworks",
-    "#category": ("", "pixiv", "artworks"),
     "#class"   : pixiv.PixivArtworksExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/en/users/173530/avatar",
-    "#category": ("", "pixiv", "avatar"),
     "#class"   : pixiv.PixivAvatarExtractor,
     "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66",
 },
 
 {
     "#url"     : "https://www.pixiv.net/en/users/194921/background",
-    "#category": ("", "pixiv", "background"),
     "#class"   : pixiv.PixivBackgroundExtractor,
     "#pattern" : r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg",
 },
 
 {
     "#url"     : "https://pixiv.me/del_shannon",
-    "#category": ("", "pixiv", "me"),
     "#class"   : pixiv.PixivMeExtractor,
     "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf",
 },
 
 {
     "#url"     : "https://pixiv.me/del_shanno",
-    "#category": ("", "pixiv", "me"),
     "#class"   : pixiv.PixivMeExtractor,
     "#exception": exception.NotFoundError,
 },
@@ -144,7 +125,6 @@ __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/artworks/966412",
     "#comment" : "related works (#1237)",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
     "#sha1_url"    : "90c1715b07b0d1aad300bce256a0bc71f42540ba",
     "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a",
@@ -155,7 +135,6 @@ __tests__ = (
 
 {
     "#url"     : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
     "#exception": exception.NotFoundError,
 },
@@ -163,7 +142,6 @@ __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629",
     "#comment" : "ugoira",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
     "#results" : "https://i.pximg.net/img-zip-ugoira/img/2018/01/15/13/24/48/66806629_ugoira1920x1080.zip",
 
@@ -175,7 +153,6 @@ __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/artworks/101003492",
     "#comment" : "original ugoira frames (#6056)",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
     "#options" : {"ugoira": "original"},
     "#results" : (
@@ -193,7 +170,6 @@ __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/artworks/966412",
     "#comment" : "related works (#1237)",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
     "#options" : {"related": True},
     "#range"   : "1-10",
@@ -321,55 +297,46 @@ __tests__ = (
 
 {
     "#url"     : "https://www.pixiv.net/en/artworks/966412",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/i/966412",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "http://img.pixiv.net/img/soundcross/42626136.jpg",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "https://www.phixiv.net/en/artworks/966412",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
 {
     "#url"     : "https://phixiv.net/member_illust.php?mode=medium&illust_id=966412",
-    "#category": ("", "pixiv", "work"),
     "#class"   : pixiv.PixivWorkExtractor,
 },
 
@@ -384,7 +351,6 @@ __tests__ = (
 
 {
     "#url"     : "https://www.pixiv.net/en/users/173530/bookmarks/artworks",
-    "#category": ("", "pixiv", "favorite"),
     "#class"   : pixiv.PixivFavoriteExtractor,
     "#results" : (
         "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg",
@@ -397,7 +363,6 @@ __tests__ = (
 
 {
     "#url"     : "https://www.pixiv.net/bookmark.php?id=173530",
-    "#category": ("", "pixiv", "favorite"),
     "#class"   : pixiv.PixivFavoriteExtractor,
     "#results" : (
         "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg",
@@ -411,7 +376,6 @@ __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93",
     "#comment" : "bookmarks with specific tag",
-    "#category": ("", "pixiv", "favorite"),
     "#class"   : pixiv.PixivFavoriteExtractor,
     "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c",
 },
@@ -419,7 +383,6 @@ __tests__ = (
 {
     "#url"     : "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1",
     "#comment" : "bookmarks with specific tag (legacy url)",
-    "#category": ("", "pixiv", "favorite"),
     "#class"   : pixiv.PixivFavoriteExtractor,
     "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c",
 },
@@ -467,7 +430,6 @@ __tests__ = (
 {
     "#url"     : "https://touch.pixiv.net/bookmark.php?id=173530",
     "#comment" : "touch URLs",
-    "#category": ("", "pixiv", "favorite"),
     "#class"   : pixiv.PixivFavoriteExtractor,
 },
 
@@ -479,13 +441,11 @@ __tests__ = (
 
 {
     "#url"     : "https://www.pixiv.net/ranking.php?mode=daily&date=20170818",
-    "#category": ("", "pixiv", "ranking"),
     "#class"   : pixiv.PixivRankingExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/ranking.php",
-    "#category": ("", "pixiv", "ranking"),
     "#class"   : pixiv.PixivRankingExtractor,
     "#options" : {"max-posts": 10},
 
@@ -498,20 +458,17 @@ __tests__ = (
 
 {
     "#url"     : "https://touch.pixiv.net/ranking.php",
-    "#category": ("", "pixiv", "ranking"),
     "#class"   : pixiv.PixivRankingExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/ranking.php?mode=unknown",
-    "#category": ("", "pixiv", "ranking"),
     "#class"   : pixiv.PixivRankingExtractor,
     "#exception": exception.StopExtraction,
 },
 
 {
     "#url"     : "https://www.pixiv.net/en/tags/Original",
-    "#category": ("", "pixiv", "search"),
     "#class"   : pixiv.PixivSearchExtractor,
     "#range"   : "1-10",
     "#count"   : 10,
@@ -519,64 +476,54 @@ __tests__ = (
 
 {
     "#url"     : "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag",
-    "#category": ("", "pixiv", "search"),
     "#class"   : pixiv.PixivSearchExtractor,
     "#exception": exception.StopExtraction,
 },
 
 {
     "#url"     : "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag",
-    "#category": ("", "pixiv", "search"),
     "#class"   : pixiv.PixivSearchExtractor,
     "#exception": exception.StopExtraction,
 },
 
 {
     "#url"     : "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original",
-    "#category": ("", "pixiv", "search"),
     "#class"   : pixiv.PixivSearchExtractor,
     "#exception": exception.StopExtraction,
 },
 
 {
     "#url"     : "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag",
-    "#category": ("", "pixiv", "search"),
     "#class"   : pixiv.PixivSearchExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original",
-    "#category": ("", "pixiv", "search"),
     "#class"   : pixiv.PixivSearchExtractor,
 },
 
 {
     "#url"     : "https://touch.pixiv.net/search.php?word=Original",
-    "#category": ("", "pixiv", "search"),
     "#class"   : pixiv.PixivSearchExtractor,
 },
 
 {
     "#url"     : "https://www.pixiv.net/bookmark_new_illust.php",
-    "#category": ("", "pixiv", "follow"),
     "#class"   : pixiv.PixivFollowExtractor,
 },
 
 {
     "#url"     : "https://touch.pixiv.net/bookmark_new_illust.php",
-    "#category": ("", "pixiv", "follow"),
     "#class"   : pixiv.PixivFollowExtractor,
 },
 
 {
     "#url"     : "https://www.pixivision.net/en/a/2791",
-    "#category": ("", "pixiv", "pixivision"),
     "#class"   : pixiv.PixivPixivisionExtractor,
 },
 
 {
     "#url"     : "https://pixivision.net/a/2791",
-    "#category": ("", "pixiv", "pixivision"),
     "#class"   : pixiv.PixivPixivisionExtractor,
     "#count"   : 7,
 
@@ -586,7 +533,6 @@ __tests__ = (
 
 {
     "#url"     : "https://www.pixiv.net/user/10509347/series/21859",
-    "#category": ("", "pixiv", "series"),
     "#class"   : pixiv.PixivSeriesExtractor,
     "#range"   : "1-10",
     "#count"   : 10,
@@ -604,119 +550,8 @@ __tests__ = (
     },
 },
 
-{
-    "#url"     : "https://www.pixiv.net/novel/show.php?id=12101012",
-    "#category": ("", "pixiv", "novel"),
-    "#class"   : pixiv.PixivNovelExtractor,
-    "#count"       : 1,
-    "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93",
-
-    "caption"        : "<br />第一印象から決めてました！<br /><br />素敵な表紙はいもこは妹さん(<strong><a href=\"pixiv://illusts/53802907\">illust/53802907</a></strong>)からお借りしました。<br /><br />たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！<br /><br />■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　<br /><br />■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！",
-    "create_date"    : "2019-12-19T23:14:36+09:00",
-    "date"           : "dt:2019-12-19 14:14:36",
-    "extension"      : "txt",
-    "id"             : 12101012,
-    "image_urls"     : dict,
-    "is_bookmarked"  : False,
-    "is_muted"       : False,
-    "is_mypixiv_only": False,
-    "is_original"    : False,
-    "is_x_restricted": False,
-    "novel_ai_type"  : 0,
-    "page_count"     : 1,
-    "rating"         : "General",
-    "restrict"       : 0,
-    "series"         : {
-        "id"   : 1479656,
-        "title": "一目惚れした彼らの話",
-    },
-    "tags"           : [
-        "鬼滅の夢",
-        "女主人公",
-        "煉獄杏寿郎",
-        "涙腺崩壊",
-        "なにこれすごい",
-        "来世で幸せになって欲しい",
-        "キメ学世界線できっと幸せになってる!!",
-        "あなたが神か!!",
-        "キメ学編を·····",
-        "鬼滅の夢小説10000users入り",
-    ],
-    "text_length"    : 9569,
-    "title"          : "本当は、一目惚れだった",
-    "total_bookmarks": range(17900, 20000),
-    "total_comments" : range(200, 400),
-    "total_view"     : range(158000, 300000),
-    "user"           : {
-        "account": "46_maru",
-        "id"     : 888268,
-    },
-    "visible"        : True,
-    "x_restrict"     : 0,
-},
-
-{
-    "#url"     : "https://www.pixiv.net/novel/show.php?id=16422450",
-    "#comment" : "embeds // covers (#5373)",
-    "#category": ("", "pixiv", "novel"),
-    "#class"   : pixiv.PixivNovelExtractor,
-    "#options" : {
-        "embeds": True,
-        "covers": True,
-    },
-    "#count"   : 4,
-},
-
-{
-    "#url"     : "https://www.pixiv.net/novel/show.php?id=12101012",
-    "#comment" : "full series",
-    "#category": ("", "pixiv", "novel"),
-    "#class"   : pixiv.PixivNovelExtractor,
-    "#options" : {"full-series": True},
-    "#count"   : 2,
-},
-
-{
-    "#url"     : "https://www.pixiv.net/n/19612040",
-    "#comment" : "short URL",
-    "#category": ("", "pixiv", "novel"),
-    "#class"   : pixiv.PixivNovelExtractor,
-},
-
-{
-    "#url"     : "https://www.pixiv.net/en/users/77055466/novels",
-    "#category": ("", "pixiv", "novel-user"),
-    "#class"   : pixiv.PixivNovelUserExtractor,
-    "#pattern" : "^text:",
-    "#range"   : "1-5",
-    "#count"   : 5,
-},
-
-{
-    "#url"     : "https://www.pixiv.net/novel/series/1479656",
-    "#category": ("", "pixiv", "novel-series"),
-    "#class"   : pixiv.PixivNovelSeriesExtractor,
-    "#count"       : 2,
-    "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b",
-},
-
-{
-    "#url"     : "https://www.pixiv.net/en/users/77055466/bookmarks/novels",
-    "#category": ("", "pixiv", "novel-bookmark"),
-    "#class"   : pixiv.PixivNovelBookmarkExtractor,
-    "#count"       : 1,
-    "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089",
-},
-
-{
-    "#url"     : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide",
-    "#category": ("", "pixiv", "novel-bookmark"),
-    "#class"   : pixiv.PixivNovelBookmarkExtractor,
-},
-
 {
     "#url"     : "https://sketch.pixiv.net/@nicoby",
-    "#category": ("", "pixiv", "sketch"),
     "#class"   : pixiv.PixivSketchExtractor,
     "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)",
     "#count"   : ">= 35",

test/results/pixivnovel.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import pixiv
+
+
+__tests__ = (
+{
+    "#url"     : "https://www.pixiv.net/novel/show.php?id=12101012",
+    "#class"   : pixiv.PixivNovelNovelExtractor,
+    "#count"       : 1,
+    "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93",
+
+    "caption"        : "<br />第一印象から決めてました！<br /><br />素敵な表紙はいもこは妹さん(<strong><a href=\"pixiv://illusts/53802907\">illust/53802907</a></strong>)からお借りしました。<br /><br />たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！<br /><br />■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　<br /><br />■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！",
+    "create_date"    : "2019-12-19T23:14:36+09:00",
+    "date"           : "dt:2019-12-19 14:14:36",
+    "extension"      : "txt",
+    "id"             : 12101012,
+    "image_urls"     : dict,
+    "is_bookmarked"  : False,
+    "is_muted"       : False,
+    "is_mypixiv_only": False,
+    "is_original"    : False,
+    "is_x_restricted": False,
+    "novel_ai_type"  : 0,
+    "page_count"     : 1,
+    "rating"         : "General",
+    "restrict"       : 0,
+    "series"         : {
+        "id"   : 1479656,
+        "title": "一目惚れした彼らの話",
+    },
+    "tags"           : [
+        "鬼滅の夢",
+        "女主人公",
+        "煉獄杏寿郎",
+        "涙腺崩壊",
+        "なにこれすごい",
+        "来世で幸せになって欲しい",
+        "キメ学世界線できっと幸せになってる!!",
+        "あなたが神か!!",
+        "キメ学編を·····",
+        "鬼滅の夢小説10000users入り",
+    ],
+    "text_length"    : 9569,
+    "title"          : "本当は、一目惚れだった",
+    "total_bookmarks": range(17900, 20000),
+    "total_comments" : range(200, 400),
+    "total_view"     : range(158000, 300000),
+    "user"           : {
+        "account": "46_maru",
+        "id"     : 888268,
+    },
+    "visible"        : True,
+    "x_restrict"     : 0,
+},
+
+{
+    "#url"     : "https://www.pixiv.net/novel/show.php?id=16422450",
+    "#comment" : "embeds // covers (#5373)",
+    "#class"   : pixiv.PixivNovelNovelExtractor,
+    "#options" : {
+        "embeds": True,
+        "covers": True,
+    },
+    "#count"   : 4,
+},
+
+{
+    "#url"     : "https://www.pixiv.net/novel/show.php?id=12101012",
+    "#comment" : "full series",
+    "#class"   : pixiv.PixivNovelNovelExtractor,
+    "#options" : {"full-series": True},
+    "#count"   : 2,
+},
+
+{
+    "#url"     : "https://www.pixiv.net/n/19612040",
+    "#comment" : "short URL",
+    "#class"   : pixiv.PixivNovelNovelExtractor,
+},
+
+{
+    "#url"     : "https://www.pixiv.net/en/users/77055466/novels",
+    "#class"   : pixiv.PixivNovelUserExtractor,
+    "#pattern" : "^text:",
+    "#range"   : "1-5",
+    "#count"   : 5,
+},
+
+{
+    "#url"     : "https://www.pixiv.net/novel/series/1479656",
+    "#class"   : pixiv.PixivNovelSeriesExtractor,
+    "#count"       : 2,
+    "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b",
+},
+
+{
+    "#url"     : "https://www.pixiv.net/en/users/77055466/bookmarks/novels",
+    "#class"   : pixiv.PixivNovelBookmarkExtractor,
+    "#count"       : 1,
+    "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089",
+},
+
+{
+    "#url"     : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide",
+    "#class"   : pixiv.PixivNovelBookmarkExtractor,
+},
+
+)