fix supportedsites.py

docs/supportedsites.rst

@@ -15,8 +15,10 @@ Site                 URL                                 Capabilities
 500px                https://500px.com/                  Galleries, individual Images, User Profiles
 8kun                 https://8kun.top/                   Boards, Threads
 8muses               https://comics.8muses.com/          Albums
+Acidimg              https://acidimg.cc/                 individual Images
 Adobe Portfolio      https://www.myportfolio.com/        Galleries
 Adult Empire         https://www.adultempire.com/        Galleries
+Allgirlbooru         https://allgirl.booru.org/          Posts, Tag Searches
 arch.b4k.co          https://arch.b4k.co/                Boards, Search Results, Threads
 Archive of Sins      https://archiveofsins.com/          Boards, Search Results, Threads
 Archived.Moe         https://archived.moe/               Boards, Search Results, Threads
@@ -56,17 +58,23 @@ Hiperdex             https://hiperdex.com/               Artists, Chapters, Mang
 Hitomi.la            https://hitomi.la/                  Galleries, Tag Searches
 Hypnohub             https://hypnohub.net/               Pools, Popular Images, Posts, Tag Searches
 Idol Complex         https://idol.sankakucomplex.com/    Pools, Posts, Tag Searches                         Supported
+Illusioncardsbooru   https://illusioncards.booru.org/    Posts, Tag Searches
 ImageBam             http://www.imagebam.com/            Galleries, individual Images
 ImageChest           https://imgchest.com/               Galleries
 ImageFap             https://www.imagefap.com/           Galleries, individual Images, User Profiles
+Imagetwist           https://imagetwist.com/             individual Images
+Imagevenue           http://imagevenue.com/              individual Images
 ImgBB                https://imgbb.com/                  Albums, individual Images, User Profiles           Supported
 imgbox               https://imgbox.com/                 Galleries, individual Images
+Imgclick             https://imgclick.net/               individual Images
+Imgspice             https://imgspice.com/               individual Images
 imgth                https://imgth.com/                  Galleries
 imgur                https://imgur.com/                  |imgur-C|
+Imxto                https://imx.to/                     individual Images
 Inkbunny             https://inkbunny.net/               Posts, User Profiles                               Supported
 Instagram            https://www.instagram.com/          |instagram-C|                                      Supported
 Issuu                https://issuu.com/                  Publications, User Profiles
-Joyreactor           http://joyreactor.com/              Posts, Search Results, Tag Searches, User Profiles
+Joyreactor           http://joyreactor.cc/               Posts, Search Results, Tag Searches, User Profiles
 Keenspot             http://www.keenspot.com/            Comics
 Kemonoparty          https://kemono.party/               Posts, User Profiles
 Khinsider            https://downloads.khinsider.com/    Soundtracks
@@ -103,11 +111,13 @@ PhotoVogue           https://www.vogue.it/en/photovogue/ User Profiles
 Piczel               https://piczel.tv/                  Folders, individual Images, User Profiles
 Pillowfort           https://www.pillowfort.social/      Posts, User Profiles
 Pinterest            https://www.pinterest.com/          |pinterest-C|                                      Supported
+Pixhost              https://pixhost.to/                 individual Images
 Pixiv                https://www.pixiv.net/              |pixiv-C|                                          `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Pixnet               https://www.pixnet.net/             Folders, individual Images, Sets, User Profiles
 Plurk                https://www.plurk.com/              Posts, Timelines
 Pornhub              https://www.pornhub.com/            Galleries, User Profiles
 Pornreactor          http://pornreactor.cc/              Posts, Search Results, Tag Searches, User Profiles
+Postimg              https://postimages.org/             individual Images
 PowerManga           https://read.powermanga.org/        Chapters, Manga
 Pururin              https://pururin.io/                 Galleries
 Read Comic Online    https://readcomiconline.to/         Comic Issues, Comics
@@ -130,11 +140,15 @@ SlideShare           https://www.slideshare.net/         Presentations
 SmugMug              https://www.smugmug.com/            |smugmug-C|                                        `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Speaker Deck         https://speakerdeck.com/            Presentations
 SubscribeStar        https://www.subscribestar.com/      Posts, User Profiles                               Supported
+Tbib                 https://tbib.org/                   Pools, Posts, Tag Searches
 The /b/ Archive      https://thebarchive.com/            Boards, Search Results, Threads
+Thecollection        https://the-collection.booru.org/   Posts, Tag Searches
 Tsumino              https://www.tsumino.com/            Galleries, Search Results                          Supported
 Tumblr               https://www.tumblr.com/             Likes, Posts, Tag Searches, User Profiles          `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
+Turboimagehost       https://www.turboimagehost.com/     individual Images
 Twitter              https://twitter.com/                |twitter-C|                                        Supported
 Unsplash             https://unsplash.com/               |unsplash-C|
+Vipr                 https://vipr.im/                    individual Images
 VSCO                 https://vsco.co/                    Collections, individual Images, User Profiles
 Wallhaven            https://wallhaven.cc/               individual Images, Search Results                  `API Key <configuration.rst#extractorwallhavenapi-key>`__
 Warosu               https://warosu.org/                 Threads
@@ -146,16 +160,6 @@ xHamster             https://xhamster.com/               Galleries, User Profile
 XVideos              https://www.xvideos.com/            Galleries, User Profiles
 Yandere              https://yande.re/                   Pools, Popular Images, Posts, Tag Searches
 |yuki-S|             https://yuki.la/                    Threads
-Acidimg              https://acidimg.cc/                 individual Images
-Imagetwist           https://imagetwist.com/             individual Images
-Imagevenue           http://imagevenue.com/              individual Images
-Imgclick             https://imgclick.net/               individual Images
-Imgspice             https://imgspice.com/               individual Images
-Imxto                https://imx.to/                     individual Images
-Pixhost              https://pixhost.to/                 individual Images
-Postimg              https://postimages.org/             individual Images
-Turboimagehost       https://www.turboimagehost.com/     individual Images
-Vipr                 https://vipr.im/                    individual Images
 かべうち                 https://kabe-uchiroom.com/          User Profiles
 もえぴりあ                https://vanilla-rock.com/           Posts, Tag Searches
 半次元                  https://bcy.net/                    Posts, User Profiles

scripts/supportedsites.py

@@ -232,16 +232,13 @@ def domain(cls):
     return ""
 
 
-def category_text(cls):
+def category_text(c):
     """Return a human-readable representation of a category"""
-    c = cls.category
     return CATEGORY_MAP.get(c) or c.capitalize()
 
 
-def subcategory_text(cls):
+def subcategory_text(c, sc):
     """Return a human-readable representation of a subcategory"""
-    c, sc = cls.category, cls.subcategory
-
     if c in SUBCATEGORY_MAP:
         scm = SUBCATEGORY_MAP[c]
         if sc in scm:
@@ -254,66 +251,61 @@ def subcategory_text(cls):
     return sc if sc.endswith("s") else sc + "s"
 
 
-def category_key(cls):
+def category_key(c):
     """Generate sorting keys by category"""
-    key = category_text(cls).lower()
+    return category_text(c[0]).lower()
-    if cls.__module__.endswith(".imagehosts"):
-        key = "zz" + key
-    return key
 
 
-def subcategory_key(cls):
+def subcategory_key(sc):
     """Generate sorting keys by subcategory"""
-    if cls.subcategory == "issue":
+    return "A" if sc == "issue" else sc
-        return "A"
-    return cls.subcategory
 
 
 def build_extractor_list():
     """Generate a sorted list of lists of extractor classes"""
-    extractors = collections.defaultdict(list)
+    categories = collections.defaultdict(list)
+    domains = {}
 
-    # get lists of extractor classes grouped by category
+    for extr in extractor._list_classes():
-    for extr in extractor.extractors():
+        category = extr.category
-        if not extr.category or extr.category in IGNORE_LIST:
+        if category in IGNORE_LIST:
             continue
-        extractors[extr.category].append(extr)
+        if category:
+            categories[category].append(extr.subcategory)
+            if category not in domains:
+                domains[category] = domain(extr)
+        else:
+            for category, root in extr.instances:
+                categories[category].append(extr.subcategory)
+                if category not in domains:
+                    domains[category] = root + "/"
 
-    # sort extractor lists with the same category
+    # sort subcategory lists
-    for extrlist in extractors.values():
+    for subcategories in categories.values():
-        extrlist.sort(key=subcategory_key)
+        subcategories.sort(key=subcategory_key)
 
-    # ugly hack to add e-hentai.org
+    # add e-hentai.org
-    eh = []
+    categories["e-hentai"] = categories["exhentai"]
-    for extr in extractors["exhentai"]:
+    domains["e-hentai"] = domains["exhentai"].replace("x", "-")
-        class eh_extr(extr):
-            category = "e-hentai"
-            root = "https://e-hentai.org"
-        eh.append(eh_extr)
-    extractors["e-hentai"] = eh
 
-    # sort lists by category
+    return categories, domains
-    return sorted(
-        extractors.values(),
-        key=lambda lst: category_key(lst[0]),
-    )
 
 
 # define table columns
 COLUMNS = (
     ("Site", 20,
-     lambda x: category_text(x[0])),
+     lambda c, scs, d: category_text(c)),
     ("URL" , 35,
-     lambda x: domain(x[0])),
+     lambda c, scs, d: d),
     ("Capabilities", 50,
-     lambda x: ", ".join(subcategory_text(extr) for extr in x
+     lambda c, scs, d: ", ".join(subcategory_text(c, sc) for sc in scs
-                         if subcategory_text(extr))),
+                                 if subcategory_text(c, sc))),
     ("Authentication", 16,
-     lambda x: AUTH_MAP.get(x[0].category, "")),
+     lambda c, scs, d: AUTH_MAP.get(c, "")),
 )
 
 
-def write_output(fobj, columns, extractors):
+def write_output(fp, columns, categories, domains):
 
     def pad(output, col, category=None):
         size = col[1]
@@ -326,7 +318,7 @@ def write_output(fobj, columns, extractors):
 
         return output + " " * (size - len(output))
 
-    w = fobj.write
+    w = fp.write
     subs = []
 
     # caption
@@ -343,9 +335,11 @@ def write_output(fobj, columns, extractors):
     w(sep)
 
     # table body
-    for lst in extractors:
+    clist = sorted(categories.items(), key=category_key)
+    for category, subcategories in clist:
+        domain = domains[category]
         w(" ".join(
-            pad(col[2](lst), col, lst[0].category)
+            pad(col[2](category, subcategories, domain), col, category)
             for col in columns
         ).strip())
         w("\n")
@@ -359,6 +353,7 @@ def write_output(fobj, columns, extractors):
         w(".. {} replace:: {}\n".format(sub, value))
 
 
+categories, domains = build_extractor_list()
 outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst"
 with open(util.path("docs", outfile), "w") as file:
-    write_output(file, COLUMNS, build_extractor_list())
+    write_output(file, COLUMNS, categories, domains)