[xenforo] support 'allthefallen.moe/forum' (#3249 #8268)

2025-12-15 17:20:32 +01:00
parent 207f6137c5
commit c268f793a4
5 changed files with 139 additions and 18 deletions
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1867,6 +1867,12 @@ Consider all listed sites to potentially be NSFW.
    <td>Forums, Posts, Threads</td>
    <td>Supported</td>
 </tr>
+<tr id="atfforum" title="atfforum">
+    <td>All The Fallen</td>
+    <td>https://allthefallen.moe/forum/</td>
+    <td>Forums, Posts, Threads</td>
+    <td></td>
+</tr>

 <tr id="moebooru" title="moebooru">
    <td colspan="4"><strong>Moebooru and MyImouto</strong></td>
--- a/gallery_dl/extractor/xenforo.py
+++ b/gallery_dl/extractor/xenforo.py
@@ -16,8 +16,6 @@ from ..cache import cache
 class XenforoExtractor(BaseExtractor):
    """Base class for xenforo extractors"""
    basecategory = "xenforo"
-    #  cookies_domain = "simpcity.cr"
-    cookies_names = ("ogaddgmetaprof_user",)
    directory_fmt = ("{category}", "{thread[section]}",
                     "{thread[title]} ({thread[id]})")
    filename_fmt = "{post[id]}_{num:>02}_{id}_{filename}.{extension}"
@@ -35,9 +33,9 @@ class XenforoExtractor(BaseExtractor):
            r'(?s)(?:'
            r'<video (.*?\ssrc="[^"]+".*?)</video>'
            r'|<a [^>]*?href="[^"]*?'
-            r'(/attachments/[^"]+".*?)</a>'
+            r'(/(?:index\.php\?)?attachments/[^"]+".*?)</a>'
            r'|<div [^>]*?data-src="[^"]*?'
-            r'(/attachments/[^"]+".*?)/>'
+            r'(/(?:index\.php\?)attachments/[^"]+".*?)/>'
            r'|(?:<a [^>]*?href="|<iframe [^>]*?src="|'
            r'''onclick="loadMedia\(this, ')([^"']+)'''
            r')'
@@ -61,8 +59,11 @@ class XenforoExtractor(BaseExtractor):
                    data["num"] += 1
                    data["num_external"] += 1
                    data["type"] = "external"
-                    if ext.startswith("//"):
-                        ext = "https:" + ext
+                    if ext[0] == "/":
+                        if ext[1] == "/":
+                            ext = "https:" + ext
+                        else:
+                            continue
                    yield Message.Queue, ext, data

                elif video:
@@ -163,6 +164,8 @@ class XenforoExtractor(BaseExtractor):

        url = f"{base}/page-{'9999' if pnum is None else pnum}"
        with self.request_page(url) as response:
+            if pnum is None and not response.history:
+                self._require_auth()
            url = response.url
            if url[-1] == "/":
                pnum = 1
@@ -247,10 +250,10 @@ class XenforoExtractor(BaseExtractor):

        return post

-    def _require_auth(self, response):
+    def _require_auth(self, response=None):
        raise exception.AuthRequired(
            ("username & password", "authenticated cookies"), None,
-            self._extract_error(response.text))
+            None if response is None else self._extract_error(response.text))

    def _validate(self, response):
        if response.status_code == 403 and b">Log in<" in response.content:
@@ -269,17 +272,24 @@ BASE_PATTERN = XenforoExtractor.update({
        "pattern": r"(?:www\.)?nudostar\.com/forum",
        "cookies": ("xf_user",),
    },
+    "atfforum": {
+        "root": "https://allthefallen.moe/forum",
+        "pattern": r"(?:www\.)?allthefallen\.moe/forum",
+        "cookies": ("xf_user",),
+    },
 })


 class XenforoPostExtractor(XenforoExtractor):
    subcategory = "post"
-    pattern = rf"{BASE_PATTERN}/(?:threads/[^/?#]+/post-|posts/)(\d+)"
+    pattern = (rf"{BASE_PATTERN}(/(?:index\.php\?)?threads"
+               rf"/[^/?#]+/post-|/posts/)(\d+)")
    example = "https://simpcity.cr/threads/TITLE.12345/post-54321"

    def posts(self):
+        path = self.groups[-2]
        post_id = self.groups[-1]
-        url = f"{self.root}/posts/{post_id}/"
+        url = f"{self.root}{path}{post_id}/"
        page = self.request_page(url).text

        pos = page.find(f'data-content="post-{post_id}"')
@@ -293,7 +303,8 @@ class XenforoPostExtractor(XenforoExtractor):

 class XenforoThreadExtractor(XenforoExtractor):
    subcategory = "thread"
-    pattern = rf"{BASE_PATTERN}(/threads/(?:[^/?#]+\.)?\d+)(?:/page-(\d+))?"
+    pattern = (rf"{BASE_PATTERN}(/(?:index\.php\?)?threads"
+               rf"/(?:[^/?#]+\.)?\d+)(?:/page-(\d+))?")
    example = "https://simpcity.cr/threads/TITLE.12345/"

    def posts(self):
@@ -321,12 +332,13 @@ class XenforoThreadExtractor(XenforoExtractor):

 class XenforoForumExtractor(XenforoExtractor):
    subcategory = "forum"
-    pattern = rf"{BASE_PATTERN}(/forums/(?:[^/?#]+\.)?[^/?#]+)(?:/page-(\d+))?"
+    pattern = (rf"{BASE_PATTERN}(/(?:index\.php\?)?forums"
+               rf"/(?:[^/?#]+\.)?[^/?#]+)(?:/page-(\d+))?")
    example = "https://simpcity.cr/forums/TITLE.123/"

    def items(self):
        extract_threads = text.re(
-            r'(/threads/[^"]+)"[^>]+data-xf-init=').findall
+            r'(/(?:index\.php\?)?threads/[^"]+)"[^>]+data-xf-init=').findall

        data = {"_extractor": XenforoThreadExtractor}
        path = self.groups[-2]
--- a/scripts/supportedsites.py
+++ b/scripts/supportedsites.py
@@ -34,6 +34,7 @@ CATEGORY_MAP = {
    "artstation"     : "ArtStation",
    "aryion"         : "Eka's Portal",
    "atfbooru"       : "ATFBooru",
+    "atfforum"       : "All The Fallen",
    "azurlanewiki"   : "Azur Lane Wiki",
    "b4k"            : "arch.b4k.dev",
    "baraag"         : "baraag",
--- a/test/results/atfforum.py
+++ b/test/results/atfforum.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import xenforo
+
+
+__tests__ = (
+{
+    "#url"     : "https://www.allthefallen.moe/forum/index.php?threads/final-fantasy-xiv.57090/post-21765744",
+    "#category": ("xenforo", "atfforum", "post"),
+    "#class"   : xenforo.XenforoPostExtractor,
+    "#auth"    : True,
+    "#results" : "https://allthefallen.moe/forum/index.php?attachments/ffxiv_dx11-2025-04-28-09-16-54-png.4368606/",
+
+    "count"       : 1,
+    "extension"   : "png",
+    "filename"    : "ffxiv_dx11 2025-04-28 09-16-54",
+    "id"          : 4368606,
+    "num"         : 1,
+    "num_external": 0,
+    "num_internal": 1,
+    "type"        : "inline",
+    "post"        : {
+        "attachments": "",
+        "author"     : "mayumiXIV",
+        "author_id"  : "965353",
+        "author_url" : "https://allthefallen.moe/forum/index.php?members/mayumixiv.965353/",
+        "count"      : 1,
+        "date"       : "dt:2025-04-28 15:28:24",
+        "id"         : "21765744",
+        "content"    : str
+    },
+    "thread"      : {
+        "author"    : "Kupowo",
+        "author_id" : "649590",
+        "author_url": "https://allthefallen.moe/forum/index.php?members/kupowo.649590/",
+        "date"      : "dt:2023-12-25 21:15:53",
+        "id"        : "57090",
+        "section"   : "Gaming",
+        "title"     : "Final Fantasy XIV",
+        "url"       : "https://allthefallen.moe/forum/index.php?threads/final-fantasy-xiv.57090/",
+        "posts"     : range(210, 280),
+        "views"     : range(7300, 9000),
+        "tags"      : [
+            "ff14",
+            "final fantasy 14",
+            "final fantasy xiv",
+        ],
+    },
+},
+
+{
+    "#url"     : "https://www.allthefallen.moe/forum/index.php?threads/final-fantasy-xiv.57090/",
+    "#category": ("xenforo", "atfforum", "thread"),
+    "#class"   : xenforo.XenforoThreadExtractor,
+    "#auth"    : True,
+    "#count"   : range(50, 90),
+
+    "count"       : int,
+    "num"         : int,
+    "num_external": int,
+    "num_internal": int,
+    "type"        : {"inline", "external"},
+    "post"        : {
+        "attachments": str,
+        "author"     : str,
+        "author_id"  : r"re:\d",
+        "author_url" : r"re:https://allthefallen.moe/forum/index.php\?members/.+",
+        "count"      : range(0, 9),
+        "date"       : "type:datetime",
+        "id"         : r"re:\d+",
+        "content"    : str,
+    },
+    "thread"      : {
+        "author"    : "Kupowo",
+        "author_id" : "649590",
+        "author_url": "https://allthefallen.moe/forum/index.php?members/kupowo.649590/",
+        "date"      : "dt:2023-12-25 21:15:53",
+        "id"        : "57090",
+        "section"   : "Gaming",
+        "title"     : "Final Fantasy XIV",
+        "url"       : "https://allthefallen.moe/forum/index.php?threads/final-fantasy-xiv.57090/",
+        "posts"     : range(210, 280),
+        "views"     : range(7300, 9000),
+        "tags"      : [
+            "ff14",
+            "final fantasy 14",
+            "final fantasy xiv",
+        ],
+    },
+},
+
+{
+    "#url"     : "https://www.allthefallen.moe/forum/index.php?forums/announcements.16/",
+    "#category": ("xenforo", "atfforum", "forum"),
+    "#class"   : xenforo.XenforoForumExtractor,
+    "#pattern" : xenforo.XenforoThreadExtractor.pattern,
+    "#auth"    : True,
+    "#count"   : range(100, 200),
+},
+
+)
--- a/test/results/simpcity.py
+++ b/test/results/simpcity.py
@@ -82,10 +82,7 @@ __tests__ = (
    "#category": ("xenforo", "simpcity", "post"),
    "#class"   : xenforo.XenforoPostExtractor,
    "#auth"    : True,
-    "#results" : (
-        "/goto/post?id=13358068",
-        "https://cyberdrop.cr/a/Sh9GlG38",
-    ),
+    "#results" : "https://cyberdrop.cr/a/Sh9GlG38",
 },

 {
@@ -222,7 +219,7 @@ __tests__ = (
    "#category": ("xenforo", "simpcity", "thread"),
    "#class"   : xenforo.XenforoThreadExtractor,
    "#auth"    : True,
-    "#pattern" : r"https://(jpg6\.su/img/\w+|bunkr\.\w+/[fiv]/\w+|pixeldrain.com/l/\w+|alua.com/tatakai)|/goto/post",
+    "#pattern" : r"https://(jpg6\.su/img/\w+|bunkr\.\w+/[fiv]/\w+|pixeldrain.com/l/\w+|alua.com/tatakai)|saint2.cr/embed",
    "#count"   : range(100, 300),

    "count" : int,