[xenforo] support 'allthefallen.moe/forum' (#3249 #8268)

gallery_dl/extractor/xenforo.py

@@ -16,8 +16,6 @@ from ..cache import cache
 class XenforoExtractor(BaseExtractor):
     """Base class for xenforo extractors"""
     basecategory = "xenforo"
-    #  cookies_domain = "simpcity.cr"
-    cookies_names = ("ogaddgmetaprof_user",)
     directory_fmt = ("{category}", "{thread[section]}",
                      "{thread[title]} ({thread[id]})")
     filename_fmt = "{post[id]}_{num:>02}_{id}_{filename}.{extension}"
@@ -35,9 +33,9 @@ class XenforoExtractor(BaseExtractor):
             r'(?s)(?:'
             r'<video (.*?\ssrc="[^"]+".*?)</video>'
             r'|<a [^>]*?href="[^"]*?'
-            r'(/attachments/[^"]+".*?)</a>'
+            r'(/(?:index\.php\?)?attachments/[^"]+".*?)</a>'
             r'|<div [^>]*?data-src="[^"]*?'
-            r'(/attachments/[^"]+".*?)/>'
+            r'(/(?:index\.php\?)attachments/[^"]+".*?)/>'
             r'|(?:<a [^>]*?href="|<iframe [^>]*?src="|'
             r'''onclick="loadMedia\(this, ')([^"']+)'''
             r')'
@@ -61,8 +59,11 @@ class XenforoExtractor(BaseExtractor):
                     data["num"] += 1
                     data["num_external"] += 1
                     data["type"] = "external"
-                    if ext.startswith("//"):
-                        ext = "https:" + ext
+                    if ext[0] == "/":
+                        if ext[1] == "/":
+                            ext = "https:" + ext
+                        else:
+                            continue
                     yield Message.Queue, ext, data
 
                 elif video:
@@ -163,6 +164,8 @@ class XenforoExtractor(BaseExtractor):
 
         url = f"{base}/page-{'9999' if pnum is None else pnum}"
         with self.request_page(url) as response:
+            if pnum is None and not response.history:
+                self._require_auth()
             url = response.url
             if url[-1] == "/":
                 pnum = 1
@@ -247,10 +250,10 @@ class XenforoExtractor(BaseExtractor):
 
         return post
 
-    def _require_auth(self, response):
+    def _require_auth(self, response=None):
         raise exception.AuthRequired(
             ("username & password", "authenticated cookies"), None,
-            self._extract_error(response.text))
+            None if response is None else self._extract_error(response.text))
 
     def _validate(self, response):
         if response.status_code == 403 and b">Log in<" in response.content:
@@ -269,17 +272,24 @@ BASE_PATTERN = XenforoExtractor.update({
         "pattern": r"(?:www\.)?nudostar\.com/forum",
         "cookies": ("xf_user",),
     },
+    "atfforum": {
+        "root": "https://allthefallen.moe/forum",
+        "pattern": r"(?:www\.)?allthefallen\.moe/forum",
+        "cookies": ("xf_user",),
+    },
 })
 
 
 class XenforoPostExtractor(XenforoExtractor):
     subcategory = "post"
-    pattern = rf"{BASE_PATTERN}/(?:threads/[^/?#]+/post-|posts/)(\d+)"
+    pattern = (rf"{BASE_PATTERN}(/(?:index\.php\?)?threads"
+               rf"/[^/?#]+/post-|/posts/)(\d+)")
     example = "https://simpcity.cr/threads/TITLE.12345/post-54321"
 
     def posts(self):
+        path = self.groups[-2]
         post_id = self.groups[-1]
-        url = f"{self.root}/posts/{post_id}/"
+        url = f"{self.root}{path}{post_id}/"
         page = self.request_page(url).text
 
         pos = page.find(f'data-content="post-{post_id}"')
@@ -293,7 +303,8 @@ class XenforoPostExtractor(XenforoExtractor):
 
 class XenforoThreadExtractor(XenforoExtractor):
     subcategory = "thread"
-    pattern = rf"{BASE_PATTERN}(/threads/(?:[^/?#]+\.)?\d+)(?:/page-(\d+))?"
+    pattern = (rf"{BASE_PATTERN}(/(?:index\.php\?)?threads"
+               rf"/(?:[^/?#]+\.)?\d+)(?:/page-(\d+))?")
     example = "https://simpcity.cr/threads/TITLE.12345/"
 
     def posts(self):
@@ -321,12 +332,13 @@ class XenforoThreadExtractor(XenforoExtractor):
 
 class XenforoForumExtractor(XenforoExtractor):
     subcategory = "forum"
-    pattern = rf"{BASE_PATTERN}(/forums/(?:[^/?#]+\.)?[^/?#]+)(?:/page-(\d+))?"
+    pattern = (rf"{BASE_PATTERN}(/(?:index\.php\?)?forums"
+               rf"/(?:[^/?#]+\.)?[^/?#]+)(?:/page-(\d+))?")
     example = "https://simpcity.cr/forums/TITLE.123/"
 
     def items(self):
         extract_threads = text.re(
-            r'(/threads/[^"]+)"[^>]+data-xf-init=').findall
+            r'(/(?:index\.php\?)?threads/[^"]+)"[^>]+data-xf-init=').findall
 
         data = {"_extractor": XenforoThreadExtractor}
         path = self.groups[-2]