[common] add 'request_location()' convenience function

gallery_dl/extractor/common.py

@@ -240,6 +240,11 @@ class Extractor():
 
         raise exception.HttpError(msg, response)
 
+    def request_location(self, url, **kwargs):
+        kwargs.setdefault("method", "HEAD")
+        kwargs.setdefault("allow_redirects", False)
+        return self.request(url, **kwargs).headers.get("location", "")
+
     _handle_429 = util.false
 
     def wait(self, seconds=None, until=None, adjust=1.0,

gallery_dl/extractor/fanbox.py

@@ -296,8 +296,7 @@ class FanboxExtractor(Extractor):
             url = "https://www.pixiv.net/fanbox/"+content_id
             # resolve redirect
             try:
-                url = self.request(url, method="HEAD",
+                url = self.request_location(url)
-                                   allow_redirects=False).headers["location"]
             except Exception as exc:
                 url = None
                 self.log.warning("Unable to extract fanbox embed %s (%s: %s)",
@@ -392,13 +391,7 @@ class FanboxRedirectExtractor(Extractor):
     pattern = r"(?:https?://)?(?:www\.)?pixiv\.net/fanbox/creator/(\d+)"
     example = "https://www.pixiv.net/fanbox/creator/12345"
 
-    def __init__(self, match):
-        Extractor.__init__(self, match)
-        self.user_id = match.group(1)
-
     def items(self):
-        url = "https://www.pixiv.net/fanbox/creator/" + self.user_id
+        url = "https://www.pixiv.net/fanbox/creator/" + self.groups[0]
-        data = {"_extractor": FanboxCreatorExtractor}
+        location = self.request_location(url, notfound="user")
-        response = self.request(
+        yield Message.Queue, location, {"_extractor": FanboxCreatorExtractor}
-            url, method="HEAD", allow_redirects=False, notfound="user")
-        yield Message.Queue, response.headers["Location"], data

gallery_dl/extractor/patreon.py

@@ -109,11 +109,7 @@ class PatreonExtractor(Extractor):
 
     def _attachments(self, post):
         for attachment in post.get("attachments") or ():
-            url = self.request(
+            url = self.request_location(attachment["url"], fatal=False)
-                attachment["url"], method="HEAD",
-                allow_redirects=False, fatal=False,
-            ).headers.get("Location")
-
             if url:
                 yield "attachment", url, attachment["name"]
 

gallery_dl/extractor/pinterest.py

@@ -380,15 +380,10 @@ class PinterestPinitExtractor(PinterestExtractor):
     pattern = r"(?:https?://)?pin\.it/([^/?#]+)"
     example = "https://pin.it/abcde"
 
-    def __init__(self, match):
-        PinterestExtractor.__init__(self, match)
-        self.shortened_id = match.group(1)
-
     def items(self):
         url = "https://api.pinterest.com/url_shortener/{}/redirect/".format(
-            self.shortened_id)
+            self.groups[0])
-        response = self.request(url, method="HEAD", allow_redirects=False)
+        location = self.request_location(url)
-        location = response.headers.get("Location")
         if not location or not PinterestPinExtractor.pattern.match(location):
             raise exception.NotFoundError("pin")
         yield Message.Queue, location, {"_extractor": PinterestPinExtractor}

gallery_dl/extractor/pixiv.py

@@ -516,16 +516,10 @@ class PixivMeExtractor(PixivExtractor):
     pattern = r"(?:https?://)?pixiv\.me/([^/?#]+)"
     example = "https://pixiv.me/USER"
 
-    def __init__(self, match):
-        PixivExtractor.__init__(self, match)
-        self.account = match.group(1)
-
     def items(self):
-        url = "https://pixiv.me/" + self.account
+        url = "https://pixiv.me/" + self.groups[0]
-        data = {"_extractor": PixivUserExtractor}
+        location = self.request_location(url, notfound="user")
-        response = self.request(
+        yield Message.Queue, location, {"_extractor": PixivUserExtractor}
-            url, method="HEAD", allow_redirects=False, notfound="user")
-        yield Message.Queue, response.headers["Location"], data
 
 
 class PixivWorkExtractor(PixivExtractor):

gallery_dl/extractor/postmill.py

@@ -153,17 +153,13 @@ class PostmillPostExtractor(PostmillExtractor):
 class PostmillShortURLExtractor(PostmillExtractor):
     """Extractor for short submission URLs"""
     subcategory = "shorturl"
-    pattern = BASE_PATTERN + r"/(\d+)$"
+    pattern = BASE_PATTERN + r"(/\d+)$"
     example = "https://raddle.me/123"
 
-    def __init__(self, match):
-        PostmillExtractor.__init__(self, match)
-        self.post_id = match.group(3)
-
     def items(self):
-        url = self.root + "/" + self.post_id
+        url = self.root + self.groups[2]
-        response = self.request(url, method="HEAD", allow_redirects=False)
+        location = self.request_location(url)
-        full_url = text.urljoin(url, response.headers["Location"])
+        full_url = text.urljoin(url, location)
         yield Message.Queue, full_url, {"_extractor": PostmillPostExtractor}
 
 

gallery_dl/extractor/reddit.py

@@ -357,10 +357,9 @@ class RedditRedirectExtractor(Extractor):
             sub_type = "user"
         url = "https://www.reddit.com/{}/{}/s/{}".format(
             sub_type, subreddit, share_url)
+        location = self.request_location(url, notfound="submission")
         data = {"_extractor": RedditSubmissionExtractor}
-        response = self.request(url, method="HEAD", allow_redirects=False,
+        yield Message.Queue, location, data
-                                notfound="submission")
-        yield Message.Queue, response.headers["Location"], data
 
 
 class RedditAPI():

gallery_dl/extractor/seiga.py

@@ -43,9 +43,7 @@ class SeigaExtractor(Extractor):
     def get_image_url(self, image_id):
         """Get url for an image with id 'image_id'"""
         url = "{}/image/source/{}".format(self.root, image_id)
-        response = self.request(
+        location = self.request_location(url, notfound="image")
-            url, method="HEAD", allow_redirects=False, notfound="image")
-        location = response.headers["location"]
         if "nicovideo.jp/login" in location:
             raise exception.StopExtraction(
                 "HTTP redirect to login page (%s)", location.partition("?")[0])

gallery_dl/extractor/tiktok.py

@@ -183,10 +183,7 @@ class TiktokVmpostExtractor(TiktokExtractor):
         url = text.ensure_http_scheme(self.url)
         headers = {"User-Agent": "facebookexternalhit/1.1"}
 
-        response = self.request(url, headers=headers, method="HEAD",
+        url = self.request_location(url, headers=headers, notfound="post")
-                                allow_redirects=False, notfound="post")
-
-        url = response.headers.get("Location")
         if not url or len(url) <= 28:
             # https://www.tiktok.com/?_r=1
             raise exception.NotFoundError("post")

gallery_dl/extractor/urlshortener.py

@@ -32,21 +32,13 @@ BASE_PATTERN = UrlshortenerExtractor.update({
 class UrlshortenerLinkExtractor(UrlshortenerExtractor):
     """Extractor for general-purpose URL shorteners"""
     subcategory = "link"
-    pattern = BASE_PATTERN + r"/([^/?#]+)"
+    pattern = BASE_PATTERN + r"(/[^/?#]+)"
     example = "https://bit.ly/abcde"
 
-    def __init__(self, match):
-        UrlshortenerExtractor.__init__(self, match)
-        self.id = match.group(match.lastindex)
-
-    def _init(self):
-        self.headers = self.config_instance("headers")
-
     def items(self):
-        response = self.request(
+        url = self.root + self.groups[-1]
-            "{}/{}".format(self.root, self.id), headers=self.headers,
+        location = self.request_location(
-            method="HEAD", allow_redirects=False, notfound="URL")
+            url, headers=self.config_instance("headers"), notfound="URL")
-        try:
+        if not location:
-            yield Message.Queue, response.headers["location"], {}
-        except KeyError:
             raise exception.StopExtraction("Unable to resolve short URL")
+        yield Message.Queue, location, {}