[urlshortener] force HTTPS
This commit is contained in:
@@ -28,7 +28,7 @@ class UrlshortenerExtractor(BaseExtractor):
|
|||||||
def __init__(self, match):
|
def __init__(self, match):
|
||||||
BaseExtractor.__init__(self, match)
|
BaseExtractor.__init__(self, match)
|
||||||
self.headers = INSTANCES[self.category].get("headers")
|
self.headers = INSTANCES[self.category].get("headers")
|
||||||
self.url = match.group()
|
self.id = match.group(match.lastindex)
|
||||||
|
|
||||||
def request(self, url, **kwargs):
|
def request(self, url, **kwargs):
|
||||||
kwargs["headers"] = self.headers
|
kwargs["headers"] = self.headers
|
||||||
@@ -36,7 +36,8 @@ class UrlshortenerExtractor(BaseExtractor):
|
|||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
response = self.request(
|
response = self.request(
|
||||||
self.url, method="HEAD", allow_redirects=False, notfound="URL")
|
"{}/{}".format(self.root, self.id), method="HEAD",
|
||||||
|
allow_redirects=False, notfound="URL")
|
||||||
if "location" not in response.headers:
|
if "location" not in response.headers:
|
||||||
raise exception.StopExtraction("Unable to resolve short URL")
|
raise exception.StopExtraction("Unable to resolve short URL")
|
||||||
yield Message.Queue, response.headers["location"], {}
|
yield Message.Queue, response.headers["location"], {}
|
||||||
@@ -56,4 +57,4 @@ INSTANCES = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
UrlshortenerExtractor.pattern = \
|
UrlshortenerExtractor.pattern = \
|
||||||
UrlshortenerExtractor.update(INSTANCES) + r"/[^/?#&]+"
|
UrlshortenerExtractor.update(INSTANCES) + r"/([^/?#&]+)"
|
||||||
|
|||||||
Reference in New Issue
Block a user