implement decorator for cloudflare bypass

this method for enabling and caching a cloudflare bypass for a
requests.session object allows for different cache-timeouts for
different domains
This commit is contained in:
Mike Fährmann
2016-11-20 18:05:49 +01:00
parent 6e98538d36
commit 9e3788175e
3 changed files with 20 additions and 7 deletions

View File

@@ -22,6 +22,9 @@ class KissmangaExtractor(Extractor):
def __init__(self, match):
Extractor.__init__(self)
self.url = match.group(0)
self.session.headers["Referer"] = self.url_base
request = cloudflare.bypass(url_base, 24*60*60)(Extractor.request)
class KissmangaMangaExtractor(KissmangaExtractor):
@@ -33,7 +36,6 @@ class KissmangaMangaExtractor(KissmangaExtractor):
})]
def items(self):
cloudflare.bypass_ddos_protection(self.session, self.url_base)
yield Message.Version, 1
for chapter in self.get_chapters():
yield Message.Queue, self.url_base + chapter
@@ -62,7 +64,6 @@ class KissmangaChapterExtractor(KissmangaExtractor):
]
def items(self):
cloudflare.bypass_ddos_protection(self.session, self.url_base)
page = self.request(self.url).text
data = self.get_job_metadata(page)
imgs = self.get_image_urls(page)

View File

@@ -9,7 +9,7 @@
"""Extract comic-issues and entire comics from http://readcomiconline.to/"""
from .common import Extractor
from .. import text
from .. import text, cloudflare
from . import kissmanga
import re
@@ -23,6 +23,10 @@ class ReadcomiconlineExtractor(Extractor):
def __init__(self, match):
Extractor.__init__(self)
self.url = match.group(0)
self.session.headers["Referer"] = self.url_base
request = cloudflare.bypass(url_base, 30*60)(Extractor.request)
class ReadcomiconlineComicExtractor(ReadcomiconlineExtractor,