add 'extractor.*.user-agent' config option

This commit is contained in:
Mike Fährmann
2017-11-15 13:54:40 +01:00
parent 6913eeaa40
commit e6814aebe2
9 changed files with 26 additions and 15 deletions

View File

@@ -37,10 +37,6 @@ class TwitterTweetExtractor(Extractor):
self.path, self.user, self.tid = match.groups()
def items(self):
self.session.headers["User-Agent"] = (
"Mozilla/5.0 (X11; Linux x86_64; rv:48.0) "
"Gecko/20100101 Firefox/48.0"
)
page = self.request("https://twitter.com/" + self.path).text
data = self.get_job_metadata()
imgs = self.get_image_urls(page)