4a348990f4
This change introduces 'extractor.*.retries/timeout/verify' options as a general way to set these values for all HTTP requests. 'downloader.http.retries/timeout/verify' is a way to override these options for file downloads only and will fall back to 'extractor.*.…* values if they haven't been explicitly set. Also: downloader classes now take an extractor object as first argument instead of a requests.session.
35 lines
828 B
Python
35 lines
828 B
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2014-2018 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Downloader module for text: URLs"""
|
|
|
|
from .common import DownloaderBase
|
|
|
|
|
|
class Downloader(DownloaderBase):
|
|
scheme = "text"
|
|
|
|
def __init__(self, extractor, output):
|
|
DownloaderBase.__init__(self, extractor, output)
|
|
self.content = b""
|
|
|
|
def connect(self, url, offset):
|
|
data = url.encode()
|
|
self.content = data[offset + 5:]
|
|
return offset, len(data) - 5
|
|
|
|
def receive(self, file):
|
|
file.write(self.content)
|
|
|
|
def reset(self):
|
|
self.content = b""
|
|
|
|
@staticmethod
|
|
def get_extension():
|
|
return "txt"
|