[downloader:http] fix issues from inaccurate 'time.sleep()'
(#3143)
Reverts part of c59b98c8 by going back to using a global timer
instead of a per-chunk one.
Reintroduces the issue of ignoring rate limits after
suspending and resuming the process.
This commit is contained in:
@@ -274,42 +274,38 @@ class HttpDownloader(DownloaderBase):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def receive(fp, content, bytes_total, bytes_downloaded):
|
def receive(fp, content, bytes_total, bytes_start):
|
||||||
write = fp.write
|
write = fp.write
|
||||||
for data in content:
|
for data in content:
|
||||||
write(data)
|
write(data)
|
||||||
|
|
||||||
def _receive_rate(self, fp, content, bytes_total, bytes_downloaded):
|
def _receive_rate(self, fp, content, bytes_total, bytes_start):
|
||||||
rate = self.rate
|
rate = self.rate
|
||||||
progress = self.progress
|
|
||||||
bytes_start = bytes_downloaded
|
|
||||||
write = fp.write
|
write = fp.write
|
||||||
t1 = tstart = time.time()
|
progress = self.progress
|
||||||
|
|
||||||
|
bytes_downloaded = 0
|
||||||
|
time_start = time.time()
|
||||||
|
|
||||||
for data in content:
|
for data in content:
|
||||||
write(data)
|
time_current = time.time()
|
||||||
|
time_elapsed = time_current - time_start
|
||||||
|
bytes_downloaded += len(data)
|
||||||
|
|
||||||
t2 = time.time() # current time
|
write(data)
|
||||||
elapsed = t2 - t1 # elapsed time
|
|
||||||
num_bytes = len(data)
|
|
||||||
|
|
||||||
if progress is not None:
|
if progress is not None:
|
||||||
bytes_downloaded += num_bytes
|
if time_elapsed >= progress:
|
||||||
tdiff = t2 - tstart
|
|
||||||
if tdiff >= progress:
|
|
||||||
self.out.progress(
|
self.out.progress(
|
||||||
bytes_total, bytes_downloaded,
|
bytes_total,
|
||||||
int((bytes_downloaded - bytes_start) / tdiff),
|
bytes_start + bytes_downloaded,
|
||||||
|
int(bytes_downloaded / time_elapsed),
|
||||||
)
|
)
|
||||||
|
|
||||||
if rate:
|
if rate:
|
||||||
expected = num_bytes / rate # expected elapsed time
|
time_expected = bytes_downloaded / rate
|
||||||
if elapsed < expected:
|
if time_expected > time_elapsed:
|
||||||
# sleep if less time elapsed than expected
|
time.sleep(time_expected - time_elapsed)
|
||||||
time.sleep(expected - elapsed)
|
|
||||||
t2 = time.time()
|
|
||||||
|
|
||||||
t1 = t2
|
|
||||||
|
|
||||||
def _find_extension(self, response):
|
def _find_extension(self, response):
|
||||||
"""Get filename extension from MIME type"""
|
"""Get filename extension from MIME type"""
|
||||||
|
|||||||
Reference in New Issue
Block a user