[downloader:http] fix issues from inaccurate 'time.sleep()'

(#3143)

Reverts part of c59b98c8 by going back to using a global timer
instead of a per-chunk one.

Reintroduces the issue of ignoring rate limits after
suspending and resuming the process.
This commit is contained in:
Mike Fährmann
2022-11-10 13:09:12 +01:00
parent 8d543b040d
commit a4ff20cf16

View File

@@ -274,42 +274,38 @@ class HttpDownloader(DownloaderBase):
return True return True
@staticmethod @staticmethod
def receive(fp, content, bytes_total, bytes_downloaded): def receive(fp, content, bytes_total, bytes_start):
write = fp.write write = fp.write
for data in content: for data in content:
write(data) write(data)
def _receive_rate(self, fp, content, bytes_total, bytes_downloaded): def _receive_rate(self, fp, content, bytes_total, bytes_start):
rate = self.rate rate = self.rate
progress = self.progress
bytes_start = bytes_downloaded
write = fp.write write = fp.write
t1 = tstart = time.time() progress = self.progress
bytes_downloaded = 0
time_start = time.time()
for data in content: for data in content:
write(data) time_current = time.time()
time_elapsed = time_current - time_start
bytes_downloaded += len(data)
t2 = time.time() # current time write(data)
elapsed = t2 - t1 # elapsed time
num_bytes = len(data)
if progress is not None: if progress is not None:
bytes_downloaded += num_bytes if time_elapsed >= progress:
tdiff = t2 - tstart
if tdiff >= progress:
self.out.progress( self.out.progress(
bytes_total, bytes_downloaded, bytes_total,
int((bytes_downloaded - bytes_start) / tdiff), bytes_start + bytes_downloaded,
int(bytes_downloaded / time_elapsed),
) )
if rate: if rate:
expected = num_bytes / rate # expected elapsed time time_expected = bytes_downloaded / rate
if elapsed < expected: if time_expected > time_elapsed:
# sleep if less time elapsed than expected time.sleep(time_expected - time_elapsed)
time.sleep(expected - elapsed)
t2 = time.time()
t1 = t2
def _find_extension(self, response): def _find_extension(self, response):
"""Get filename extension from MIME type""" """Get filename extension from MIME type"""