Skip to content

Commit

Permalink
[downloader:http] fix issues from inaccurate 'time.sleep()'
Browse files Browse the repository at this point in the history
(#3143)

Reverts part of c59b98c by going back to using a global timer
instead of a per-chunk one.

Reintroduces the issue of ignoring rate limits after
suspending and resuming the process.
  • Loading branch information
mikf committed Nov 10, 2022
1 parent 8d543b0 commit a4ff20c
Showing 1 changed file with 17 additions and 21 deletions.
38 changes: 17 additions & 21 deletions gallery_dl/downloader/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,42 +274,38 @@ def _download_impl(self, url, pathfmt):
return True

@staticmethod
def receive(fp, content, bytes_total, bytes_downloaded):
def receive(fp, content, bytes_total, bytes_start):
write = fp.write
for data in content:
write(data)

def _receive_rate(self, fp, content, bytes_total, bytes_downloaded):
def _receive_rate(self, fp, content, bytes_total, bytes_start):
rate = self.rate
progress = self.progress
bytes_start = bytes_downloaded
write = fp.write
t1 = tstart = time.time()
progress = self.progress

bytes_downloaded = 0
time_start = time.time()

for data in content:
write(data)
time_current = time.time()
time_elapsed = time_current - time_start
bytes_downloaded += len(data)

t2 = time.time() # current time
elapsed = t2 - t1 # elapsed time
num_bytes = len(data)
write(data)

if progress is not None:
bytes_downloaded += num_bytes
tdiff = t2 - tstart
if tdiff >= progress:
if time_elapsed >= progress:
self.out.progress(
bytes_total, bytes_downloaded,
int((bytes_downloaded - bytes_start) / tdiff),
bytes_total,
bytes_start + bytes_downloaded,
int(bytes_downloaded / time_elapsed),
)

if rate:
expected = num_bytes / rate # expected elapsed time
if elapsed < expected:
# sleep if less time elapsed than expected
time.sleep(expected - elapsed)
t2 = time.time()

t1 = t2
time_expected = bytes_downloaded / rate
if time_expected > time_elapsed:
time.sleep(time_expected - time_elapsed)

def _find_extension(self, response):
"""Get filename extension from MIME type"""
Expand Down

0 comments on commit a4ff20c

Please sign in to comment.