diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index 3f45716d1b1..eeed36dda0e 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -654,15 +654,14 @@ def get_src_dir(self): def download_url_to_file_with_progress(url: str, file_name) -> None: - print("Download %s -> %s ..." % (url, file_name)) + print("Download with %s -> %s ..." % (url, file_name)) class Progress(object): last_report = 0 - def progress(self, count, block, total): + def write_update(self, total, amount): if total == -1: total = "(Unknown)" - amount = count * block if sys.stdout.isatty(): sys.stdout.write("\r downloading %s of %s " % (amount, total)) @@ -675,10 +674,33 @@ def progress(self, count, block, total): self.last_report = now sys.stdout.flush() + def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount): + self.write_update(total, amount) + + def progress_urllib(self, count, block, total): + amount = count * block + self.write_update(total, amount) + progress = Progress() start = time.time() try: - (_filename, headers) = urlretrieve(url, file_name, reporthook=progress.progress) + if os.environ.get("GETDEPS_USE_LIBCURL") is not None: + import pycurl + + with open(file_name, "wb") as f: + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(pycurl.WRITEDATA, f) + # display progress + c.setopt(pycurl.NOPROGRESS, False) + c.setopt(pycurl.XFERINFOFUNCTION, progress.progress_pycurl) + c.perform() + c.close() + headers = None + else: + (_filename, headers) = urlretrieve( + url, file_name, reporthook=progress.progress_urllib + ) except (OSError, IOError) as exc: # noqa: B014 raise TransientFailure( "Failed to download %s to %s: %s" % (url, file_name, str(exc)) @@ -687,7 +709,8 @@ def progress(self, count, block, total): end = time.time() sys.stdout.write(" [Complete in %f seconds]\n" % (end - start)) sys.stdout.flush() - print(f"{headers}") + if headers is not None: + print(f"{headers}") class ArchiveFetcher(Fetcher):