From 3b5ba970cff8f5f8d405a099ea56d2010336abf1 Mon Sep 17 00:00:00 2001
From: David Greenberg <dgrnbrg@fb.com>
Date: Mon, 14 Mar 2022 09:19:39 -0700
Subject: [PATCH] use libcurl in getdeps (#76)

Summary:
Pull Request resolved: https://github.com/facebookincubator/fizz/pull/76

X-link: https://github.com/facebook/proxygen/pull/402

X-link: https://github.com/facebook/folly/pull/1735

X-link: https://github.com/facebook/bistro/pull/60

X-link: https://github.com/facebook/watchman/pull/1012

X-link: https://github.com/facebook/fbthrift/pull/487

X-link: https://github.com/facebook/fboss/pull/114

X-link: https://github.com/facebook/fb303/pull/27

When using getdeps inside of a container, Python's urllib isn't able to download from dewey lfs (see this post for details https://fb.workplace.com/groups/systemd.and.friends/permalink/2747692278870647/).

This allows for getdeps to use `libcurl` to fetch dependencies, which allows for a getdeps build to work inside the container environment.

Reviewed By: mackorone

Differential Revision: D34696330

fbshipit-source-id: 06cae87eef40dfa3cecacacee49234b6737d546f
---
 build/fbcode_builder/getdeps/fetcher.py | 33 +++++++++++++++++++++----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py
index 3f45716d1b1..eeed36dda0e 100644
--- a/build/fbcode_builder/getdeps/fetcher.py
+++ b/build/fbcode_builder/getdeps/fetcher.py
@@ -654,15 +654,14 @@ def get_src_dir(self):
 
 
 def download_url_to_file_with_progress(url: str, file_name) -> None:
-    print("Download %s -> %s ..." % (url, file_name))
+    print("Download with %s -> %s ..." % (url, file_name))
 
     class Progress(object):
         last_report = 0
 
-        def progress(self, count, block, total):
+        def write_update(self, total, amount):
             if total == -1:
                 total = "(Unknown)"
-            amount = count * block
 
             if sys.stdout.isatty():
                 sys.stdout.write("\r downloading %s of %s " % (amount, total))
@@ -675,10 +674,33 @@ def progress(self, count, block, total):
                     self.last_report = now
             sys.stdout.flush()
 
+        def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount):
+            self.write_update(total, amount)
+
+        def progress_urllib(self, count, block, total):
+            amount = count * block
+            self.write_update(total, amount)
+
     progress = Progress()
     start = time.time()
     try:
-        (_filename, headers) = urlretrieve(url, file_name, reporthook=progress.progress)
+        if os.environ.get("GETDEPS_USE_LIBCURL") is not None:
+            import pycurl
+
+            with open(file_name, "wb") as f:
+                c = pycurl.Curl()
+                c.setopt(pycurl.URL, url)
+                c.setopt(pycurl.WRITEDATA, f)
+                # display progress
+                c.setopt(pycurl.NOPROGRESS, False)
+                c.setopt(pycurl.XFERINFOFUNCTION, progress.progress_pycurl)
+                c.perform()
+                c.close()
+            headers = None
+        else:
+            (_filename, headers) = urlretrieve(
+                url, file_name, reporthook=progress.progress_urllib
+            )
     except (OSError, IOError) as exc:  # noqa: B014
         raise TransientFailure(
             "Failed to download %s to %s: %s" % (url, file_name, str(exc))
@@ -687,7 +709,8 @@ def progress(self, count, block, total):
     end = time.time()
     sys.stdout.write(" [Complete in %f seconds]\n" % (end - start))
     sys.stdout.flush()
-    print(f"{headers}")
+    if headers is not None:
+        print(f"{headers}")
 
 
 class ArchiveFetcher(Fetcher):