Build puller/loader from source for repository rules

.gitignore

@@ -13,9 +13,8 @@
 *.dylib
 *.dll
 
-# Fortran module files
-*.mod
-*.smod
+# Vendored modules
+vendor/**/go.mod
 
 # Python 2 bytecode files
 *.pyc

container/BUILD

@@ -98,6 +98,7 @@ bzl_library(
         ":layer",
         ":layer_tools",
         ":providers",
+        "//internal:execution_bzl",
         "//skylib:filetype",
         "//skylib:hash",
         "//skylib:label",

container/load.bzl

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 "container_load rule"
+
+load("//internal:execution.bzl", "env_execute", "executable_extension")
+
 _DOC = """A repository rule that examines the contents of a docker save tarball and creates a container_import target.
 
 This extracts the tarball amd creates a filegroup of the untarred objects in OCI intermediate layout.
@@ -34,23 +37,28 @@ container_import(
     layers = glob(["*.tar.gz"]),
 )""")
 
-    loader = repository_ctx.attr._loader_linux_amd64
-    if repository_ctx.os.name.lower().startswith("mac os"):
-        loader = repository_ctx.attr._loader_darwin
-    elif repository_ctx.os.name.lower().startswith("linux"):
-        arch = repository_ctx.execute(["uname", "-m"]).stdout.strip()
-        if arch == "arm64" or arch == "aarch64":
-            loader = repository_ctx.attr._loader_linux_arm64
-        elif arch == "s390x":
-            loader = repository_ctx.attr._loader_linux_s390x
-
-    result = repository_ctx.execute([
-        repository_ctx.path(loader),
+    loader_args = [
         "-directory",
         repository_ctx.path("image"),
         "-tarball",
         repository_ctx.path(repository_ctx.attr.file),
-    ])
+    ]
+
+    if repository_ctx.attr.use_precompiled_binaries:
+        loader = repository_ctx.attr._loader_linux_amd64
+        if repository_ctx.os.name.lower().startswith("mac os"):
+            loader = repository_ctx.attr._loader_darwin
+        elif repository_ctx.os.name.lower().startswith("linux"):
+            arch = repository_ctx.execute(["uname", "-m"]).stdout.strip()
+            if arch == "arm64" or arch == "aarch64":
+                loader = repository_ctx.attr._loader_linux_arm64
+            elif arch == "s390x":
+                loader = repository_ctx.attr._loader_linux_s390x
+        loader_path = repository_ctx.path(loader)
+        result = repository_ctx.execute([loader_path] + loader_args)
+    else:
+        loader_path = str(repository_ctx.path(Label("@rules_docker_repository_tools//:bin/loader{}".format(executable_extension(repository_ctx)))))
+        result = env_execute(repository_ctx, [loader_path] + loader_args)
 
     if result.return_code:
         fail("Importing from tarball failed (status %s): %s" % (result.return_code, result.stderr))
@@ -84,6 +92,12 @@ container_load = repository_rule(
             default = Label("@loader_linux_s390x//file:downloaded"),
             cfg = "exec",
         ),
+        "use_precompiled_binaries": attr.bool(
+            doc = """Whether to use precompiled binaries.
+            If true, the loader will be fetched from a prebuilt binary (legacy).
+            If false, loader will be built from source.""",
+            default = True,
+        ),
     },
     implementation = _impl,
 )

container/pull.bzl

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 "container_pull rule"
+
+load("//internal:execution.bzl", "env_execute", "executable_extension")
+
 _DOC = """A repository rule that pulls down a Docker base image in a manner suitable for use with the `base` attribute of `container_image`.
 
 This is based on google/containerregistry using google/go-containerregistry.
@@ -135,6 +138,12 @@ _container_pull_attrs = {
 
         This attribute will be overridden by the PULLER_TIMEOUT environment variable, if it is set.""",
     ),
+    "use_precompiled_binaries": attr.bool(
+        doc = """Whether to use precompiled binaries.
+        If true, the loader will be fetched from a prebuilt binary (legacy).
+        If false, loader will be built from source.""",
+        default = True,
+    ),
 }
 
 def _impl(repository_ctx):
@@ -148,18 +157,22 @@ def _impl(repository_ctx):
 
     import_rule_tags = "[\"{}\"]".format("\", \"".join(repository_ctx.attr.import_tags))
 
-    puller = repository_ctx.attr.puller_linux_amd64
-    if repository_ctx.os.name.lower().startswith("mac os"):
-        puller = repository_ctx.attr.puller_darwin
-    elif repository_ctx.os.name.lower().startswith("linux"):
-        arch = repository_ctx.execute(["uname", "-m"]).stdout.strip()
-        if arch == "arm64" or arch == "aarch64":
-            puller = repository_ctx.attr.puller_linux_arm64
-        elif arch == "s390x":
-            puller = repository_ctx.attr.puller_linux_s390x
+    if repository_ctx.attr.use_precompiled_binaries:
+        puller = repository_ctx.attr.puller_linux_amd64
+        if repository_ctx.os.name.lower().startswith("mac os"):
+            puller = repository_ctx.attr.puller_darwin
+        elif repository_ctx.os.name.lower().startswith("linux"):
+            arch = repository_ctx.execute(["uname", "-m"]).stdout.strip()
+            if arch == "arm64" or arch == "aarch64":
+                puller = repository_ctx.attr.puller_linux_arm64
+            elif arch == "s390x":
+                puller = repository_ctx.attr.puller_linux_s390x
+        puller_path = repository_ctx.path(puller)
+    else:
+        puller_path = str(repository_ctx.path(Label("@rules_docker_repository_tools//:bin/puller{}".format(executable_extension(repository_ctx)))))
 
     args = [
-        repository_ctx.path(puller),
+        puller_path,
         "-directory",
         repository_ctx.path("image"),
         "-os",
@@ -235,7 +248,26 @@ def _impl(repository_ctx):
             ),
         }
 
-    result = repository_ctx.execute(args, **kwargs)
+    # Only allow environment variables that influence the pusher through.
+    env = {
+        k: v
+        for k, v in repository_ctx.os.environ.items()
+        if k.lower() in (
+            "home",
+            "path",  # TODO(user): confirm if this is necessary as it can bust cache.
+            "ssh_auth_sock",
+            "ssl_cert_file",
+            "ssl_cert_dir",
+            "http_proxy",
+            "https_proxy",
+            "no_proxy",
+        )
+    }
+
+    if repository_ctx.attr.use_precompiled_binaries:
+        result = repository_ctx.execute(args, **kwargs)
+    else:
+        result = env_execute(repository_ctx, args, environment = env, **kwargs)
     if result.return_code:
         fail("Pull command failed: %s (%s)" % (result.stderr, " ".join([str(a) for a in args])))
 

container/push.bzl

@@ -77,6 +77,7 @@ def _impl(ctx):
     digester_args += digester_img_args
     tarball = image.get("legacy")
     if tarball:
+        # buildifier: disable=print
         print("Pushing an image based on a tarball can be very " +
               "expensive. If the image set on %s is the output of a " % ctx.label +
               "docker_build, consider dropping the '.tar' extension. " +

docs/container.md

@@ -161,8 +161,7 @@ The created target can be referenced as `@label_name//image`.
 
 <pre>
 container_pull(<a href="#container_pull-name">name</a>, <a href="#container_pull-architecture">architecture</a>, <a href="#container_pull-cpu_variant">cpu_variant</a>, <a href="#container_pull-cred_helpers">cred_helpers</a>, <a href="#container_pull-digest">digest</a>, <a href="#container_pull-docker_client_config">docker_client_config</a>,
-               <a href="#container_pull-import_tags">import_tags</a>, <a href="#container_pull-os">os</a>, <a href="#container_pull-os_features">os_features</a>, <a href="#container_pull-os_version">os_version</a>, <a href="#container_pull-platform_features">platform_features</a>, <a href="#container_pull-puller_darwin">puller_darwin</a>,
-               <a href="#container_pull-puller_linux_amd64">puller_linux_amd64</a>, <a href="#container_pull-puller_linux_arm64">puller_linux_arm64</a>, <a href="#container_pull-puller_linux_s390x">puller_linux_s390x</a>, <a href="#container_pull-registry">registry</a>, <a href="#container_pull-repo_mapping">repo_mapping</a>,
+               <a href="#container_pull-import_tags">import_tags</a>, <a href="#container_pull-os">os</a>, <a href="#container_pull-os_features">os_features</a>, <a href="#container_pull-os_version">os_version</a>, <a href="#container_pull-platform_features">platform_features</a>, <a href="#container_pull-registry">registry</a>, <a href="#container_pull-repo_mapping">repo_mapping</a>,
                <a href="#container_pull-repository">repository</a>, <a href="#container_pull-tag">tag</a>, <a href="#container_pull-timeout">timeout</a>)
 </pre>
 
@@ -204,10 +203,6 @@ please use the bazel startup flag `--loading_phase_threads=1` in your bazel invo
 | <a id="container_pull-os_features"></a>os_features |  Specifies os features when pulling a multi-platform manifest list.   | List of strings | optional | [] |
 | <a id="container_pull-os_version"></a>os_version |  Which os version to pull if this image refers to a multi-platform manifest list.   | String | optional | "" |
 | <a id="container_pull-platform_features"></a>platform_features |  Specifies platform features when pulling a multi-platform manifest list.   | List of strings | optional | [] |
-| <a id="container_pull-puller_darwin"></a>puller_darwin |  Exposed to provide a way to test other pullers on macOS   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | @go_puller_darwin//file:downloaded |
-| <a id="container_pull-puller_linux_amd64"></a>puller_linux_amd64 |  Exposed to provide a way to test other pullers on Linux   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | @go_puller_linux_amd64//file:downloaded |
-| <a id="container_pull-puller_linux_arm64"></a>puller_linux_arm64 |  Exposed to provide a way to test other pullers on Linux   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | @go_puller_linux_arm64//file:downloaded |
-| <a id="container_pull-puller_linux_s390x"></a>puller_linux_s390x |  Exposed to provide a way to test other pullers on Linux   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | @go_puller_linux_s390x//file:downloaded |
 | <a id="container_pull-registry"></a>registry |  The registry from which we are pulling.   | String | required |  |
 | <a id="container_pull-repo_mapping"></a>repo_mapping |  A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.&lt;p&gt;For example, an entry <code>"@foo": "@bar"</code> declares that, for any time this repository depends on <code>@foo</code> (such as a dependency on <code>@foo//some:target</code>, it should actually resolve that dependency within globally-declared <code>@bar</code> (<code>@bar//some:target</code>).   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | required |  |
 | <a id="container_pull-repository"></a>repository |  The name of the image.   | String | required |  |

go.mod

@@ -0,0 +1,13 @@
+module github.com/bazelbuild/rules_docker
+
+go 1.15
+
+// IMPORTANT: Keep in versions in sync with repositories/go_repositories.bzl
+// until we add a gazelle update-repos rule.
+require (
+	github.com/ghodss/yaml v1.0.0
+	github.com/google/go-containerregistry v0.5.1
+	github.com/kylelemons/godebug v1.1.0
+	github.com/pkg/errors v0.9.1
+	gopkg.in/yaml.v2 v2.3.0
+)