From 637ef8bd504d9b6c8ec174249fc56bcd6559127c Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 21 Mar 2023 17:41:46 +0100 Subject: [PATCH 1/3] Add nixpkgs_http_repository A more general version of nixpkgs_git_repository that is not limited to the specific archives offered by Github, but can be applied to other HTTP provided repositories in general. This is a pre-requisite for some of the tags for the new Bazel module extension designed in design/bzlmod/README.md, in particular the http and github tags. --- README.md | 1 + core/README.md | 1 + core/nixpkgs.bzl | 181 +++++++++++++++++++++++++++++++++++++++++--- docs/BUILD.bazel | 1 + nixpkgs/nixpkgs.bzl | 3 + 5 files changed, 176 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c5bc5baff..343afc938 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ See [examples](/examples/toolchains) for how to use `rules_nixpkgs` with differe ## Rules * [nixpkgs_git_repository](#nixpkgs_git_repository) +* [nixpkgs_http_repository](#nixpkgs_http_repository) * [nixpkgs_local_repository](#nixpkgs_local_repository) * [nixpkgs_package](#nixpkgs_package) * [nixpkgs_cc_configure](#nixpkgs_cc_configure) diff --git a/core/README.md b/core/README.md index d90e78f3a..ccc6294d9 100644 --- a/core/README.md +++ b/core/README.md @@ -28,6 +28,7 @@ See [examples](/examples/toolchains) for how to use `rules_nixpkgs` with differe ## Rules * [nixpkgs_git_repository](#nixpkgs_git_repository) +* [nixpkgs_http_repository](#nixpkgs_http_repository) * [nixpkgs_local_repository](#nixpkgs_local_repository) * [nixpkgs_package](#nixpkgs_package) diff --git a/core/nixpkgs.bzl b/core/nixpkgs.bzl index a678b7bc3..0a04f24a1 100644 --- a/core/nixpkgs.bzl +++ b/core/nixpkgs.bzl @@ -26,6 +26,7 @@ See [examples](/examples/toolchains) for how to use `rules_nixpkgs` with differe ## Rules * [nixpkgs_git_repository](#nixpkgs_git_repository) +* [nixpkgs_http_repository](#nixpkgs_http_repository) * [nixpkgs_local_repository](#nixpkgs_local_repository) * [nixpkgs_package](#nixpkgs_package) """ @@ -41,7 +42,27 @@ load( "is_supported_platform", ) -def _nixpkgs_git_repository_impl(repository_ctx): +def _nixpkgs_http_repository_impl(repository_ctx): + url_set = bool(repository_ctx.attr.url) + urls_set = bool(repository_ctx.attr.urls) + + if url_set and urls_set: + fail("Duplicate URL attributes, specify only one of 'url' or 'urls'.") + + if not url_set and not urls_set: + fail("Missing URL, specify one of 'url' or 'urls'.") + + if url_set: + url = repository_ctx.attr.url + else: + url = repository_ctx.attr.urls + + integrity_set = bool(repository_ctx.attr.integrity) + sha256_set = bool(repository_ctx.attr.sha256) + + if integrity_set and sha256_set: + fail("Duplicate integrity attributes, specify only one of 'integrity' or 'sha256'.") + repository_ctx.file( "BUILD", content = 'filegroup(name = "srcs", srcs = glob(["**"]), visibility = ["//visibility:public"])', @@ -51,25 +72,163 @@ def _nixpkgs_git_repository_impl(repository_ctx): # label for default.nix. repository_ctx.symlink("default.nix", repository_ctx.attr.unmangled_name) + # TODO return sha256/integrity result for reproducibility warning. repository_ctx.download_and_extract( - url = "%s/archive/%s.tar.gz" % (repository_ctx.attr.remote, repository_ctx.attr.revision), - stripPrefix = "nixpkgs-" + repository_ctx.attr.revision, + url = url, sha256 = repository_ctx.attr.sha256, + stripPrefix = repository_ctx.attr.strip_prefix, + auth = repository_ctx.attr.auth, + integrity = repository_ctx.attr.integrity, ) -_nixpkgs_git_repository = repository_rule( - implementation = _nixpkgs_git_repository_impl, +_nixpkgs_http_repository = repository_rule( + implementation = _nixpkgs_http_repository_impl, attrs = { # The workspace name as specified by the user. Needed for bzlmod # compatibility, as other means of retrieving the name only return the # mangled name, not the user defined name. "unmangled_name": attr.string(mandatory = True), - "revision": attr.string(mandatory = True), - "remote": attr.string(default = "https://github.com/NixOS/nixpkgs"), - "sha256": attr.string(doc = ""), + "url": attr.string(), + "urls": attr.string_list(), + "auth": attr.string_dict(), + "strip_prefix": attr.string(), + "integrity": attr.string(), + "sha256": attr.string(), }, ) +def nixpkgs_http_repository( + *, + name, + url = None, + urls = None, + auth = None, + strip_prefix = None, + integrity = None, + sha256 = None, + **kwargs): + """Download a Nixpkgs repository over HTTP. + + Args: + name: String + + A unique name for this repository. + + url: String + + A URL to download the repository from. + + This must be a file, http or https URL. Redirections are followed. + + More flexibility can be achieved by the urls parameter that allows + to specify alternative URLs to fetch from. + + urls: List of String + + A list of URLs to download the repository from. + + Each entry must be a file, http or https URL. Redirections are followed. + + URLs are tried in order until one succeeds, so you should list local mirrors first. + If all downloads fail, the rule will fail. + + auth: Dict of String + + An optional dict mapping host names to custom authorization patterns. + + If a URL's host name is present in this dict the value will be used as a pattern when + generating the authorization header for the http request. This enables the use of custom + authorization schemes used in a lot of common cloud storage providers. + + The pattern currently supports 2 tokens: <login> and + <password>, which are replaced with their equivalent value + in the netrc file for the same host name. After formatting, the result is set + as the value for the Authorization field of the HTTP request. + + Example attribute and netrc for a http download to an oauth2 enabled API using a bearer token: + + 
+        auth_patterns = {
+            "storage.cloudprovider.com": "Bearer <password>"
+        }
+
+ + netrc: + + 
+        machine storage.cloudprovider.com
+                password RANDOM-TOKEN
+
+ + The final HTTP request would have the following header: + + 
+        Authorization: Bearer RANDOM-TOKEN
+
+ + strip_prefix: String + + A directory prefix to strip from the extracted files. + + Many archives contain a top-level directory that contains all of the useful + files in archive. This field can be used to strip it from all of the + extracted files. + + For example, suppose you are using `nixpkgs-22.11.zip`, which contains + the directory `nixpkgs-22.11/` under which there is the `default.nix` + file and the `pkgs/` directory. Specify `strip_prefix = + "nixpkgs-22.11"` to use the `nixpkgs-22.11` directory as your top-level + directory. + + Note that if there are files outside of this directory, they will be + discarded and inaccessible (e.g., a top-level license file). This includes + files/directories that start with the prefix but are not in the directory + (e.g., `nixpkgs-22.11.release-notes`). If the specified prefix does not + match a directory in the archive, Bazel will return an error. + + integrity: String + + Expected checksum in Subresource Integrity format of the file downloaded. + + This must match the checksum of the file downloaded. _It is a security risk + to omit the checksum as remote files can change._ At best omitting this + field will make your build non-hermetic. It is optional to make development + easier but either this attribute or `sha256` should be set before shipping. + + sha256: String + The expected SHA-256 of the file downloaded. + + This must match the SHA-256 of the file downloaded. _It is a security risk + to omit the SHA-256 as remote files can change._ At best omitting this + field will make your build non-hermetic. It is optional to make development + easier but should be set before shipping. + + **kwargs: Additional arguments to forward to the underlying repository rule. + """ + if url != None: + kwargs["url"] = url + + if urls != None: + kwargs["urls"] = urls + + if auth != None: + kwargs["auth"] = auth + + if strip_prefix != None: + kwargs["strip_prefix"] = strip_prefix + + if integrity != None: + kwargs["integrity"] = integrity + + if sha256 != None: + kwargs["sha256"] = sha256 + + _nixpkgs_http_repository( + name = name, + unmangled_name = name, + **kwargs + ) + def nixpkgs_git_repository( *, name, @@ -96,12 +255,12 @@ def nixpkgs_git_repository( The SHA256 used to verify the integrity of the repository. **kwargs: Additional arguments to forward to the underlying repository rule. """ - _nixpkgs_git_repository( + _nixpkgs_http_repository( name = name, unmangled_name = name, - revision = revision, - remote = remote, + url = "https://github.com/NixOS/nixpkgs/archive/%s.tar.gz" % revision, sha256 = sha256, + strip_prefix = "nixpkgs-%s" % revision, **kwargs ) diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index fa2d6118e..90ba34e32 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -11,6 +11,7 @@ generate_documentation( input = "@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", symbol_names = [ "nixpkgs_git_repository", + "nixpkgs_http_repository", "nixpkgs_local_repository", "nixpkgs_package", "nixpkgs_cc_configure", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 794e14091..693a78a91 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -26,6 +26,7 @@ See [examples](/examples/toolchains) for how to use `rules_nixpkgs` with differe ## Rules * [nixpkgs_git_repository](#nixpkgs_git_repository) +* [nixpkgs_http_repository](#nixpkgs_http_repository) * [nixpkgs_local_repository](#nixpkgs_local_repository) * [nixpkgs_package](#nixpkgs_package) * [nixpkgs_cc_configure](#nixpkgs_cc_configure) @@ -118,6 +119,7 @@ load( load( "@rules_nixpkgs_core//:nixpkgs.bzl", _nixpkgs_git_repository = "nixpkgs_git_repository", + _nixpkgs_http_repository = "nixpkgs_http_repository", _nixpkgs_local_repository = "nixpkgs_local_repository", _nixpkgs_package = "nixpkgs_package", ) @@ -157,6 +159,7 @@ load( # aliases for backwards compatibility prior to `bzlmod` nixpkgs_git_repository = _nixpkgs_git_repository +nixpkgs_http_repository = _nixpkgs_http_repository nixpkgs_local_repository = _nixpkgs_local_repository nixpkgs_package = _nixpkgs_package nixpkgs_python_configure = _nixpkgs_python_configure From 2423bd296e0c9ab64be42ed1a71023f04df5f4a5 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 21 Mar 2023 17:55:28 +0100 Subject: [PATCH 2/3] Generate documentation --- README.md | 211 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) diff --git a/README.md b/README.md index 343afc938..f2858406a 100644 --- a/README.md +++ b/README.md @@ -616,6 +616,217 @@ Additional arguments to forward to the underlying repository rule. + + +### nixpkgs_http_repository + +
+nixpkgs_http_repository(name, url, urls, auth, strip_prefix, integrity, sha256, kwargs)
+
+ +Download a Nixpkgs repository over HTTP. + +#### Parameters + + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
name + +required. + +

+ +String + +A unique name for this repository. + +

+
url + +optional. +default is None + +

+ +String + +A URL to download the repository from. + +This must be a file, http or https URL. Redirections are followed. + +More flexibility can be achieved by the urls parameter that allows +to specify alternative URLs to fetch from. + +

+
urls + +optional. +default is None + +

+ +List of String + +A list of URLs to download the repository from. + +Each entry must be a file, http or https URL. Redirections are followed. + +URLs are tried in order until one succeeds, so you should list local mirrors first. +If all downloads fail, the rule will fail. + +

+
auth + +optional. +default is None + +

+ +Dict of String + +An optional dict mapping host names to custom authorization patterns. + +If a URL's host name is present in this dict the value will be used as a pattern when +generating the authorization header for the http request. This enables the use of custom +authorization schemes used in a lot of common cloud storage providers. + +The pattern currently supports 2 tokens: <login> and +<password>, which are replaced with their equivalent value +in the netrc file for the same host name. After formatting, the result is set +as the value for the Authorization field of the HTTP request. + +Example attribute and netrc for a http download to an oauth2 enabled API using a bearer token: + +

+auth_patterns = {
+    "storage.cloudprovider.com": "Bearer <password>"
+}
+
+ +netrc: + +
+machine storage.cloudprovider.com
+        password RANDOM-TOKEN
+
+ +The final HTTP request would have the following header: + +
+Authorization: Bearer RANDOM-TOKEN
+
+ +

+
strip_prefix + +optional. +default is None + +

+ +String + +A directory prefix to strip from the extracted files. + +Many archives contain a top-level directory that contains all of the useful +files in archive. This field can be used to strip it from all of the +extracted files. + +For example, suppose you are using `nixpkgs-22.11.zip`, which contains +the directory `nixpkgs-22.11/` under which there is the `default.nix` +file and the `pkgs/` directory. Specify `strip_prefix = +"nixpkgs-22.11"` to use the `nixpkgs-22.11` directory as your top-level +directory. + +Note that if there are files outside of this directory, they will be +discarded and inaccessible (e.g., a top-level license file). This includes +files/directories that start with the prefix but are not in the directory +(e.g., `nixpkgs-22.11.release-notes`). If the specified prefix does not +match a directory in the archive, Bazel will return an error. + +

+
integrity + +optional. +default is None + +

+ +String + +Expected checksum in Subresource Integrity format of the file downloaded. + +This must match the checksum of the file downloaded. _It is a security risk +to omit the checksum as remote files can change._ At best omitting this +field will make your build non-hermetic. It is optional to make development +easier but either this attribute or `sha256` should be set before shipping. + +

+
sha256 + +optional. +default is None + +

+ +String +The expected SHA-256 of the file downloaded. + +This must match the SHA-256 of the file downloaded. _It is a security risk +to omit the SHA-256 as remote files can change._ At best omitting this +field will make your build non-hermetic. It is optional to make development +easier but should be set before shipping. + +

+
kwargs + +optional. + +

+ +Additional arguments to forward to the underlying repository rule. + +

+
+ + ### nixpkgs_java_configure From 41c75af0aa170378ff2283c89f6200ce4b27f418 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 21 Mar 2023 17:59:09 +0100 Subject: [PATCH 3/3] Test the http repository rule --- testing/core/MODULE.bazel | 2 ++ testing/core/tests/BUILD.bazel | 1 + testing/core/tests/nixpkgs_repositories.bzl | 14 ++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/testing/core/MODULE.bazel b/testing/core/MODULE.bazel index cdb8ce09b..d87c68b4f 100644 --- a/testing/core/MODULE.bazel +++ b/testing/core/MODULE.bazel @@ -17,6 +17,7 @@ local_path_override( non_module_deps = use_extension("//:non_module_deps.bzl", "non_module_deps") use_repo(non_module_deps, "nixpkgs") use_repo(non_module_deps, "remote_nixpkgs") +use_repo(non_module_deps, "http_nixpkgs") use_repo(non_module_deps, "nixpkgs_content") use_repo(non_module_deps, "hello") use_repo(non_module_deps, "expr-test") @@ -26,6 +27,7 @@ use_repo(non_module_deps, "extra-args-test") use_repo(non_module_deps, "nix-file-test") use_repo(non_module_deps, "nix-file-deps-test") use_repo(non_module_deps, "nixpkgs-git-repository-test") +use_repo(non_module_deps, "nixpkgs-http-repository-test") use_repo(non_module_deps, "nixpkgs-local-repository-test") use_repo(non_module_deps, "relative-imports") use_repo(non_module_deps, "output-filegroup-test") diff --git a/testing/core/tests/BUILD.bazel b/testing/core/tests/BUILD.bazel index 9d6a39763..3327e8ba2 100644 --- a/testing/core/tests/BUILD.bazel +++ b/testing/core/tests/BUILD.bazel @@ -25,6 +25,7 @@ expand_location_unit_test_suite() "nix-file-test", "nix-file-deps-test", "nixpkgs-git-repository-test", + "nixpkgs-http-repository-test", "nixpkgs-local-repository-test", "relative-imports", ] diff --git a/testing/core/tests/nixpkgs_repositories.bzl b/testing/core/tests/nixpkgs_repositories.bzl index 692a91303..d58d33433 100644 --- a/testing/core/tests/nixpkgs_repositories.bzl +++ b/testing/core/tests/nixpkgs_repositories.bzl @@ -1,6 +1,7 @@ load( "@rules_nixpkgs_core//:nixpkgs.bzl", "nixpkgs_git_repository", + "nixpkgs_http_repository", "nixpkgs_local_repository", "nixpkgs_package", ) @@ -19,6 +20,13 @@ def nixpkgs_repositories(): sha256 = "0f8c25433a6611fa5664797cd049c80faefec91575718794c701f3b033f2db01", ) + nixpkgs_http_repository( + name = "http_nixpkgs", + url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/22.05.tar.gz", + strip_prefix = "nixpkgs-22.05", + sha256 = "0f8c25433a6611fa5664797cd049c80faefec91575718794c701f3b033f2db01", + ) + # same as @nixpkgs but using the `nix_file_content` parameter nixpkgs_local_repository( name = "nixpkgs_content", @@ -89,6 +97,12 @@ def nixpkgs_repositories(): repositories = {"nixpkgs": "@remote_nixpkgs"}, ) + nixpkgs_package( + name = "nixpkgs-http-repository-test", + attribute_path = "hello", + repositories = {"nixpkgs": "@http_nixpkgs"}, + ) + nixpkgs_package( name = "nixpkgs-local-repository-test", nix_file_content = "with import {}; hello",