From b27ca73617c5319d4b15a9e5ba499cf476679a95 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 6 Sep 2023 02:57:04 -0700 Subject: [PATCH] Prepare for Bazel Bzlmod build Working towards: https://github.com/bazelbuild/bazel/issues/18957 - Updated MODULE.bazel and distdir_deps.bzl to match all dependencies - Introduced the repo_cach_tar rule to help with boostrap build later. - Introduced the test_repo_extension extension to prefetch testing dependencies. - Use --experimental_downloader_config to overcome potential github.com instability. RELNOTES: None. PiperOrigin-RevId: 563045981 Change-Id: I8726a838bfe55908d87d17063121863f604e6211 --- .bazelrc | 5 +- MODULE.bazel | 131 ++++++++++++------ bazel_downloader.cfg | 2 + distdir.bzl | 56 +++++++- distdir_deps.bzl | 115 ++++++++++++--- extensions.bzl | 66 +++++++-- src/BUILD | 1 + .../devtools/build/lib/bazel/rules/java/BUILD | 1 + .../lib/bazel/rules/java/jdk.WORKSPACE.tmpl | 5 + .../lib/blackbox/bazel/DefaultToolsSetup.java | 1 + src/tools/bzlmod/BUILD | 10 ++ src/tools/bzlmod/utils.bzl | 99 +++++++++++++ 12 files changed, 414 insertions(+), 78 deletions(-) create mode 100644 bazel_downloader.cfg create mode 100644 src/tools/bzlmod/BUILD create mode 100644 src/tools/bzlmod/utils.bzl diff --git a/.bazelrc b/.bazelrc index f59dfd259531a8..5d76405341ad8f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -36,8 +36,9 @@ build:windows_arm64 --platforms=//:windows_arm64 build:windows_arm64 --extra_toolchains=@local_config_cc//:cc-toolchain-arm64_windows # Enable Bzlmod -build:bzlmod --enable_bzlmod -build:bzlmod --check_direct_dependencies=error +common:bzlmod --enable_bzlmod +common:bzlmod --check_direct_dependencies=error +common:bzlmod --experimental_downloader_config=bazel_downloader.cfg # Enable Java 11 language features (https://github.com/bazelbuild/bazel/issues/14592) build --java_language_version=11 diff --git a/MODULE.bazel b/MODULE.bazel index 895033325cf39c..bc299bff3a3d2e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,12 +6,16 @@ module( repo_name = "io_bazel", ) +# ========================================= +# Bazel module dependencies +# ========================================= + bazel_dep(name = "rules_license", version = "0.0.7") bazel_dep(name = "bazel_skylib", version = "1.4.1") bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") bazel_dep(name = "grpc", version = "1.48.1.bcr.1", repo_name = "com_github_grpc_grpc") bazel_dep(name = "platforms", version = "0.0.7") -bazel_dep(name = "rules_pkg", version = "0.7.0") +bazel_dep(name = "rules_pkg", version = "0.9.1") bazel_dep(name = "stardoc", version = "0.5.3", repo_name = "io_bazel_skydoc") bazel_dep(name = "zstd-jni", version = "1.5.2-3.bcr.1") bazel_dep(name = "blake3", version = "1.3.3") @@ -22,6 +26,7 @@ bazel_dep(name = "rules_proto", version = "5.3.0-21.7") bazel_dep(name = "rules_jvm_external", version = "5.2") bazel_dep(name = "rules_python", version = "0.24.0") bazel_dep(name = "rules_testing", version = "0.0.4") +bazel_dep(name = "googletest", version = "1.12.1", repo_name = "com_google_googletest") # TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR. bazel_dep(name = "remoteapis", version = "") @@ -43,6 +48,14 @@ local_path_override( path = "./third_party/googleapis", ) +# The following Bazel modules are not direct dependencies for building Bazel, +# but are required for visibility from DIST_ARCHIVE_REPOS in distdir_deps.bzl +bazel_dep(name = "apple_support", version = "1.5.0") +bazel_dep(name = "abseil-cpp", version = "20220623.1") +bazel_dep(name = "c-ares", version = "1.15.0") +bazel_dep(name = "rules_go", version = "0.39.1") +bazel_dep(name = "upb", version = "0.0.0-20220923-a547704") + # ========================================= # Java dependencies # ========================================= @@ -60,40 +73,34 @@ maven.install( use_repo(maven, "maven") java_toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains") -use_repo(java_toolchains, "local_jdk") - -# ========================================= -# Other Bazel internal dependencies -# - embedded JDKs -# - repos for Debian build -# ========================================= - -bazel_internal_deps = use_extension("//:extensions.bzl", "bazel_internal_deps") -use_repo( - bazel_internal_deps, - "debian_cc_deps", - "openjdk_linux_aarch64_vanilla", - "openjdk_linux_ppc64le_vanilla", - "openjdk_linux_s390x_vanilla", - "openjdk_linux_vanilla", - "openjdk_macos_aarch64_vanilla", - "openjdk_macos_x86_64_vanilla", - "openjdk_win_arm64_vanilla", - "openjdk_win_vanilla", +use_repo(java_toolchains, + "local_jdk", + # The following are required for visibility from TEST_REPOS in distdir_deps.bzl + "remote_java_tools", + "remote_java_tools_darwin_x86_64", + "remote_java_tools_darwin_arm64", + "remote_java_tools_linux", + "remote_java_tools_windows", + "remotejdk11_linux", + "remotejdk11_linux_aarch64", + "remotejdk11_linux_ppc64le", + "remotejdk11_linux_s390x", + "remotejdk11_macos", + "remotejdk11_macos_aarch64", + "remotejdk11_win", + "remotejdk11_win_arm64", + "remotejdk17_linux", + "remotejdk17_linux_s390x", + "remotejdk17_macos", + "remotejdk17_macos_aarch64", + "remotejdk17_win", + "remotejdk17_win_arm64", + "remotejdk20_linux", + "remotejdk20_macos", + "remotejdk20_macos_aarch64", + "remotejdk20_win", ) -# ========================================= -# Register platforms & toolchains -# ========================================= - -register_execution_platforms("//:default_host_platform") - -register_toolchains("@bazel_tools//tools/python:autodetecting_toolchain") - -register_toolchains("@local_config_winsdk//:all") - -register_toolchains("//src/main/res:empty_rc_toolchain") - # ========================================= # Python dependencies # ========================================= @@ -110,24 +117,63 @@ pip.parse( use_repo(pip, "bazel_pip_dev_deps") # ========================================= -# Other dev dependencies +# Other Bazel build dependencies # ========================================= -bazel_dep(name = "googletest", version = "1.12.1", repo_name = "com_google_googletest") +bazel_build_deps = use_extension("//:extensions.bzl", "bazel_build_deps") +use_repo( + bazel_build_deps, + "bootstrap_repo_cache", + "debian_cc_deps", + "openjdk_linux_aarch64_vanilla", + "openjdk_linux_ppc64le_vanilla", + "openjdk_linux_s390x_vanilla", + "openjdk_linux_vanilla", + "openjdk_macos_aarch64_vanilla", + "openjdk_macos_x86_64_vanilla", + "openjdk_win_arm64_vanilla", + "openjdk_win_vanilla", +) + +# Required only by `--extra_toolchains=@local_config_cc//:cc-toolchain-arm64_windows` from .bazelrc +cc_configure = use_extension("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_configure_extension") +use_repo(cc_configure, "local_config_cc") -bazel_dev_deps = use_extension("//:extensions.bzl", "bazel_dev_deps") +# ========================================= +# Other Bazel testing dependencies +# ========================================= + +bazel_test_deps = use_extension("//:extensions.bzl", "bazel_test_deps") use_repo( - bazel_dev_deps, + bazel_test_deps, "local_bazel_source_list", "local_config_winsdk", "bazelci_rules", ) +test_repo_extension = use_extension("//:extensions.bzl", "test_repo_extension") +use_repo(test_repo_extension, "test_repos") + bazel_rbe_deps = use_extension("//:rbe_extension.bzl", "bazel_rbe_deps") use_repo(bazel_rbe_deps, "rbe_ubuntu1804_java11") +remote_coverage_tools_extension = use_extension("//tools/test:extensions.bzl", "remote_coverage_tools_extension") +use_repo(remote_coverage_tools_extension, "remote_coverage_tools") + +# ========================================= +# Register platforms & toolchains +# ========================================= + +register_execution_platforms("//:default_host_platform") + +register_toolchains("@bazel_tools//tools/python:autodetecting_toolchain") + +register_toolchains("@local_config_winsdk//:all") + +register_toolchains("//src/main/res:empty_rc_toolchain") + # ========================================= -# Android tools Java dependencies +# Android tools dependencies # ========================================= maven_android = use_extension("@rules_jvm_external//:extensions.bzl", "maven") @@ -145,8 +191,7 @@ maven_android.install( use_repo(maven_android, "maven_android") bazel_android_deps = use_extension("//:extensions.bzl", "bazel_android_deps") -use_repo( - bazel_android_deps, - "android_gmaven_r8", - "desugar_jdk_libs", -) +use_repo(bazel_android_deps, "desugar_jdk_libs") + +remote_android_extensions = use_extension("//tools/android:android_extensions.bzl", "remote_android_tools_extensions") +use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools") diff --git a/bazel_downloader.cfg b/bazel_downloader.cfg new file mode 100644 index 00000000000000..efa65d2e9c3c32 --- /dev/null +++ b/bazel_downloader.cfg @@ -0,0 +1,2 @@ +rewrite (github.com)/(.*) https://mirror.bazel.build/$1/$2 +rewrite (github.com)/(.*) https://$1/$2 diff --git a/distdir.bzl b/distdir.bzl index d01467cc72cd1c..0fb2e3b540454d 100644 --- a/distdir.bzl +++ b/distdir.bzl @@ -14,6 +14,7 @@ """Defines a repository rule that generates an archive consisting of the specified files to fetch""" load("//:distdir_deps.bzl", "DEPS_BY_NAME") +load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts") load("//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar") _BUILD = """ @@ -22,6 +23,7 @@ load("@rules_pkg//pkg:tar.bzl", "pkg_tar") pkg_tar( name="archives", srcs = {srcs}, + strip_prefix = "{strip_prefix}", package_dir = "{dirname}", visibility = ["//visibility:public"], ) @@ -34,7 +36,7 @@ def _distdir_tar_impl(ctx): ctx.file("WORKSPACE", "") ctx.file( "BUILD", - _BUILD.format(srcs = ctx.attr.archives, dirname = ctx.attr.dirname), + _BUILD.format(srcs = ctx.attr.archives, strip_prefix = "", dirname = ctx.attr.dirname), ) _distdir_tar_attrs = { @@ -74,6 +76,58 @@ def distdir_tar(name, archives, sha256, urls, dirname, dist_deps = None): dirname = dirname, ) +def _repo_cache_tar_impl(ctx): + """Generate a repository cache as a tar file. + + This repository rule does the following: + 1. parse all http artifacts required for generating the given list of repositories from the lock file. + 2. downloads all http artifacts to create a repository cache directory structure. + 3. creates a pkg_tar target which packages the repository cache directory structure. + """ + lockfile_path = ctx.path(ctx.attr.lockfile) + http_artifacts = parse_http_artifacts(ctx, lockfile_path, ctx.attr.repos) + + archive_files = [] + readme_content = "This directory contains repository cache artifacts for the following URLs:\n\n" + for artifact in http_artifacts: + url = artifact["url"] + if "integrity" in artifact: + # ./tempfile could be a hard link if --experimental_repository_cache_hardlinks is used, + # therefore we must delete it before creating or writing it again. + ctx.delete("./tempfile") + checksum = ctx.download(url, "./tempfile", executable = False, integrity = artifact["integrity"]) + artifact["sha256"] = checksum.sha256 + + if "sha256" in artifact: + sha256 = artifact["sha256"] + output_file = "content_addressable/sha256/%s/file" % sha256 + ctx.download(url, output_file, sha256, executable = False) + archive_files.append(output_file) + readme_content += "- %s (SHA256: %s)\n" % (url, sha256) + else: + fail("Could not find integrity or sha256 hash for artifact %s" % url) + + ctx.file("README.md", readme_content) + ctx.file( + "BUILD", + _BUILD.format( + srcs = archive_files + ["README.md"], + strip_prefix = "external/" + ctx.attr.name, + dirname = ctx.attr.dirname, + ), + ) + +_repo_cache_tar_attrs = { + "lockfile": attr.label(default = Label("//:MODULE.bazel.lock")), + "dirname": attr.string(default = "repository_cache"), + "repos": attr.string_list(), +} + +repo_cache_tar = repository_rule( + implementation = _repo_cache_tar_impl, + attrs = _repo_cache_tar_attrs, +) + def dist_http_archive(name, **kwargs): """Wraps http_archive, providing attributes like sha and urls from the central list. diff --git a/distdir_deps.bzl b/distdir_deps.bzl index c87edb166e44dc..c7b280a6d41ce7 100644 --- a/distdir_deps.bzl +++ b/distdir_deps.bzl @@ -18,6 +18,84 @@ This is internal source and is not intended to tell you what version you should use for each dependency. """ +load("//src/tools/bzlmod:utils.bzl", "get_canonical_repo_name") + +################################################################################## +# +# The list of repositories required while bootstrapping Bazel offline +# +################################################################################## +DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [ + "abseil-cpp", + "apple_support", + "bazel_skylib", + "blake3", + "c-ares", + "com_github_grpc_grpc", + "com_google_protobuf", + "io_bazel_skydoc", + "platforms", + "rules_cc", + "rules_go", + "rules_java", + "rules_jvm_external", + "rules_license", + "rules_pkg", + "rules_proto", + "rules_python", + "upb", + "zlib", + "zstd-jni", +]] + [(get_canonical_repo_name("com_github_grpc_grpc") + suffix) for suffix in [ + # Extra grpc dependencies introduced via its module extension + "~grpc_repo_deps_ext~bazel_gazelle", # TODO: Should be a bazel_dep + "~grpc_repo_deps_ext~bazel_skylib", # TODO: Should be removed + "~grpc_repo_deps_ext~com_envoyproxy_protoc_gen_validate", + "~grpc_repo_deps_ext~com_github_cncf_udpa", + "~grpc_repo_deps_ext~com_google_googleapis", + "~grpc_repo_deps_ext~envoy_api", + "~grpc_repo_deps_ext~rules_cc", # TODO: Should be removed +]] + +################################################################################## +# +# The list of repositories required while running Bazel integration tests offline +# +################################################################################## +TEST_REPOS = [get_canonical_repo_name(repo) for repo in [ + "android_tools", + "android_gmaven_r8", + "bazel_skylib", + "com_google_protobuf", + "remote_coverage_tools", + "remote_java_tools", + "remote_java_tools_darwin_x86_64", + "remote_java_tools_darwin_arm64", + "remote_java_tools_linux", + "remote_java_tools_windows", + "remotejdk11_linux", + "remotejdk11_linux_aarch64", + "remotejdk11_macos", + "remotejdk11_macos_aarch64", + "remotejdk11_win", + "remotejdk17_linux", + "remotejdk17_linux_s390x", + "remotejdk17_macos", + "remotejdk17_macos_aarch64", + "remotejdk17_win", + "remotejdk20_linux", + "remotejdk20_macos", + "remotejdk20_macos_aarch64", + "remotejdk20_win", + "rules_cc", + "rules_java", + "rules_license", + "rules_proto", + "rules_python", + "rules_pkg", + "rules_testing", +]] + DIST_DEPS = { ######################################## # @@ -100,12 +178,11 @@ DIST_DEPS = { # ################################################# "com_google_protobuf": { - "archive": "v21.7.tar.gz", - "sha256": "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", + "archive": "protobuf-all-21.7.zip", + "sha256": "5493a21f5ed3fc502e66fec6b9449c06a551ced63002fa48903c40dfa8de7a4a", "strip_prefix": "protobuf-21.7", "urls": [ - "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", - "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + "https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protobuf-all-21.7.zip", ], "patch_args": ["-p1"], "patches": ["//third_party/protobuf:21.7.patch"], @@ -116,7 +193,7 @@ DIST_DEPS = { "@rules_license//licenses/generic:notice", ], "license_text": "LICENSE", - "package_version": "3.19.6", + "package_version": "21.7", }, "com_github_grpc_grpc": { "archive": "v1.48.1.tar.gz", @@ -302,16 +379,16 @@ DIST_DEPS = { "package_version": "8.0.40", }, "bazel_skylib": { - "archive": "bazel-skylib-1.3.0.tar.gz", - "sha256": "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", + "archive": "bazel-skylib-1.4.1.tar.gz", + "sha256": "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7", "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz", ], "used_in": [ "additional_distfiles", ], - "package_version": "1.3.0", + "package_version": "1.4.1", }, "io_bazel_skydoc": { "archive": "1ef781ced3b1443dca3ed05dec1989eca1a4e1cd.tar.gz", @@ -338,16 +415,16 @@ DIST_DEPS = { "package_version": "0.0.7", }, "rules_pkg": { - "archive": "rules_pkg-0.8.0.tar.gz", - "sha256": "eea0f59c28a9241156a47d7a8e32db9122f3d50b505fae0f33de6ce4d9b61834", + "archive": "rules_pkg-0.9.1.tar.gz", + "sha256": "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.8.0/rules_pkg-0.8.0.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.8.0/rules_pkg-0.8.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", ], "used_in": [ "additional_distfiles", ], - "package_version": "0.8.0", + "package_version": "0.9.1", }, "rules_jvm_external": { "archive": "rules_jvm_external-5.2.tar.gz", @@ -366,10 +443,10 @@ DIST_DEPS = { "package_version": "5.2", }, "rules_python": { - "sha256": "ffc7b877c95413c82bfd5482c017edcf759a6250d8b24e82f41f3c8b8d9e287e", - "strip_prefix": "rules_python-0.19.0", - "urls": ["https://github.com/bazelbuild/rules_python/releases/download/0.19.0/rules_python-0.19.0.tar.gz"], - "archive": "rules_python-0.19.0.tar.gz", + "sha256": "0a8003b044294d7840ac7d9d73eef05d6ceb682d7516781a4ec62eeb34702578", + "strip_prefix": "rules_python-0.24.0", + "urls": ["https://github.com/bazelbuild/rules_python/releases/download/0.24.0/rules_python-0.24.0.tar.gz"], + "archive": "rules_python-0.24.0.tar.gz", "used_in": ["additional_distfiles"], }, "rules_testing": { diff --git a/extensions.bzl b/extensions.bzl index de87962a44abb2..83acdcdadf0f18 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -16,33 +16,73 @@ """ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("//:distdir.bzl", "dist_http_archive", "repo_cache_tar") +load("//:distdir_deps.bzl", "DIST_ARCHIVE_REPOS", "TEST_REPOS") load("//:repositories.bzl", "embedded_jdk_repositories") -load("//:distdir.bzl", "dist_http_archive", "dist_http_jar") -load("//tools/distributions/debian:deps.bzl", "debian_deps") -load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository") load("//src/main/res:winsdk_configure.bzl", "winsdk_configure") +load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository") +load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts") +load("//tools/distributions/debian:deps.bzl", "debian_deps") -### Extra dependencies for building Bazel - -def _bazel_internal_deps(_ctx): +### Dependencies for building Bazel +def _bazel_build_deps(_ctx): embedded_jdk_repositories() debian_deps() + repo_cache_tar(name = "bootstrap_repo_cache", repos = DIST_ARCHIVE_REPOS, dirname = "derived/repository_cache") -bazel_internal_deps = module_extension(implementation = _bazel_internal_deps) - -### Extra dependencies for testing Bazel +bazel_build_deps = module_extension(implementation = _bazel_build_deps) -def _bazel_dev_deps(_ctx): +### Dependencies for testing Bazel +def _bazel_test_deps(_ctx): list_source_repository(name = "local_bazel_source_list") dist_http_archive(name = "bazelci_rules") winsdk_configure(name = "local_config_winsdk") -bazel_dev_deps = module_extension(implementation = _bazel_dev_deps) +bazel_test_deps = module_extension(implementation = _bazel_test_deps) + +_HUB_TEST_REPO_BUILD = """ +filegroup( + name="srcs", + srcs = {srcs}, + visibility = ["//visibility:public"], +) +""" + +def _hub_test_repo_impl(ctx): + ctx.file( + "BUILD", + _HUB_TEST_REPO_BUILD.format(srcs = ["@%s//file" % repo for repo in ctx.attr.repos]), + ) + +hub_test_repo = repository_rule( + implementation = _hub_test_repo_impl, + attrs = {"repos": attr.string_list()}, +) + +def _test_repo_extension_impl(ctx): + """This module extension is used to fetch http artifacts required for generating TEST_REPOS.""" + lockfile_path = ctx.path(Label("//:MODULE.bazel.lock")) + http_artifacts = parse_http_artifacts(ctx, lockfile_path, TEST_REPOS) + name = "test_repo_" + cnt = 1 + for artifact in http_artifacts: + # Define one http_file for each artifact so that we can fetch them in parallel. + http_file( + name = name + str(cnt), + url = artifact["url"], + sha256 = artifact["sha256"] if "sha256" in artifact else None, + integrity = artifact["integrity"] if "integrity" in artifact else None, + ) + cnt += 1 + + # write a repo rule that depends on all the http_file rules + hub_test_repo(name = "test_repos", repos = [(name + str(i)) for i in range(1, cnt)]) -### Extra dependencies for Bazel Android tools +test_repo_extension = module_extension(implementation = _test_repo_extension_impl) +### Dependencies for Bazel Android tools def _bazel_android_deps(_ctx): - dist_http_jar(name = "android_gmaven_r8") dist_http_archive(name = "desugar_jdk_libs") bazel_android_deps = module_extension(implementation = _bazel_android_deps) diff --git a/src/BUILD b/src/BUILD index 876f836ee7a004..103569bb9daecc 100644 --- a/src/BUILD +++ b/src/BUILD @@ -374,6 +374,7 @@ filegroup( "//src/test/tools:srcs", "//src/tools/android:srcs", "//src/tools/android/java/com/google/devtools/build/android:srcs", + "//src/tools/bzlmod:srcs", "//src/tools/execlog:srcs", "//src/tools/launcher:srcs", "//src/tools/remote:srcs", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD index 7963ca317cbb26..ef9e6fee1e371b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD @@ -90,6 +90,7 @@ gen_workspace_stanza( urls = {urls}, """, repos = [ + "bazel_skylib", "rules_cc", "rules_java", "rules_proto", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl index 7ff1f1b1949223..dc4753b282e95a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl @@ -2,6 +2,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +maybe( + http_archive, +{bazel_skylib} +) + maybe( http_archive, {rules_java} diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java b/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java index d2463dcd8d8574..e487ebacef61e6 100644 --- a/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java +++ b/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java @@ -28,6 +28,7 @@ public class DefaultToolsSetup implements ToolsSetup { private static ImmutableList repos = ImmutableList.builder() + .add("bazel_skylib") .add("rules_cc") .add("rules_proto") .add("rules_java") diff --git a/src/tools/bzlmod/BUILD b/src/tools/bzlmod/BUILD new file mode 100644 index 00000000000000..161c1bff2a2a8f --- /dev/null +++ b/src/tools/bzlmod/BUILD @@ -0,0 +1,10 @@ +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//:__pkg__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) diff --git a/src/tools/bzlmod/utils.bzl b/src/tools/bzlmod/utils.bzl new file mode 100644 index 00000000000000..d080fcea73126a --- /dev/null +++ b/src/tools/bzlmod/utils.bzl @@ -0,0 +1,99 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper functions for Bzlmod build""" + +def get_canonical_repo_name(apparent_repo_name): + """Returns the canonical repo name for the given apparent repo name seen by the module this bzl file belongs to.""" + if not apparent_repo_name.startswith("@"): + apparent_repo_name = "@" + apparent_repo_name + return Label(apparent_repo_name).workspace_name + +def extract_url(attributes): + """Extracts the url from the given attributes. + + Args: + attributes: The attributes to extract the url from. + + Returns: + The url extracted from the given attributes. + """ + if "urls" in attributes: + return attributes["urls"][0][2:] + elif "url" in attributes: + return attributes["url"][2:] + else: + fail("Could not find url in attributes %s" % attributes) + +def parse_http_artifacts(ctx, lockfile_path, required_repos): + """Parses the http artifacts required from for fetching the given repos from the lockfile. + + Args: + ctx: the repository / module extension ctx object. + lockfile_path: The path of the lockfile to extract the http artifacts from. + required_repos: The list of required repos to extract the http artifacts for, + only support `http_archive`, `http_file` and `http_jar` repo rules. + + Returns: + A list of http artifacts in the form of + [{"integrity": , "url": }, {"sha256": , "url": }, ...] + + All lockfile string values are prefixed with `--`, hence the `[2:]` is needed to remove the prefix. + """ + lockfile = json.decode(ctx.read(lockfile_path)) + http_artifacts = [] + found_repos = [] + for _, module in lockfile["moduleDepGraph"].items(): + if "repoSpec" in module and module["repoSpec"]["ruleClassName"] == "http_archive": + repo_spec = module["repoSpec"] + attributes = repo_spec["attributes"] + repo_name = attributes["name"][2:] + + if repo_name not in required_repos: + continue + found_repos.append(repo_name) + + http_artifacts.append({ + "integrity": attributes["integrity"][2:], + "url": extract_url(attributes), + }) + if "remote_patches" in attributes: + for patch, integrity in attributes["remote_patches"].items(): + http_artifacts.append({ + "integrity": integrity[2:], + "url": patch[2:], + }) + + for _, extension in lockfile["moduleExtensions"].items(): + # TODO(pcloudy): update this when lockfile format changes (https://github.com/bazelbuild/bazel/issues/19154) + for _, repo_spec in extension["generatedRepoSpecs"].items(): + rule_class = repo_spec["ruleClassName"] + if rule_class == "http_archive" or rule_class == "http_file" or rule_class == "http_jar": + attributes = repo_spec["attributes"] + repo_name = attributes["name"][2:] + + if repo_name not in required_repos: + continue + found_repos.append(repo_name) + + http_artifacts.append({ + "sha256": attributes["sha256"][2:], + "url": extract_url(attributes), + }) + + missing_repos = [repo for repo in required_repos if repo not in found_repos] + if missing_repos: + fail("Could not find all required repos, missing: %s" % missing_repos) + + return http_artifacts