Skip to content

Commit

Permalink
Prepare for Bazel Bzlmod build
Browse files Browse the repository at this point in the history 
Working towards: #18957

- Updated MODULE.bazel and distdir_deps.bzl to match all dependencies
- Introduced the repo_cach_tar rule to help with boostrap build later.
- Introduced the test_repo_extension extension to prefetch testing dependencies.
- Use --experimental_downloader_config to overcome potential github.com instability.

RELNOTES: None.
PiperOrigin-RevId: 563045981
Change-Id: I8726a838bfe55908d87d17063121863f604e6211
  • Loading branch information
@meteorcloudy @copybara-github
meteorcloudy authored and copybara-github committed Sep 6, 2023
1 parent cd9f8e0 commit b27ca73
Show file tree
Hide file tree
Showing 12 changed files with 414 additions and 78 deletions.
5 changes: 3 additions & 2 deletions .bazelrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ build:windows_arm64 --platforms=//:windows_arm64
build:windows_arm64 --extra_toolchains=@local_config_cc//:cc-toolchain-arm64_windows

# Enable Bzlmod
build:bzlmod --enable_bzlmod
build:bzlmod --check_direct_dependencies=error
common:bzlmod --enable_bzlmod
common:bzlmod --check_direct_dependencies=error
common:bzlmod --experimental_downloader_config=bazel_downloader.cfg

# Enable Java 11 language features (https://github.com/bazelbuild/bazel/issues/14592)
build --java_language_version=11
Expand Down
131 changes: 88 additions & 43 deletions MODULE.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,16 @@ module(
repo_name = "io_bazel",
)

# =========================================
# Bazel module dependencies
# =========================================

bazel_dep(name = "rules_license", version = "0.0.7")
bazel_dep(name = "bazel_skylib", version = "1.4.1")
bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
bazel_dep(name = "grpc", version = "1.48.1.bcr.1", repo_name = "com_github_grpc_grpc")
bazel_dep(name = "platforms", version = "0.0.7")
bazel_dep(name = "rules_pkg", version = "0.7.0")
bazel_dep(name = "rules_pkg", version = "0.9.1")
bazel_dep(name = "stardoc", version = "0.5.3", repo_name = "io_bazel_skydoc")
bazel_dep(name = "zstd-jni", version = "1.5.2-3.bcr.1")
bazel_dep(name = "blake3", version = "1.3.3")
Expand All @@ -22,6 +26,7 @@ bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
bazel_dep(name = "rules_jvm_external", version = "5.2")
bazel_dep(name = "rules_python", version = "0.24.0")
bazel_dep(name = "rules_testing", version = "0.0.4")
bazel_dep(name = "googletest", version = "1.12.1", repo_name = "com_google_googletest")

# TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR.
bazel_dep(name = "remoteapis", version = "")
Expand All @@ -43,6 +48,14 @@ local_path_override(
path = "./third_party/googleapis",
)

# The following Bazel modules are not direct dependencies for building Bazel,
# but are required for visibility from DIST_ARCHIVE_REPOS in distdir_deps.bzl
bazel_dep(name = "apple_support", version = "1.5.0")
bazel_dep(name = "abseil-cpp", version = "20220623.1")
bazel_dep(name = "c-ares", version = "1.15.0")
bazel_dep(name = "rules_go", version = "0.39.1")
bazel_dep(name = "upb", version = "0.0.0-20220923-a547704")

# =========================================
# Java dependencies
# =========================================
Expand All @@ -60,40 +73,34 @@ maven.install(
use_repo(maven, "maven")

java_toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains")
use_repo(java_toolchains, "local_jdk")

# =========================================
# Other Bazel internal dependencies
# - embedded JDKs
# - repos for Debian build
# =========================================

bazel_internal_deps = use_extension("//:extensions.bzl", "bazel_internal_deps")
use_repo(
bazel_internal_deps,
"debian_cc_deps",
"openjdk_linux_aarch64_vanilla",
"openjdk_linux_ppc64le_vanilla",
"openjdk_linux_s390x_vanilla",
"openjdk_linux_vanilla",
"openjdk_macos_aarch64_vanilla",
"openjdk_macos_x86_64_vanilla",
"openjdk_win_arm64_vanilla",
"openjdk_win_vanilla",
use_repo(java_toolchains,
"local_jdk",
# The following are required for visibility from TEST_REPOS in distdir_deps.bzl
"remote_java_tools",
"remote_java_tools_darwin_x86_64",
"remote_java_tools_darwin_arm64",
"remote_java_tools_linux",
"remote_java_tools_windows",
"remotejdk11_linux",
"remotejdk11_linux_aarch64",
"remotejdk11_linux_ppc64le",
"remotejdk11_linux_s390x",
"remotejdk11_macos",
"remotejdk11_macos_aarch64",
"remotejdk11_win",
"remotejdk11_win_arm64",
"remotejdk17_linux",
"remotejdk17_linux_s390x",
"remotejdk17_macos",
"remotejdk17_macos_aarch64",
"remotejdk17_win",
"remotejdk17_win_arm64",
"remotejdk20_linux",
"remotejdk20_macos",
"remotejdk20_macos_aarch64",
"remotejdk20_win",
)

# =========================================
# Register platforms & toolchains
# =========================================

register_execution_platforms("//:default_host_platform")

register_toolchains("@bazel_tools//tools/python:autodetecting_toolchain")

register_toolchains("@local_config_winsdk//:all")

register_toolchains("//src/main/res:empty_rc_toolchain")

# =========================================
# Python dependencies
# =========================================
Expand All @@ -110,24 +117,63 @@ pip.parse(
use_repo(pip, "bazel_pip_dev_deps")

# =========================================
# Other dev dependencies
# Other Bazel build dependencies
# =========================================

bazel_dep(name = "googletest", version = "1.12.1", repo_name = "com_google_googletest")
bazel_build_deps = use_extension("//:extensions.bzl", "bazel_build_deps")
use_repo(
bazel_build_deps,
"bootstrap_repo_cache",
"debian_cc_deps",
"openjdk_linux_aarch64_vanilla",
"openjdk_linux_ppc64le_vanilla",
"openjdk_linux_s390x_vanilla",
"openjdk_linux_vanilla",
"openjdk_macos_aarch64_vanilla",
"openjdk_macos_x86_64_vanilla",
"openjdk_win_arm64_vanilla",
"openjdk_win_vanilla",
)

# Required only by `--extra_toolchains=@local_config_cc//:cc-toolchain-arm64_windows` from .bazelrc
cc_configure = use_extension("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_configure_extension")
use_repo(cc_configure, "local_config_cc")

bazel_dev_deps = use_extension("//:extensions.bzl", "bazel_dev_deps")
# =========================================
# Other Bazel testing dependencies
# =========================================

bazel_test_deps = use_extension("//:extensions.bzl", "bazel_test_deps")
use_repo(
bazel_dev_deps,
bazel_test_deps,
"local_bazel_source_list",
"local_config_winsdk",
"bazelci_rules",
)

test_repo_extension = use_extension("//:extensions.bzl", "test_repo_extension")
use_repo(test_repo_extension, "test_repos")

bazel_rbe_deps = use_extension("//:rbe_extension.bzl", "bazel_rbe_deps")
use_repo(bazel_rbe_deps, "rbe_ubuntu1804_java11")

remote_coverage_tools_extension = use_extension("//tools/test:extensions.bzl", "remote_coverage_tools_extension")
use_repo(remote_coverage_tools_extension, "remote_coverage_tools")

# =========================================
# Register platforms & toolchains
# =========================================

register_execution_platforms("//:default_host_platform")

register_toolchains("@bazel_tools//tools/python:autodetecting_toolchain")

register_toolchains("@local_config_winsdk//:all")

register_toolchains("//src/main/res:empty_rc_toolchain")

# =========================================
# Android tools Java dependencies
# Android tools dependencies
# =========================================

maven_android = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
Expand All @@ -145,8 +191,7 @@ maven_android.install(
use_repo(maven_android, "maven_android")

bazel_android_deps = use_extension("//:extensions.bzl", "bazel_android_deps")
use_repo(
bazel_android_deps,
"android_gmaven_r8",
"desugar_jdk_libs",
)
use_repo(bazel_android_deps, "desugar_jdk_libs")

remote_android_extensions = use_extension("//tools/android:android_extensions.bzl", "remote_android_tools_extensions")
use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools")
2 changes: 2 additions & 0 deletions bazel_downloader.cfg
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
rewrite (github.com)/(.*) https://mirror.bazel.build/$1/$2
rewrite (github.com)/(.*) https://$1/$2
56 changes: 55 additions & 1 deletion distdir.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""Defines a repository rule that generates an archive consisting of the specified files to fetch"""

load("//:distdir_deps.bzl", "DEPS_BY_NAME")
load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts")
load("//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar")

_BUILD = """
Expand All @@ -22,6 +23,7 @@ load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
pkg_tar(
name="archives",
srcs = {srcs},
strip_prefix = "{strip_prefix}",
package_dir = "{dirname}",
visibility = ["//visibility:public"],
)
Expand All @@ -34,7 +36,7 @@ def _distdir_tar_impl(ctx):
ctx.file("WORKSPACE", "")
ctx.file(
"BUILD",
_BUILD.format(srcs = ctx.attr.archives, dirname = ctx.attr.dirname),
_BUILD.format(srcs = ctx.attr.archives, strip_prefix = "", dirname = ctx.attr.dirname),
)

_distdir_tar_attrs = {
Expand Down Expand Up @@ -74,6 +76,58 @@ def distdir_tar(name, archives, sha256, urls, dirname, dist_deps = None):
dirname = dirname,
)

def _repo_cache_tar_impl(ctx):
"""Generate a repository cache as a tar file.
This repository rule does the following:
1. parse all http artifacts required for generating the given list of repositories from the lock file.
2. downloads all http artifacts to create a repository cache directory structure.
3. creates a pkg_tar target which packages the repository cache directory structure.
"""
lockfile_path = ctx.path(ctx.attr.lockfile)
http_artifacts = parse_http_artifacts(ctx, lockfile_path, ctx.attr.repos)

archive_files = []
readme_content = "This directory contains repository cache artifacts for the following URLs:\n\n"
for artifact in http_artifacts:
url = artifact["url"]
if "integrity" in artifact:
# ./tempfile could be a hard link if --experimental_repository_cache_hardlinks is used,
# therefore we must delete it before creating or writing it again.
ctx.delete("./tempfile")
checksum = ctx.download(url, "./tempfile", executable = False, integrity = artifact["integrity"])
artifact["sha256"] = checksum.sha256

if "sha256" in artifact:
sha256 = artifact["sha256"]
output_file = "content_addressable/sha256/%s/file" % sha256
ctx.download(url, output_file, sha256, executable = False)
archive_files.append(output_file)
readme_content += "- %s (SHA256: %s)\n" % (url, sha256)
else:
fail("Could not find integrity or sha256 hash for artifact %s" % url)

ctx.file("README.md", readme_content)
ctx.file(
"BUILD",
_BUILD.format(
srcs = archive_files + ["README.md"],
strip_prefix = "external/" + ctx.attr.name,
dirname = ctx.attr.dirname,
),
)

_repo_cache_tar_attrs = {
"lockfile": attr.label(default = Label("//:MODULE.bazel.lock")),
"dirname": attr.string(default = "repository_cache"),
"repos": attr.string_list(),
}

repo_cache_tar = repository_rule(
implementation = _repo_cache_tar_impl,
attrs = _repo_cache_tar_attrs,
)

def dist_http_archive(name, **kwargs):
"""Wraps http_archive, providing attributes like sha and urls from the central list.
Expand Down
Loading

0 comments on commit b27ca73

Please sign in to comment.