From d39d35cafa9734bf0b93eeb46bcf57209336b7d8 Mon Sep 17 00:00:00 2001 From: Mike Bland Date: Thu, 3 Oct 2024 22:02:13 -0400 Subject: [PATCH] Begin toolchainized protoc implementation I solved the rest of the build failures from #1618. Now multiple jobs are failing due to hanging scalac worker jobs as described in the PR description. - Proto Toolchainisation Design Doc https://docs.google.com/document/d/1CE6wJHNfKbUPBr7-mmk_0Yo3a4TaqcTPE0OWNuQkhPs/edit - bazelbuild/bazel: Protobuf repo recompilation sensitivity https://github.com/bazelbuild/bazel/issues/7095 - bazelbuild/rules_proto: Implement proto toolchainisation https://github.com/bazelbuild/rules_proto/issues/179 - rules_proto 6.0.0 release notes mentioning Protobuf Toolchainisation https://github.com/bazelbuild/rules_proto/releases/tag/6.0.0 It occurred to me to try adopting Proto Toolchainisation to see if that might resolve the issue. I've got it successfully generating the `@io_bazel_rules_scala_protoc` repo and registering toolchains for `@rules_proto//proto:toolchain_type`. However, it's currently dying because there's no toolchain registered for '@rules_java//java/proto:toolchain_type'. ```txt bazel build //src/... ERROR: .../src/protobuf/io/bazel/rules_scala/BUILD:4:14: in @@_builtins//:common/java/proto/java_proto_library.bzl%bazel_java_proto_aspect aspect on proto_library rule //src/protobuf/io/bazel/rules_scala:diagnostics_proto: Traceback (most recent call last): File "/virtual_builtins_bzl/common/java/proto/java_proto_library.bzl", line 53, column 53, in _bazel_java_proto_aspect_impl File "/virtual_builtins_bzl/common/proto/proto_common.bzl", line 364, column 17, in _find_toolchain Error in fail: No toolchains registered for '@rules_java//java/proto:toolchain_type'. ERROR: Analysis of target '//src/protobuf/io/bazel/rules_scala:diagnostics_proto' failed ERROR: Analysis of target '//src/java/io/bazel/rulesscala/scalac:scalac' failed; build aborted: Analysis failed ``` --- .bazelrc | 5 ++ WORKSPACE | 9 +++ scala/protoc_deps.bzl | 17 +++++ scala/protoc_toolchain.bzl | 148 +++++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 scala/protoc_deps.bzl create mode 100644 scala/protoc_toolchain.bzl diff --git a/.bazelrc b/.bazelrc index 128fb9d38..5010fa794 100644 --- a/.bazelrc +++ b/.bazelrc @@ -7,3 +7,8 @@ build:windows --worker_quit_after_build --enable_runfiles # Remove these upon completing Bzlmod compatibility work. # - https://github.com/bazelbuild/rules_scala/issues/1482 build --noenable_bzlmod --enable_workspace + +# Remove once proto toolchainization becomes the default +# - https://bazel.build/reference/command-line-reference#flag--incompatible_enable_proto_toolchain_resolution +# - https://docs.google.com/document/d/1CE6wJHNfKbUPBr7-mmk_0Yo3a4TaqcTPE0OWNuQkhPs/edit +build --incompatible_enable_proto_toolchain_resolution diff --git a/WORKSPACE b/WORKSPACE index 37e83a6d9..18bbba700 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -46,6 +46,15 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains") rules_proto_toolchains() +load("//scala:protoc_deps.bzl", "scala_protoc_setup") + +scala_protoc_setup() + +load("//scala:protoc_toolchain.bzl", "protoc_toolchains") + +protoc_toolchains(name = "io_bazel_rules_scala_protoc") +register_toolchains("@io_bazel_rules_scala_protoc//:all") + load("//scala:scala_cross_version.bzl", "default_maven_server_urls") load("//twitter_scrooge:twitter_scrooge.bzl", "twitter_scrooge") diff --git a/scala/protoc_deps.bzl b/scala/protoc_deps.bzl new file mode 100644 index 000000000..932da10d2 --- /dev/null +++ b/scala/protoc_deps.bzl @@ -0,0 +1,17 @@ +"""com_google_protobuf dependencies for the protoc_toolchain""" + +load ( + ":protoc_toolchain.bzl", + "PROTOBUF_SHA256", + "PROTOBUF_URL", + "PROTOBUF_VERSION", +) +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def scala_protoc_setup(): + http_archive( + name = "com_google_protobuf", + sha256 = PROTOBUF_SHA256, + strip_prefix = "protobuf-%s" % PROTOBUF_VERSION, + url = PROTOBUF_URL, + ) diff --git a/scala/protoc_toolchain.bzl b/scala/protoc_toolchain.bzl new file mode 100644 index 000000000..1c7e37d2e --- /dev/null +++ b/scala/protoc_toolchain.bzl @@ -0,0 +1,148 @@ +"""Prepares the precompiled protoc toolchain""" + +PROTOBUF_VERSION = "28.2" +PROTOBUF_SHA256 = ( + "b2340aa47faf7ef10a0328190319d3f3bee1b24f426d4ce8f4253b6f27ce16db" +) +PROTOBUF_URL = ( + "https://github.com/protocolbuffers/protobuf/releases/download/" + + "v{version}/protobuf-{version}.tar.gz".format(version=PROTOBUF_VERSION) +) + +_PROTOC_RELEASES_URL = "https://github.com/protocolbuffers/protobuf/releases" +_PROTOC_DOWNLOAD_URL = ( + _PROTOC_RELEASES_URL + "/download/" + + "v{version}/protoc-{version}-".format(version=PROTOBUF_VERSION) + + "{platform}.zip" +) + +_PROTOC_BUILDS = { + "linux-aarch_64": struct( + integrity = "sha256-kdglPNwPDw/FHCtpyAZ3mWYy9SWthFBL+ltO44rT5Jw=", + exec_compat = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ), + "linux-ppcle_64": struct( + integrity = "sha256-xcFrR2f/iGYJDuEOhwkDABBSAbXbHs7zgO3XPCM03Ng=", + exec_compat = [ + "@platforms//os:linux", + "@platforms//cpu:ppc64le", + ], + ), + "linux-s390_64": struct( + integrity = "sha256-ESIsQ4+G6Hsv2vaKKmzB2ytiKBP9btiRUJ4vOw4F7hs=", + exec_compat = [ + "@platforms//os:linux", + "@platforms//cpu:s390x", + ], + ), + "linux-x86_32": struct( + integrity = "sha256-ucjToo5Lq5WcwQ5smjjxfFlGc3Npv+AT9RqG19Ns//E=", + exec_compat = [ + "@platforms//os:linux", + "@platforms//cpu:x86_32" + ], + ), + "linux-x86_64": struct( + integrity = "sha256-L+v9QrWc6Too63iQGaRwo90ESWGbwE+E2tEzPaJh3sE=", + exec_compat = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64" + ], + ), + "osx-aarch_64": struct( + integrity = "sha256-e7BI9ShBeJ2exhmDvgzkyeT7O9mhQ0YoILqaO+CgN5c=", + exec_compat = [ + "@platforms//os:osx", + "@platforms//cpu:aarch64", + ], + ), + "osx-x86_64": struct( + integrity = "sha256-Iy8H0Sv0gGIHp57CxzeDAcUuby9+/dIcDdQW8L2hA+w=", + exec_compat = [ + "@platforms//os:osx", + "@platforms//cpu:x86_64" + ], + ), + "win32": struct( + integrity = "sha256-V6hpbqvtUgl19PpGkUGwuC+iEIYj0YtyOqi+yae0u1g=", + exec_compat = [ + "@platforms//os:windows", + "@platforms//cpu:x86_32" + ], + ), + "win64": struct( + integrity = "sha256-S94ZJx7XyrkANXDyjG5MTXGWPq8SEahr87sl2biVF3o=", + exec_compat = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64" + ], + ), +} + +_PROTOC_BUILD_TEMPLATE="""load( + "@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl", + "proto_toolchain", +) + +proto_toolchains = {{ +{proto_toolchains} +}} + +[ + proto_toolchain( + name = build, + proto_compiler = ":%s/bin/protoc%s" % ( + build, ".exe" if build.startswith("win") else "" + ), + exec_compatible_with = specs, + ) + for build, specs in proto_toolchains.items() +] +""" + +_PROTOC_TOOLCHAIN_TEMPLATE= """ {build}: [\n{specs}\n ],""" + +def _protoc_toolchains_impl(repository_ctx): + proto_toolchains = [] + + for build in repository_ctx.attr.protoc_builds: + protoc_build = _PROTOC_BUILDS[build] + repository_ctx.download_and_extract( + url = _PROTOC_DOWNLOAD_URL.format(platform = build), + output = build, + integrity = protoc_build.integrity, + ) + proto_toolchains.append(_PROTOC_TOOLCHAIN_TEMPLATE.format( + build = '"%s"' % build, + specs = "\n".join([ + ' "%s",' % s for s in protoc_build.exec_compat + ]) + )) + + build_content = _PROTOC_BUILD_TEMPLATE.format( + name = repository_ctx.attr.name, + proto_toolchains = "\n".join(proto_toolchains) + ) + repository_ctx.file("BUILD", content=build_content, executable=False) + +protoc_toolchains = repository_rule( + implementation = _protoc_toolchains_impl, + attrs = { + "protoc_builds": attr.string_list( + doc = ( + "os and arch identifiers for precompiled protoc release " + + "download filenames from " + _PROTOC_RELEASES_URL + ), + default = [ + "linux-aarch_64", + "linux-x86_64", + "osx-aarch_64", + "osx-x86_64", + "win64", + ], + ), + } +)