Skip to content

Commit

Permalink
Begin toolchainized protoc implementation
Browse files Browse the repository at this point in the history
I solved the rest of the build failures from bazelbuild#1618. Now multiple jobs
are failing due to hanging scalac worker jobs as described in the PR
description.

- Proto Toolchainisation Design Doc
  https://docs.google.com/document/d/1CE6wJHNfKbUPBr7-mmk_0Yo3a4TaqcTPE0OWNuQkhPs/edit

- bazelbuild/bazel: Protobuf repo recompilation sensitivity
  bazelbuild/bazel#7095

- bazelbuild/rules_proto: Implement proto toolchainisation
  bazelbuild/rules_proto#179

- rules_proto 6.0.0 release notes mentioning Protobuf Toolchainisation
  https://github.com/bazelbuild/rules_proto/releases/tag/6.0.0

It occurred to me to try adopting Proto Toolchainisation to see if that
might resolve the issue. I've got it successfully generating the
`@io_bazel_rules_scala_protoc` repo and registering toolchains for
`@rules_proto//proto:toolchain_type`. However, it's currently dying
because there's no toolchain registered for
'@rules_java//java/proto:toolchain_type'.

```txt
bazel build //src/...
ERROR: .../src/protobuf/io/bazel/rules_scala/BUILD:4:14:
  in @@_builtins//:common/java/proto/java_proto_library.bzl%bazel_java_proto_aspect
  aspect on proto_library rule
  //src/protobuf/io/bazel/rules_scala:diagnostics_proto:

Traceback (most recent call last):
  File "/virtual_builtins_bzl/common/java/proto/java_proto_library.bzl",
    line 53, column 53, in _bazel_java_proto_aspect_impl
  File "/virtual_builtins_bzl/common/proto/proto_common.bzl",
    line 364, column 17, in _find_toolchain
Error in fail: No toolchains registered for '@rules_java//java/proto:toolchain_type'.
ERROR: Analysis of target '//src/protobuf/io/bazel/rules_scala:diagnostics_proto' failed
ERROR: Analysis of target '//src/java/io/bazel/rulesscala/scalac:scalac' failed; build aborted: Analysis failed
```
  • Loading branch information
mbland committed Oct 4, 2024
1 parent 8d85252 commit d39d35c
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 0 deletions.
5 changes: 5 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,8 @@ build:windows --worker_quit_after_build --enable_runfiles
# Remove these upon completing Bzlmod compatibility work.
# - https://github.com/bazelbuild/rules_scala/issues/1482
build --noenable_bzlmod --enable_workspace

# Remove once proto toolchainization becomes the default
# - https://bazel.build/reference/command-line-reference#flag--incompatible_enable_proto_toolchain_resolution
# - https://docs.google.com/document/d/1CE6wJHNfKbUPBr7-mmk_0Yo3a4TaqcTPE0OWNuQkhPs/edit
build --incompatible_enable_proto_toolchain_resolution
9 changes: 9 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,15 @@ load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")

rules_proto_toolchains()

load("//scala:protoc_deps.bzl", "scala_protoc_setup")

scala_protoc_setup()

load("//scala:protoc_toolchain.bzl", "protoc_toolchains")

protoc_toolchains(name = "io_bazel_rules_scala_protoc")
register_toolchains("@io_bazel_rules_scala_protoc//:all")

load("//scala:scala_cross_version.bzl", "default_maven_server_urls")
load("//twitter_scrooge:twitter_scrooge.bzl", "twitter_scrooge")

Expand Down
17 changes: 17 additions & 0 deletions scala/protoc_deps.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
"""com_google_protobuf dependencies for the protoc_toolchain"""

load (
":protoc_toolchain.bzl",
"PROTOBUF_SHA256",
"PROTOBUF_URL",
"PROTOBUF_VERSION",
)
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

def scala_protoc_setup():
http_archive(
name = "com_google_protobuf",
sha256 = PROTOBUF_SHA256,
strip_prefix = "protobuf-%s" % PROTOBUF_VERSION,
url = PROTOBUF_URL,
)
148 changes: 148 additions & 0 deletions scala/protoc_toolchain.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
"""Prepares the precompiled protoc toolchain"""

PROTOBUF_VERSION = "28.2"
PROTOBUF_SHA256 = (
"b2340aa47faf7ef10a0328190319d3f3bee1b24f426d4ce8f4253b6f27ce16db"
)
PROTOBUF_URL = (
"https://github.com/protocolbuffers/protobuf/releases/download/" +
"v{version}/protobuf-{version}.tar.gz".format(version=PROTOBUF_VERSION)
)

_PROTOC_RELEASES_URL = "https://github.com/protocolbuffers/protobuf/releases"
_PROTOC_DOWNLOAD_URL = (
_PROTOC_RELEASES_URL + "/download/" +
"v{version}/protoc-{version}-".format(version=PROTOBUF_VERSION) +
"{platform}.zip"
)

_PROTOC_BUILDS = {
"linux-aarch_64": struct(
integrity = "sha256-kdglPNwPDw/FHCtpyAZ3mWYy9SWthFBL+ltO44rT5Jw=",
exec_compat = [
"@platforms//os:linux",
"@platforms//cpu:aarch64",
],
),
"linux-ppcle_64": struct(
integrity = "sha256-xcFrR2f/iGYJDuEOhwkDABBSAbXbHs7zgO3XPCM03Ng=",
exec_compat = [
"@platforms//os:linux",
"@platforms//cpu:ppc64le",
],
),
"linux-s390_64": struct(
integrity = "sha256-ESIsQ4+G6Hsv2vaKKmzB2ytiKBP9btiRUJ4vOw4F7hs=",
exec_compat = [
"@platforms//os:linux",
"@platforms//cpu:s390x",
],
),
"linux-x86_32": struct(
integrity = "sha256-ucjToo5Lq5WcwQ5smjjxfFlGc3Npv+AT9RqG19Ns//E=",
exec_compat = [
"@platforms//os:linux",
"@platforms//cpu:x86_32"
],
),
"linux-x86_64": struct(
integrity = "sha256-L+v9QrWc6Too63iQGaRwo90ESWGbwE+E2tEzPaJh3sE=",
exec_compat = [
"@platforms//os:linux",
"@platforms//cpu:x86_64"
],
),
"osx-aarch_64": struct(
integrity = "sha256-e7BI9ShBeJ2exhmDvgzkyeT7O9mhQ0YoILqaO+CgN5c=",
exec_compat = [
"@platforms//os:osx",
"@platforms//cpu:aarch64",
],
),
"osx-x86_64": struct(
integrity = "sha256-Iy8H0Sv0gGIHp57CxzeDAcUuby9+/dIcDdQW8L2hA+w=",
exec_compat = [
"@platforms//os:osx",
"@platforms//cpu:x86_64"
],
),
"win32": struct(
integrity = "sha256-V6hpbqvtUgl19PpGkUGwuC+iEIYj0YtyOqi+yae0u1g=",
exec_compat = [
"@platforms//os:windows",
"@platforms//cpu:x86_32"
],
),
"win64": struct(
integrity = "sha256-S94ZJx7XyrkANXDyjG5MTXGWPq8SEahr87sl2biVF3o=",
exec_compat = [
"@platforms//os:windows",
"@platforms//cpu:x86_64"
],
),
}

_PROTOC_BUILD_TEMPLATE="""load(
"@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl",
"proto_toolchain",
)
proto_toolchains = {{
{proto_toolchains}
}}
[
proto_toolchain(
name = build,
proto_compiler = ":%s/bin/protoc%s" % (
build, ".exe" if build.startswith("win") else ""
),
exec_compatible_with = specs,
)
for build, specs in proto_toolchains.items()
]
"""

_PROTOC_TOOLCHAIN_TEMPLATE= """ {build}: [\n{specs}\n ],"""

def _protoc_toolchains_impl(repository_ctx):
proto_toolchains = []

for build in repository_ctx.attr.protoc_builds:
protoc_build = _PROTOC_BUILDS[build]
repository_ctx.download_and_extract(
url = _PROTOC_DOWNLOAD_URL.format(platform = build),
output = build,
integrity = protoc_build.integrity,
)
proto_toolchains.append(_PROTOC_TOOLCHAIN_TEMPLATE.format(
build = '"%s"' % build,
specs = "\n".join([
' "%s",' % s for s in protoc_build.exec_compat
])
))

build_content = _PROTOC_BUILD_TEMPLATE.format(
name = repository_ctx.attr.name,
proto_toolchains = "\n".join(proto_toolchains)
)
repository_ctx.file("BUILD", content=build_content, executable=False)

protoc_toolchains = repository_rule(
implementation = _protoc_toolchains_impl,
attrs = {
"protoc_builds": attr.string_list(
doc = (
"os and arch identifiers for precompiled protoc release " +
"download filenames from " + _PROTOC_RELEASES_URL
),
default = [
"linux-aarch_64",
"linux-x86_64",
"osx-aarch_64",
"osx-x86_64",
"win64",
],
),
}
)

0 comments on commit d39d35c

Please sign in to comment.