Skip to content

Commit

Permalink
Delete all redundant environment variables from RBE configs.
Browse files Browse the repository at this point in the history 
These environment variables are not used in any repository rules initialized in RBE configs.

PiperOrigin-RevId: 676541760
  • Loading branch information
@tensorflower-gardener @copybara-github
tensorflower-gardener authored and copybara-github committed Sep 19, 2024
1 parent 0f627cc commit a41a6f1
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 124 deletions.
96 changes: 0 additions & 96 deletions tools/toolchains/remote_config/configs.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,41 +9,34 @@ def initialize_rbe_configs():

tensorflow_rbe_config(
name = "ubuntu20.04-clang_manylinux2014-cuda12.3-cudnn8.9",
compiler = "/usr/lib/llvm-18/bin/clang",
cuda_version = "12.3.2",
cudnn_version = "8.9.7.29",
os = "ubuntu20.04-manylinux2014-multipython",
)

tensorflow_rbe_config(
name = "ubuntu20.04-clang_manylinux2014-cuda12.3-cudnn9.1",
compiler = "/usr/lib/llvm-18/bin/clang",
cuda_version = "12.3.2",
cudnn_version = "9.1.1",
os = "ubuntu20.04-manylinux2014-multipython",
)

tensorflow_rbe_config(
name = "ubuntu20.04-gcc9_manylinux2014-cuda12.3-cudnn8.9",
compiler = "/dt9/usr/bin/gcc",
compiler_prefix = "/usr/bin",
cuda_version = "12.3.2",
cudnn_version = "8.9.7.29",
os = "ubuntu20.04-manylinux2014-multipython",
)

tensorflow_rbe_config(
name = "ubuntu22.04-clang_manylinux2014-cuda12.3-cudnn8.9",
compiler = "/usr/lib/llvm-18/bin/clang",
cuda_version = "12.3.2",
cudnn_version = "8.9.7.29",
os = "ubuntu22.04-manylinux2014-multipython",
)

tensorflow_rbe_config(
name = "ubuntu22.04-gcc9_manylinux2014-cuda12.3-cudnn8.9",
compiler = "/dt9/usr/bin/gcc",
compiler_prefix = "/usr/bin",
cuda_version = "12.3.2",
cudnn_version = "8.9.7.29",
os = "ubuntu22.04-manylinux2014-multipython",
Expand All @@ -70,25 +63,6 @@ def initialize_rbe_configs():
"sigbuild-r2.16-python3.11": "docker://gcr.io/tensorflow-sigs/build@sha256:842a5ba84d3658c5bf1f8a31e16284f7becc35409da0dfd71816afa3cd28d728",
"sigbuild-r2.16-python3.12": "docker://gcr.io/tensorflow-sigs/build@sha256:40fcd1d05c672672b599d9cb3784dcf379d6aa876f043b46c6ab18237d5d4e10",
},
# Unclear why LIBC is set to 2.19 here, and yet manylinux2010 is 2.12
# and manylinux2014 is 2.17.
env = {
"ABI_LIBC_VERSION": "glibc_2.19",
"ABI_VERSION": "gcc",
"BAZEL_COMPILER": "/dt9/usr/bin/gcc",
"BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
"BAZEL_TARGET_CPU": "k8",
"BAZEL_TARGET_LIBC": "glibc_2.19",
"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
"CC": "/dt9/usr/bin/gcc",
"CC_TOOLCHAIN_NAME": "linux_gnu_x86",
"CLEAR_CACHE": "1",
"GCC_HOST_COMPILER_PATH": "/dt9/usr/bin/gcc",
"GCC_HOST_COMPILER_PREFIX": "/usr/bin",
"HOST_CXX_COMPILER": "/dt9/usr/bin/gcc",
"HOST_C_COMPILER": "/dt9/usr/bin/gcc",
"TF_ENABLE_XLA": "1",
},
)

sigbuild_tf_configs(
Expand All @@ -99,23 +73,6 @@ def initialize_rbe_configs():
"sigbuild-r2.16-clang-python3.11": "docker://gcr.io/tensorflow-sigs/build@sha256:842a5ba84d3658c5bf1f8a31e16284f7becc35409da0dfd71816afa3cd28d728",
"sigbuild-r2.16-clang-python3.12": "docker://gcr.io/tensorflow-sigs/build@sha256:40fcd1d05c672672b599d9cb3784dcf379d6aa876f043b46c6ab18237d5d4e10",
},
# Unclear why LIBC is set to 2.19 here, and yet manylinux2010 is 2.12
# and manylinux2014 is 2.17.
env = {
"ABI_LIBC_VERSION": "glibc_2.19",
"ABI_VERSION": "gcc",
"BAZEL_COMPILER": "/usr/lib/llvm-17/bin/clang",
"BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
"BAZEL_TARGET_CPU": "k8",
"BAZEL_TARGET_LIBC": "glibc_2.19",
"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
"CC": "/usr/lib/llvm-17/bin/clang",
"CC_TOOLCHAIN_NAME": "linux_gnu_x86",
"CLEAR_CACHE": "1",
"HOST_CXX_COMPILER": "/usr/lib/llvm-17/bin/clang",
"HOST_C_COMPILER": "/usr/lib/llvm-17/bin/clang",
"TF_ENABLE_XLA": "1",
},
)

sigbuild_tf_configs(
Expand All @@ -126,25 +83,6 @@ def initialize_rbe_configs():
"sigbuild-r2.17-python3.11": "docker://gcr.io/tensorflow-sigs/build@sha256:b6f572a897a69fa3311773f949b9aa9e81bc393e4fbe2c0d56d8afb03a6de080",
"sigbuild-r2.17-python3.12": "docker://gcr.io/tensorflow-sigs/build@sha256:8b856ad736147bb9c8bc9e1ec2c8e1ab17d36397905da7a5b63dadeff9310f0c",
},
# Unclear why LIBC is set to 2.19 here, and yet manylinux2010 is 2.12
# and manylinux2014 is 2.17.
env = {
"ABI_LIBC_VERSION": "glibc_2.19",
"ABI_VERSION": "gcc",
"BAZEL_COMPILER": "/dt9/usr/bin/gcc",
"BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
"BAZEL_TARGET_CPU": "k8",
"BAZEL_TARGET_LIBC": "glibc_2.19",
"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
"CC": "/dt9/usr/bin/gcc",
"CC_TOOLCHAIN_NAME": "linux_gnu_x86",
"CLEAR_CACHE": "1",
"GCC_HOST_COMPILER_PATH": "/dt9/usr/bin/gcc",
"GCC_HOST_COMPILER_PREFIX": "/usr/bin",
"HOST_CXX_COMPILER": "/dt9/usr/bin/gcc",
"HOST_C_COMPILER": "/dt9/usr/bin/gcc",
"TF_ENABLE_XLA": "1",
},
)

sigbuild_tf_configs(
Expand All @@ -155,23 +93,6 @@ def initialize_rbe_configs():
"sigbuild-r2.17-clang-python3.11": "docker://gcr.io/tensorflow-sigs/build@sha256:2d737fc9fe931507a89927eee792b1bb934215e6aaae58b1941586e3400e2645",
"sigbuild-r2.17-clang-python3.12": "docker://gcr.io/tensorflow-sigs/build@sha256:45ea78e79305f91cdae5a26094f80233bba54bbfbc612623381012f097035b9a",
},
# Unclear why LIBC is set to 2.19 here, and yet manylinux2010 is 2.12
# and manylinux2014 is 2.17.
env = {
"ABI_LIBC_VERSION": "glibc_2.19",
"ABI_VERSION": "gcc",
"BAZEL_COMPILER": "/usr/lib/llvm-18/bin/clang",
"BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
"BAZEL_TARGET_CPU": "k8",
"BAZEL_TARGET_LIBC": "glibc_2.19",
"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
"CC": "/usr/lib/llvm-18/bin/clang",
"CC_TOOLCHAIN_NAME": "linux_gnu_x86",
"CLEAR_CACHE": "1",
"HOST_CXX_COMPILER": "/usr/lib/llvm-18/bin/clang",
"HOST_C_COMPILER": "/usr/lib/llvm-18/bin/clang",
"TF_ENABLE_XLA": "1",
},
)

sigbuild_tf_configs(
Expand All @@ -182,21 +103,4 @@ def initialize_rbe_configs():
"sigbuild-r2.17-clang-cudnn9-python3.11": "docker://gcr.io/tensorflow-sigs/build@sha256:daa5bdd802fe3def188e2200ed707c73d278f6f1930bf26c933d6ba041b0e027",
"sigbuild-r2.17-clang-cudnn9-python3.12": "docker://gcr.io/tensorflow-sigs/build@sha256:23e477895dd02e45df1056d4a0a9c4229dec3a20c23fb2f3fb5832ecbd0a29bc",
},
# Unclear why LIBC is set to 2.19 here, and yet manylinux2010 is 2.12
# and manylinux2014 is 2.17.
env = {
"ABI_LIBC_VERSION": "glibc_2.19",
"ABI_VERSION": "gcc",
"BAZEL_COMPILER": "/usr/lib/llvm-18/bin/clang",
"BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
"BAZEL_TARGET_CPU": "k8",
"BAZEL_TARGET_LIBC": "glibc_2.19",
"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
"CC": "/usr/lib/llvm-18/bin/clang",
"CC_TOOLCHAIN_NAME": "linux_gnu_x86",
"CLEAR_CACHE": "1",
"HOST_CXX_COMPILER": "/usr/lib/llvm-18/bin/clang",
"HOST_C_COMPILER": "/usr/lib/llvm-18/bin/clang",
"TF_ENABLE_XLA": "1",
},
)
32 changes: 4 additions & 28 deletions tools/toolchains/remote_config/rbe_config.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,34 +9,13 @@ def _container_image_uri(container_name):
container = containers[container_name]
return "docker://%s/%s@%s" % (container["registry"], container["repository"], container["digest"])

def _tensorflow_rbe_config(name, compiler, os, rocm_version = None, cuda_version = None, cudnn_version = None, compiler_prefix = None):
def _tensorflow_rbe_config(name, os, rocm_version = None, cuda_version = None, cudnn_version = None):
if cuda_version != None and rocm_version != None:
fail("Specifying both cuda_version and rocm_version is not supported.")

env = {
"ABI_VERSION": "gcc",
"ABI_LIBC_VERSION": "glibc_2.19",
"BAZEL_COMPILER": compiler,
"BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
"BAZEL_TARGET_LIBC": "glibc_2.19",
"BAZEL_TARGET_CPU": "k8",
"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
"CC_TOOLCHAIN_NAME": "linux_gnu_x86",
"CC": compiler,
"CLEAR_CACHE": "1",
"HOST_CXX_COMPILER": compiler,
"HOST_C_COMPILER": compiler,
}
env = {}

if cuda_version != None:
# The cuda toolchain currently contains its own C++ toolchain definition,
# so we do not fetch local_config_cc.
env.update({
"TF_ENABLE_XLA": "1",
"GCC_HOST_COMPILER_PATH": compiler if not compiler.endswith("clang") else "",
"GCC_HOST_COMPILER_PREFIX": compiler_prefix if compiler_prefix != None else "/usr/bin",
})

cuda_version_in_container = ".".join(cuda_version.split(".")[:2])
cudnn_version_in_container = ".".join(cudnn_version.split(".")[:2])
container_name = "cuda%s-cudnn%s-%s" % (
Expand All @@ -49,13 +28,11 @@ def _tensorflow_rbe_config(name, compiler, os, rocm_version = None, cuda_version
"container-image": container_image,
"Pool": "default",
}

elif rocm_version != None:
# The rocm toolchain currently contains its own C++ toolchain definition,
# so we do not fetch local_config_cc.
env.update({
"TF_NEED_ROCM": "1",
"TF_ENABLE_XLA": "0",
})

container_name = "rocm-%s" % (os)
Expand Down Expand Up @@ -121,9 +98,8 @@ tensorflow_local_config = _tensorflow_local_config

# Streamlined platform configuration for the SIG Build containers.
# See //tensorflow/tools/tf_sig_build_dockerfiles
# These containers do not support ROCm and all have CUDA. We demand that the configuration
# provide all the env variables to remove hidden logic.
def sigbuild_tf_configs(name_container_map, env):
# These containers do not support ROCm and all have CUDA.
def sigbuild_tf_configs(name_container_map):
for name, container in name_container_map.items():
exec_properties = {
"container-image": container,
Expand Down

0 comments on commit a41a6f1

Please sign in to comment.