adding easyconfigs: TensorFlow-2.3.0-fosscuda-2019b-Python-3.7.4.eb a…

…nd patches: TensorFlow-2.3.0-fix_protoc_build.patch
easybuilders · Sep 4, 2020 · 4ae5f74 · 4ae5f74
1 parent aa965ab
commit 4ae5f74
Show file tree

Hide file tree

Showing 5 changed files with 346 additions and 0 deletions.
diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.1.0_fix-system-jsoncpp.patch b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.1.0_fix-system-jsoncpp.patch
@@ -0,0 +1,15 @@
+diff --git a/third_party/systemlibs/jsoncpp.BUILD b/third_party/systemlibs/jsoncpp.BUILD
+index 7d54f9289b..ba921bdfb0 100644
+--- a/third_party/systemlibs/jsoncpp.BUILD
++++ b/third_party/systemlibs/jsoncpp.BUILD
+@@ -32,8 +32,8 @@ genrule(
+
+ cc_library(
+     name = "jsoncpp",
+-    hdrs = HEADERS,
+-    includes = ["."],
++    #hdrs = HEADERS,
++    #includes = ["."],
+     linkopts = ["-ljsoncpp"],
+     visibility = ["//visibility:public"],
+ )
diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.1.0_fix-system-nasm.patch b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.1.0_fix-system-nasm.patch
@@ -0,0 +1,28 @@
+commit 5b3b9c7fe7501acd6bd69abe26fd3f9e0f1df4ef
+Author: Alexander Grund <[email protected]>
+Date:   Wed Aug 12 15:48:17 2020 +0200
+
+    Use nasmlink genrule
+
+    Avoids cyclic dependency as the name and src must not be the same
+
+diff --git a/third_party/nasm/BUILD.system b/third_party/nasm/BUILD.system
+index 7f74da7595..52f608187f 100644
+--- a/third_party/nasm/BUILD.system
++++ b/third_party/nasm/BUILD.system
+@@ -5,8 +5,14 @@ filegroup(
+     visibility = ["//visibility:public"],
+ )
+
++genrule(
++    name = "lnnasmlink",
++    outs = ["nasmlink"],
++    cmd = "ln -s $$(which nasm) $@",
++)
++
+ sh_binary(
+     name = "nasm",
+-    srcs = ["nasm"],
++    srcs = ["nasmlink"],
+     visibility = ["@libjpeg_turbo//:__pkg__"],
+ )
diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.3.0-fosscuda-2019b-Python-3.7.4.eb b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.3.0-fosscuda-2019b-Python-3.7.4.eb
@@ -0,0 +1,166 @@
+easyblock = 'PythonBundle'
+
+name = 'TensorFlow'
+version = '2.3.0'
+versionsuffix = '-Python-%(pyver)s'
+
+homepage = 'https://www.tensorflow.org/'
+description = "An open-source software library for Machine Intelligence"
+
+toolchain = {'name': 'fosscuda', 'version': '2019b'}
+toolchainopts = {'usempi': True, 'pic': True}
+
+builddependencies = [
+    ('Bazel', '3.4.1'),
+    ('protobuf', '3.10.0'),
+    # git 2.x required, see also https://github.com/tensorflow/tensorflow/issues/29053
+    ('git', '2.23.0', '-nodocs'),
+    ('pybind11', '2.4.3', versionsuffix),
+]
+dependencies = [
+    ('cuDNN', '7.6.4.38'),
+    ('NCCL', '2.4.8'),
+    ('Python', '3.7.4'),
+    ('h5py', '2.10.0', versionsuffix),
+    ('cURL', '7.66.0'),
+    ('double-conversion', '3.1.4'),
+    ('flatbuffers', '1.12.0'),
+    ('giflib', '5.2.1'),
+    ('hwloc', '1.11.12'),
+    ('ICU', '64.2'),
+    ('JsonCpp', '1.9.3'),
+    ('libjpeg-turbo', '2.0.3'),
+    ('LMDB', '0.9.24'),
+    ('NASM', '2.14.02'),
+    ('nsync', '1.24.0'),
+    ('SQLite', '3.29.0'),
+    ('PCRE', '8.43'),
+    ('protobuf-python', '3.10.0', versionsuffix),
+    ('libpng', '1.6.37'),
+    ('snappy', '1.1.7'),
+    ('SWIG', '4.0.1'),
+    ('zlib', '1.2.11'),
+    # TF 2.3 requires SciPy 1.4.1 due to potential crashes with other versions
+    # See https://github.com/tensorflow/tensorflow/commit/54daf3c5700897a6062313983933ca28e92c410d
+    # Add at bottom so it will be loaded after any dependency loading the SciPy-Bundle
+    ('scipy', '1.4.1', versionsuffix),
+]
+
+exts_default_options = {
+    'source_urls': [PYPI_SOURCE],
+    'sanity_pip_check': True,
+}
+use_pip = True
+
+# Dependencies created and updated using findPythonDeps.sh:
+# https://gist.github.com/Flamefire/49426e502cd8983757bd01a08a10ae0d
+exts_list = [
+    ('Markdown', '3.2.2', {
+        'checksums': ['1fafe3f1ecabfb514a5285fca634a53c1b32a81cb0feb154264d55bf2ff22c17'],
+    }),
+    ('pyasn1-modules', '0.2.8', {
+        'checksums': ['905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e'],
+    }),
+    ('rsa', '4.6', {
+        'checksums': ['109ea5a66744dd859bf16fe904b8d8b627adafb9408753161e766a92e7d681fa'],
+    }),
+    ('cachetools', '4.1.1', {
+        'checksums': ['bbaa39c3dede00175df2dc2b03d0cf18dd2d32a7de7beb68072d13043c9edb20'],
+    }),
+    ('google-auth', '1.20.0', {
+        'modulename': 'google.auth',
+        'checksums': ['c6e9735a2ee829a75b546702e460489db5cc35567a27fabd70b7c459f11efd58'],
+    }),
+    ('oauthlib', '3.1.0', {
+        'checksums': ['bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889'],
+    }),
+    ('requests-oauthlib', '1.3.0', {
+        'checksums': ['b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a'],
+    }),
+    ('google-auth-oauthlib', '0.4.1', {
+        'checksums': ['88d2cd115e3391eb85e1243ac6902e76e77c5fe438b7276b297fbe68015458dd'],
+    }),
+    ('Werkzeug', '1.0.1', {
+        'checksums': ['6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c'],
+    }),
+    ('absl-py', '0.9.0', {
+        'modulename': 'absl',
+        'checksums': ['75e737d6ce7723d9ff9b7aa1ba3233c34be62ef18d5859e706b8fdc828989830'],
+    }),
+    ('astunparse', '1.6.3', {
+        'checksums': ['5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872'],
+    }),
+    ('grpcio', '1.30.0', {
+        'modulename': 'grpc',
+        'checksums': ['e8f2f5d16e0164c415f1b31a8d9a81f2e4645a43d1b261375d6bab7b0adf511f'],
+    }),
+    ('tensorboard-plugin-wit', '1.7.0', {
+        'source_tmpl': 'tensorboard_plugin_wit-%(version)s-py3-none-any.whl',
+        'unpack_sources': False,
+        'checksums': ['ee775f04821185c90d9a0e9c56970ee43d7c41403beb6629385b39517129685b'],
+    }),
+    ('tensorboard', version, {
+        'source_tmpl': 'tensorboard-%(version)s-py3-none-any.whl',
+        'unpack_sources': False,
+        'checksums': ['d34609ed83ff01dd5b49ef81031cfc9c166bba0dabd60197024f14df5e8eae5e'],
+    }),
+    ('google-pasta', '0.2.0', {
+        'modulename': 'pasta',
+        'checksums': ['c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e'],
+    }),
+    ('termcolor', '1.1.0', {
+        'checksums': ['1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b'],
+    }),
+    ('tensorflow-estimator', version, {
+        'source_tmpl': 'tensorflow_estimator-%(version)s-py2.py3-none-any.whl',
+        'unpack_sources': False,
+        'checksums': ['b75e034300ccb169403cf2695adf3368da68863aeb0c14c3760064c713d5c486'],
+    }),
+    ('astor', '0.8.0', {
+        'checksums': ['37a6eed8b371f1228db08234ed7f6cfdc7817a3ed3824797e20cbb11dc2a7862'],
+    }),
+    ('Keras-Applications', '1.0.8', {
+        'source_tmpl': 'Keras_Applications-%(version)s.tar.gz',
+        'checksums': ['5579f9a12bcde9748f4a12233925a59b93b73ae6947409ff34aa2ba258189fe5'],
+    }),
+    ('gast', '0.3.3', {
+        'checksums': ['b881ef288a49aa81440d2c5eb8aeefd4c2bb8993d5f50edae7413a85bfdb3b57'],
+    }),
+    ('opt-einsum', '3.3.0', {
+        'source_tmpl': 'opt_einsum-%(version)s.tar.gz',
+        'checksums': ['59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549'],
+    }),
+    ('wrapt', '1.12.1', {
+        'checksums': ['b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7'],
+    }),
+    ('Keras-Preprocessing', '1.1.2', {
+        'source_tmpl': 'Keras_Preprocessing-%(version)s.tar.gz',
+        'checksums': ['add82567c50c8bc648c14195bf544a5ce7c1f76761536956c3d2978970179ef3'],
+    }),
+    (name, version, {
+        'patches': [
+            'TensorFlow-2.1.0_fix-cuda-build.patch',
+            'TensorFlow-2.1.0_fix-system-jsoncpp.patch',
+            'TensorFlow-2.1.0_fix-system-nasm.patch',
+            'TensorFlow-2.3.0_fix-system-protobuf.patch',
+            'TensorFlow-2.3.0_fix-protoc-build.patch',
+        ],
+        'source_tmpl': 'v%(version)s.tar.gz',
+        'source_urls': ['https://github.com/tensorflow/tensorflow/archive/'],
+        'test_script': 'TensorFlow-2.x_mnist-test.py',
+        'checksums': [
+            '2595a5c401521f20a2734c4e5d54120996f8391f00bb62a57267d930bce95350',  # v2.3.0.tar.gz
+            '78c20aeaa7784b8ceb46238a81e8c2461137d28e0b576deeba8357d23fbe1f5a',  # TensorFlow-2.1.0_fix-cuda-build.patch
+            # TensorFlow-2.1.0_fix-system-jsoncpp.patch
+            'd0c8ca54a9e2c232908016e08b982dbb63765de3472253cba5ae38d823d5f156',
+            # TensorFlow-2.1.0_fix-system-nasm.patch
+            '6671e40d60edaf1e57b1861aa3b2178d48f9b7dfb5b5c0d44db541116f848f2a',
+            # TensorFlow-2.3.0_fix-system-protobuf.patch
+            '576a5f1de3039b6cecbc6ef67c9fcecc117b779f0d18f01e2af541f6030e9518',
+            # TensorFlow-2.3.0_fix-protoc-build.patch
+            '78a8390ea5e06dfcfb25f6434721abbe678e37c4ad7f052aad766c4df399bd53',
+        ],
+    }),
+]
+
+moduleclass = 'lib'
diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.3.0_fix-protoc-build.patch b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.3.0_fix-protoc-build.patch
@@ -0,0 +1,37 @@
+commit c8231eb55bdbff94c44e4b554321017fbd01bda9
+Author: Alexander Grund <[email protected]>
+Date:   Wed Jul 29 16:57:39 2020 +0200
+
+    Fix environment for protobuf compilation
+
+    Patch using upstream https://github.com/grpc/grpc/pull/23664
+
+https://github.com/tensorflow/tensorflow/pull/41889
+diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
+index 24446d846c..ee63c4a811 100755
+--- a/tensorflow/workspace.bzl
++++ b/tensorflow/workspace.bzl
+@@ -687,6 +687,7 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
+         sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f",
+         strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd",
+         system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"),
++        patch_file = clean_dep("//third_party/grpc:generate_cc_env_fix.patch"),
+         system_link_files = {
+             "//third_party/systemlibs:BUILD": "bazel/BUILD",
+             "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD",
+diff --git a/third_party/grpc/generate_cc_env_fix.patch b/third_party/grpc/generate_cc_env_fix.patch
+new file mode 100644
+index 0000000000..51832fe962
+--- /dev/null
++++ b/third_party/grpc/generate_cc_env_fix.patch
+@@ -0,0 +1,10 @@
++--- a/bazel/generate_cc.bzl
+++++ b/bazel/generate_cc.bzl
++@@ -141,6 +141,7 @@ def generate_cc_impl(ctx):
++         outputs = out_files,
++         executable = ctx.executable._protoc,
++         arguments = arguments,
+++        use_default_shell_env = True,
++     )
++
++     return struct(files = depset(out_files))
diff --git a/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.3.0_fix-system-protobuf.patch b/easybuild/easyconfigs/t/TensorFlow/TensorFlow-2.3.0_fix-system-protobuf.patch
@@ -0,0 +1,100 @@
+Based on: https://github.com/tensorflow/tensorflow/pull/42591
+Modified by Alexander Grund (TU Dresden)
+
+From ab6c3bf9d98d7627509865e91913c25a1f8cc693 Mon Sep 17 00:00:00 2001
+From: Steven Clarkson <[email protected]>
+Date: Sat, 22 Aug 2020 16:57:25 -0700
+Subject: [PATCH] Add missing systemlib protobuf libraries
+
+---
+ third_party/systemlibs/protobuf.BUILD | 83 +++++++++++++++++++++++++++
+ 1 file changed, 83 insertions(+)
+
+diff --git a/third_party/systemlibs/protobuf.BUILD b/third_party/systemlibs/protobuf.BUILD
+index 118135d129005..ef3e0c93c6483 100644
+--- a/third_party/systemlibs/protobuf.BUILD
++++ b/third_party/systemlibs/protobuf.BUILD
+@@ -31,6 +31,7 @@ HEADERS = [
+     "google/protobuf/io/zero_copy_stream.h",
+     "google/protobuf/io/zero_copy_stream_impl_lite.h",
+     "google/protobuf/map.h",
++    "google/protobuf/port_def.inc",
+     "google/protobuf/repeated_field.h",
+     "google/protobuf/text_format.h",
+     "google/protobuf/timestamp.pb.h",
+@@ -102,3 +99,75 @@ py_library(
+     srcs_version = "PY2AND3",
+     visibility = ["//visibility:public"],
+ )
++
++proto_library(
++    name = "any_proto",
++    srcs = ["google/protobuf/any.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "api_proto",
++    srcs = ["google/protobuf/api.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "compiler_plugin_proto",
++    srcs = ["google/protobuf/compiler/plugin.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "descriptor_proto",
++    srcs = ["google/protobuf/descriptor.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "duration_proto",
++    srcs = ["google/protobuf/duration.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "empty_proto",
++    srcs = ["google/protobuf/empty.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "field_mask_proto",
++    srcs = ["google/protobuf/field_mask.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "source_context_proto",
++    srcs = ["google/protobuf/source_context.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "struct_proto",
++    srcs = ["google/protobuf/struct.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "timestamp_proto",
++    srcs = ["google/protobuf/timestamp.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "type_proto",
++    srcs = ["google/protobuf/type.proto"],
++    visibility = ["//visibility:public"],
++)
++
++proto_library(
++    name = "wrappers_proto",
++    srcs = ["google/protobuf/wrappers.proto"],
++    visibility = ["//visibility:public"],
++)