diff --git a/.bazelrc b/.bazelrc index 1de0109..b98cf42 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,7 +16,6 @@ build --cxxopt="-std=c++14" build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" build --auto_output_filter=subpackages build --copt="-Wall" --copt="-Wno-sign-compare" -build --linkopt="-lrt -lm" # TF isn't built in dbg mode, so our dbg builds will segfault due to inconsistency # of defines when using tf's headers. In particular in refcount.h. @@ -24,3 +23,5 @@ build --cxxopt="-DNDEBUG" # Options from ./configure try-import %workspace%/.reverb.bazelrc + +build --features=-supports_dynamic_linker diff --git a/configure.py b/configure.py index 5f28aa5..abe3adf 100644 --- a/configure.py +++ b/configure.py @@ -69,6 +69,13 @@ def main(): reset_configure_bazelrc() setup_python(environ_cp) + write_to_bazelrc('') + if sys.platform == 'darwin': + write_to_bazelrc('# https://github.com/googleapis/google-cloud-cpp-spanner/issues/1003') + write_to_bazelrc('build --copt=-DGRPC_BAZEL_BUILD') + else: + write_to_bazelrc('build --linkopt="-lrt -lm"') + def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, var_default): diff --git a/docker/dev.dockerfile b/docker/dev.dockerfile index 95d962e..5e689c8 100644 --- a/docker/dev.dockerfile +++ b/docker/dev.dockerfile @@ -81,6 +81,7 @@ ARG pip_dependencies=' \ numpy \ oauth2client \ pandas \ + platform \ portpicker' diff --git a/docker/release.dockerfile b/docker/release.dockerfile index 88830ef..90d1bca 100644 --- a/docker/release.dockerfile +++ b/docker/release.dockerfile @@ -64,6 +64,7 @@ ARG pip_dependencies=' \ numpy \ oauth2client \ pandas \ + platform \ portpicker' # TODO(b/154930404): Update to 2.2.0 once it's out. May need to diff --git a/oss_build.sh b/oss_build.sh index c453546..a5d63d8 100644 --- a/oss_build.sh +++ b/oss_build.sh @@ -89,18 +89,29 @@ for python_version in $PYTHON_VERSIONS; do fi if [ "$python_version" = "3.6" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3.6 && export PYTHON_LIB_PATH=/usr/local/lib/python3.6/dist-packages + ABI=cp36 elif [ "$python_version" = "3.7" ]; then - export PYTHON_BIN_PATH=/usr/local/bin/python3.7 && export PYTHON_LIB_PATH=/usr/local/lib/python3.7/dist-packages ABI=cp37 elif [ "$python_version" = "3.8" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3.8 && export PYTHON_LIB_PATH=/usr/local/lib/python3.8/dist-packages ABI=cp38 else echo "Error unknown --python. Only [3.6|3.7|3.8]" exit 1 fi + export PYTHON_BIN_PATH=`which python${python_version}` + export PYTHON_LIB_PATH=`${PYTHON_BIN_PATH} -c 'import site; print(site.getsitepackages()[0])'` + + if [ "$(uname)" = "Darwin" ]; then + bazel_config="" + version=`sw_vers -productVersion | sed 's/\./_/g' | cut -d"_" -f1,2` + PLATFORM="macosx_${version}_"`uname -m` + else + bazel_config="--config=manylinux2010" + bazel_config="" + PLATFORM="manylinux2010_x86_64" + fi + # Configures Bazel environment for selected Python version. $PYTHON_BIN_PATH configure.py @@ -111,20 +122,20 @@ for python_version in $PYTHON_VERSIONS; do # someone's system unexpectedly. We are executing the python tests after # installing the final package making this approach satisfactory. # TODO(b/157223742): Execute Python tests as well. - bazel test -c opt --copt=-mavx --config=manylinux2010 --test_output=errors //reverb/cc/... + bazel test -c opt --copt=-mavx ${bazel_config} --test_output=errors //reverb/cc/... # Builds Reverb and creates the wheel package. - bazel build -c opt --copt=-mavx --config=manylinux2010 reverb/pip_package:build_pip_package - ./bazel-bin/reverb/pip_package/build_pip_package --dst $OUTPUT_DIR $PIP_PKG_EXTRA_ARGS + bazel build -c opt --copt=-mavx ${bazel_config} reverb/pip_package:build_pip_package + ./bazel-bin/reverb/pip_package/build_pip_package --dst $OUTPUT_DIR $PIP_PKG_EXTRA_ARGS --platform "${PLATFORM}" # Installs pip package. - $PYTHON_BIN_PATH -mpip install ${OUTPUT_DIR}*${ABI}*.whl + $PYTHON_BIN_PATH -mpip install --force-reinstall ${OUTPUT_DIR}*${ABI}*.whl if [ "$PYTHON_TESTS" = "true" ]; then echo "Run Python tests..." set +e - bash run_python_tests.sh |& tee ./unittest_log.txt + bash run_python_tests.sh 2>&1 | tee ./unittest_log.txt UNIT_TEST_ERROR_CODE=$? set -e if [[ $UNIT_TEST_ERROR_CODE != 0 ]]; then diff --git a/reverb/BUILD b/reverb/BUILD index 3c5b10d..dc1269d 100644 --- a/reverb/BUILD +++ b/reverb/BUILD @@ -16,6 +16,15 @@ licenses(["notice"]) exports_files(["LICENSE"]) +config_setting( + name = "macos", + values = { + "apple_platform_type": "macos", + "cpu": "darwin", + }, + visibility = ["//visibility:public"], +) + reverb_pytype_strict_library( name = "reverb", srcs = ["__init__.py"], diff --git a/reverb/cc/platform/default/build_rules.bzl b/reverb/cc/platform/default/build_rules.bzl index aab558b..d910d86 100644 --- a/reverb/cc/platform/default/build_rules.bzl +++ b/reverb/cc/platform/default/build_rules.bzl @@ -107,12 +107,12 @@ def reverb_cc_proto_library(name, srcs = [], deps = [], **kwargs): name = "{}_static".format(name), srcs = gen_srcs, hdrs = gen_hdrs, - deps = depset(deps + reverb_tf_deps()), + deps = depset([dep.replace(":", ":libxxx") + ".so" for dep in deps] + reverb_tf_deps()), alwayslink = 1, **kwargs ) native.cc_binary( - name = "lib{}.so".format(name), + name = "libxxx{}.so".format(name), deps = ["{}_static".format(name)], linkshared = 1, **kwargs @@ -120,7 +120,7 @@ def reverb_cc_proto_library(name, srcs = [], deps = [], **kwargs): native.cc_library( name = name, hdrs = gen_hdrs, - srcs = ["lib{}.so".format(name)], + srcs = ["libxxx{}.so".format(name)], deps = depset(deps + reverb_tf_deps()), alwayslink = 1, **kwargs @@ -294,6 +294,16 @@ def reverb_gen_op_wrapper_py(name, out, kernel_lib, linkopts = [], **kwargs): fail("Argument out must end with '.py', but saw: {}".format(out)) module_name = "lib{}_gen_op".format(name) + exported_symbols_file = "%s-exported-symbols.lds" % module_name + # gen_client_ops -> reverb_client + symbol = "reverb_{}".format(name.split('_')[1]) + native.genrule( + name = module_name + "_exported_symbols", + outs = [exported_symbols_file], + cmd = "echo '*%s*' >$@" % symbol, + output_licenses = ["unencumbered"], + visibility = ["//visibility:private"], + ) version_script_file = "%s-version-script.lds" % module_name native.genrule( name = module_name + "_version_script", @@ -304,16 +314,23 @@ def reverb_gen_op_wrapper_py(name, out, kernel_lib, linkopts = [], **kwargs): ) native.cc_binary( name = "{}.so".format(module_name), - deps = [kernel_lib] + reverb_tf_deps() + [version_script_file], + deps = [kernel_lib] + reverb_tf_deps() + [ + exported_symbols_file, + version_script_file + ], copts = tf_copts() + [ "-fno-strict-aliasing", # allow a wider range of code [aliasing] to compile. "-fvisibility=hidden", # avoid symbol clashes between DSOs. ], linkshared = 1, - linkopts = linkopts + _rpath_linkopts(module_name) + [ - "-Wl,--version-script", - "$(location %s)" % version_script_file, - ], + linkopts = linkopts + _rpath_linkopts(module_name) + select({ + "//reverb:macos": [ + "-Wl,-exported_symbols_list,$(location %s)" % exported_symbols_file, + ], + "//conditions:default": [ + "-Wl,--version-script,$(location %s)" % version_script_file, + ], + }), **kwargs ) native.genrule( @@ -444,10 +461,15 @@ def reverb_pybind_extension( "-fexceptions", # pybind relies on exceptions, required to compile. "-fvisibility=hidden", # avoid pybind symbol clashes between DSOs. ], - linkopts = linkopts + _rpath_linkopts(module_name) + [ - "-Wl,--version-script", - "$(location %s)" % version_script_file, - ], + linkopts = linkopts + _rpath_linkopts(module_name) + + select({"//reverb:macos": [ + "-Wl,-exported_symbols_list,$(location %s)" % exported_symbols_file, + ], + "//conditions:default": [ + "-Wl,--version-script", + "$(location %s)" % version_script_file, + ], + }), deps = depset(deps + [ exported_symbols_file, version_script_file, diff --git a/reverb/cc/platform/default/repo.bzl b/reverb/cc/platform/default/repo.bzl index 1daac52..455307b 100644 --- a/reverb/cc/platform/default/repo.bzl +++ b/reverb/cc/platform/default/repo.bzl @@ -2,6 +2,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +def is_darwin(ctx): + if ctx.os.name.lower().find("mac") != -1: + return True + return False + # Sanitize a dependency so that it works correctly from code that includes # reverb as a submodule. def clean_dep(dep): @@ -89,7 +94,6 @@ def _find_python_solib_path(repo_ctx): fail("Could not locate python shared library path:\n{}" .format(exec_result.stderr)) version = exec_result.stdout.splitlines()[-1] - basename = "lib{}.so".format(version) exec_result = repo_ctx.execute( ["{}-config".format(version), "--configdir"], quiet = True, @@ -97,7 +101,14 @@ def _find_python_solib_path(repo_ctx): if exec_result.return_code != 0: fail("Could not locate python shared library path:\n{}" .format(exec_result.stderr)) - solib_dir = exec_result.stdout.splitlines()[-1] + + if is_darwin(repo_ctx): + basename = "lib{}.dylib".format(version) + solib_dir = "/".join(exec_result.stdout.splitlines()[-1].split("/")[:-2]) + else: + basename = "lib{}.so".format(version) + solib_dir = exec_result.stdout.splitlines()[-1] + full_path = repo_ctx.path("{}/{}".format(solib_dir, basename)) if not full_path.exists: fail("Unable to find python shared library file:\n{}/{}" @@ -220,17 +231,21 @@ filegroup( def _tensorflow_solib_repo_impl(repo_ctx): tf_lib_path = _find_tf_lib_path(repo_ctx) repo_ctx.symlink(tf_lib_path, "tensorflow_solib") + if is_darwin(repo_ctx): + suffix = "2.dylib" + else: + suffix = "so.2" + repo_ctx.file( "BUILD", content = """ cc_library( name = "framework_lib", - srcs = ["tensorflow_solib/libtensorflow_framework.so.2"], + srcs = ["tensorflow_solib/libtensorflow_framework.{suffix}"], deps = ["@python_includes", "@python_includes//:numpy_includes"], visibility = ["//visibility:public"], ) -""", - ) +""".format(suffix=suffix)) def _python_includes_repo_impl(repo_ctx): python_include_path = _find_python_include_path(repo_ctx) @@ -243,6 +258,12 @@ def _python_includes_repo_impl(repo_ctx): python_solib.basename, ) + if is_darwin(repo_ctx): + # Fix Fatal Python error: PyThreadState_Get: no current thread + python_includes_srcs = "" + else: + python_includes_srcs = 'srcs = ["%s"],' % python_solib.basename + # Note, "@python_includes" is a misnomer since we include the # libpythonX.Y.so in the srcs, so we can get access to python's various # symbols at link time. @@ -252,7 +273,7 @@ def _python_includes_repo_impl(repo_ctx): cc_library( name = "python_includes", hdrs = glob(["python_includes/**/*.h"]), - srcs = ["{}"], + {srcs} includes = ["python_includes"], visibility = ["//visibility:public"], ) @@ -262,7 +283,7 @@ cc_library( includes = ["numpy_includes"], visibility = ["//visibility:public"], ) -""".format(python_solib.basename), +""".format(srcs=python_includes_srcs), executable = False, ) @@ -330,8 +351,14 @@ def _reverb_protoc_archive(ctx): sha256 = "" version = override_version + if is_darwin(ctx): + platform = "osx" + sha256 = "" + else: + platform = "linux" + urls = [ - "https://github.com/protocolbuffers/protobuf/releases/download/v%s/protoc-%s-linux-x86_64.zip" % (version, version), + "https://github.com/protocolbuffers/protobuf/releases/download/v%s/protoc-%s-%s-x86_64.zip" % (version, version, platform), ] ctx.download_and_extract( url = urls, diff --git a/reverb/pip_package/build_pip_package.sh b/reverb/pip_package/build_pip_package.sh index f15b83d..c474e25 100755 --- a/reverb/pip_package/build_pip_package.sh +++ b/reverb/pip_package/build_pip_package.sh @@ -32,7 +32,7 @@ function build_wheel() { pushd ${TMPDIR} > /dev/null echo $(date) : "=== Building wheel" - "${PYTHON_BIN_PATH}" setup.py bdist_wheel ${PKG_NAME_FLAG} ${RELEASE_FLAG} ${TF_VERSION_FLAG} --plat manylinux2010_x86_64 > /dev/null + "${PYTHON_BIN_PATH}" setup.py bdist_wheel ${PKG_NAME_FLAG} ${RELEASE_FLAG} ${TF_VERSION_FLAG} --plat ${PLATFORM} > /dev/null DEST=${TMPDIR}/dist/ if [[ ! "$TMPDIR" -ef "$DESTDIR" ]]; then mkdir -p ${DESTDIR} @@ -71,6 +71,19 @@ function prepare_src() { # must remain where they are for TF to find them. find "${TMPDIR}/reverb/cc" -type d -name ops -prune -o -name '*.so' \ -exec mv {} "${TMPDIR}/reverb" \; + + # Copy darwin libs over so they can be loaded at runtime + so_lib_dir=$(ls $RUNFILES | grep solib) || true + if [ -n "${so_lib_dir}" ]; then + mkdir -p "${TMPDIR}/${so_lib_dir}" + proto_so_dir=$(ls ${RUNFILES}/${so_lib_dir} | grep proto) || true + for dir in ${proto_so_dir}; do + echo "===== DIR = $dir" + cp -R ${RUNFILES}/${so_lib_dir}/${dir} "${TMPDIR}/${so_lib_dir}" + done + + cp -r $TMPDIR/${so_lib_dir} `dirname $PYTHON_LIB_PATH` + fi } function usage() { @@ -80,6 +93,7 @@ function usage() { echo " --release build a release version" echo " --dst path to copy the .whl into." echo " --tf-version tensorflow version dependency passed to setup.py." + echo " --plat platform." echo "" exit 1 } @@ -91,6 +105,8 @@ function main() { # This is where the source code is copied and where the whl will be built. DST_DIR="" + PLATFORM="manylinux2010_x86_64" + while true; do if [[ "$1" == "--help" ]]; then usage @@ -103,6 +119,9 @@ function main() { elif [[ "$1" == "--tf-version" ]]; then shift TF_VERSION_FLAG="--tf-version $1" + elif [[ "$1" == "--platform" ]]; then + shift + PLATFORM=$1 fi if [[ -z "$1" ]]; then diff --git a/reverb/pip_package/setup.py b/reverb/pip_package/setup.py index 441823e..9cbc40e 100644 --- a/reverb/pip_package/setup.py +++ b/reverb/pip_package/setup.py @@ -112,6 +112,16 @@ def run_setup(self): long_description = f.read() version, project_name = self._get_version() + + so_lib_paths = [ + i for i in os.listdir('.') + if os.path.isdir(i) and fnmatch.fnmatch(i, '_solib_*') + ] + + matches = [] + for path in so_lib_paths: + matches.extend(['../' + x for x in find_files('*', path) if '.py' not in x]) + setup( name=project_name, version=version, @@ -126,6 +136,9 @@ def run_setup(self): packages=find_packages(), headers=list(find_files('*.proto', 'reverb')), include_package_data=True, + package_data={ + 'reverb': matches, + }, install_requires=self._get_required_packages(), extras_require={ 'tensorflow': self._get_tensorflow_packages(),