From 5fd4d1ce770f87fe032b8deecddebc459b55ca6c Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 28 Dec 2018 23:40:15 -0800 Subject: [PATCH] Add BB downloading for OpenBLAS (#30497) * Auto-detect binarybuilder triplet * Add OpenBLAS BinaryBuilder installation scaffolding Also make it easier to add more BB-cached versions of dependencies in the future * Enable `fixup-libgfortran.sh` to directly ask `$FC` for paths * Tell Appveyor and Travis to use BinaryBuilder OpenBLAS Also allow the build system to auto-guess the triplet --- .travis.yml | 4 +- Make.inc | 3 +- contrib/fixup-libgfortran.sh | 26 +++++-- contrib/normalize_triplet.py | 125 ++++++++++++++++++++++++++++++ contrib/windows/appveyor_build.sh | 3 +- deps/Makefile | 1 + deps/blas.mk | 20 +++++ deps/llvm.mk | 33 +------- deps/openblas.version | 2 + deps/tools/bb-install.mk | 33 ++++++++ 10 files changed, 210 insertions(+), 40 deletions(-) create mode 100755 contrib/normalize_triplet.py create mode 100644 deps/tools/bb-install.mk diff --git a/.travis.yml b/.travis.yml index 078a9249e2f7a..5d961cbfdbe1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,8 +76,8 @@ before_install: brew tap staticfloat/julia > /dev/null; brew rm --force $(brew deps --HEAD julia); brew install -v ccache gcc gmp mpfr pcre2 staticfloat/julia/openblas-julia staticfloat/julia/suite-sparse-julia staticfloat/juliadeps/libgfortran; - BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 BINARYBUILDER_TRIPLET=x86_64-apple-darwin14 BINARYBUILDER_LLVM_ASSERTS=1"; - BUILDOPTS="$BUILDOPTS USE_BINARYBUILDER_LLVM=1 LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size"; + BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 USE_BINARYBUILDER_LLVM=1 USE_BINARYBUILDER_OPENBLAS=1 BINARYBUILDER_LLVM_ASSERTS=1"; + BUILDOPTS="$BUILDOPTS LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size"; BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1"; BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas"; for lib in SUITESPARSE BLAS LAPACK GMP MPFR LIBUNWIND; do diff --git a/Make.inc b/Make.inc index 81ff923156ed8..e9cb2411981e5 100644 --- a/Make.inc +++ b/Make.inc @@ -232,11 +232,10 @@ INSTALL_F := $(JULIAHOME)/contrib/install.sh 644 INSTALL_M := $(JULIAHOME)/contrib/install.sh 755 # BinaryBuilder options -# TODO: Autodiscover triplet +USE_BINARYBUILDER_OPENBLAS := 0 USE_BINARYBUILDER_LLVM := 0 # Use the Assertions build BINARYBUILDER_LLVM_ASSERTS := 0 -BINARYBUILDER_TRIPLET := # LLVM Options LLVMROOT := $(build_prefix) diff --git a/contrib/fixup-libgfortran.sh b/contrib/fixup-libgfortran.sh index a994515d2356d..897c067de6a83 100755 --- a/contrib/fixup-libgfortran.sh +++ b/contrib/fixup-libgfortran.sh @@ -2,6 +2,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license # Run as: fixup-libgfortran.sh [--verbose] <$private_libdir> +FC=${FC:-gfortran} # If we're invoked with "--verbose", create a `debug` function that prints stuff out if [ "$1" = "--verbose" ] || [ "$1" = "-v" ]; then @@ -22,7 +23,7 @@ if [ "$UNAME" = "Linux" ]; then elif [ "$UNAME" = "Darwin" ]; then SHLIB_EXT="dylib" else - echo "WARNING: Could not autodetect platform type ('uname -s' == $UNAME); assuming Linux" >&2 + echo "WARNING: Could not autodetect platform type ('uname -s' = $UNAME); assuming Linux" >&2 UNAME="Linux" SHLIB_EXT="so" fi @@ -41,6 +42,20 @@ find_shlib() fi } +find_shlib_dir() +{ + # Usually, on platforms like OSX we get full paths when linking. However, + # if we are inspecting, say, BinaryBuilder-built OpenBLAS libraries, we will + # only get something like `@rpath/libgfortran.5.dylib` when inspecting the + # libraries. We can, as a last resort, ask `$FC` directly what the full + # filepath for this library is, but only if we don't have a direct path to it: + if [ $(dirname "$1") = "@rpath" ]; then + dirname "$($FC -print-file-name="$(basename "$1")" 2>/dev/null)" + else + dirname "$1" 2>/dev/null + fi +} + # First, discover all the places where libgfortran/libgcc is, as well as their true SONAMES for lib in lapack blas openblas; do for private_libname in ${private_libdir}/lib$lib*.$SHLIB_EXT*; do @@ -51,10 +66,11 @@ for lib in lapack blas openblas; do LIBQUADMATH_PATH=$(find_shlib "$private_libname" libquadmath) # Take the directories, add them onto LIBGFORTRAN_DIRS, which we use to - # search for these libraries in the future. - LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBGFORTRAN_PATH 2>/dev/null)" - LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBGCC_PATH 2>/dev/null)" - LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBQUADMATH_PATH 2>/dev/null)" + # search for these libraries in the future. If there is no directory, try + # asking `$FC` where such a file could be found. + LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBGFORTRAN_PATH)" + LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBGCC_PATH)" + LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBQUADMATH_PATH)" # Save the SONAMES LIBGFORTRAN_SONAMES="$LIBGFORTRAN_SONAMES $(basename "$LIBGFORTRAN_PATH")" diff --git a/contrib/normalize_triplet.py b/contrib/normalize_triplet.py new file mode 100755 index 0000000000000..42e9a9f5d5c15 --- /dev/null +++ b/contrib/normalize_triplet.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python + +import re, sys + +# This script designed to mimick `src/PlatformNames.jl` in `BinaryProvider.jl`, which has +# a method `platform_key_abi()` to parse uname-like output into something standarized. + +if len(sys.argv) < 2: + print("Usage: %s []") + sys.exit(1) + +arch_mapping = { + 'x86_64': '(x86_|amd)64', + 'i686': "i\\d86", + 'aarch64': "aarch64", + 'arm': "arm(v7l)?", + 'powerpc64le': "p(ower)?pc64le", +} +platform_mapping = { + 'darwin': "-apple-darwin[\\d\\.]*", + 'freebsd': "-(.*-)?freebsd[\\d\\.]*", + 'windows': "-w64-mingw32", + 'linux': "-(.*-)?linux", +} +libc_mapping = { + 'blank_libc': "", + 'gnu': "-gnu", + 'musl': "-musl", +} +call_abi_mapping = { + 'blank_call_abi': "", + 'eabihf': "eabihf", +} +gcc_version_mapping = { + 'blank_gcc': "", + 'gcc4': "-gcc4", + 'gcc7': "-gcc7", + 'gcc8': "-gcc8", +} +cxx_abi_mapping = { + 'blank_cxx_abi': "", + 'cxx03': "-cxx03", + 'cxx11': "-cxx11", +} + +# Helper function to collapse dictionary of mappings down into a regex of +# named capture groups joined by "|" operators +c = lambda mapping: "("+"|".join(["(?P<%s>%s)"%(k,v) for (k, v) in mapping.items()]) + ")" +mondo_regex = re.compile( + "^"+ + c(arch_mapping)+ + c(platform_mapping)+ + c(libc_mapping)+ + c(call_abi_mapping)+ + c(gcc_version_mapping)+ + c(cxx_abi_mapping)+ + "$" +) + +# Apply our mondo regex to our input: +m = mondo_regex.match(sys.argv[1]) +if m is None: + print("ERROR: Unmatchable platform string '%s'!"%(sys.argv[1])) + sys.exit(1) + +# Helper function to find the single named field within the giant regex +# that is not `nothing` for each mapping we give it. +def get_field(m, mapping): + g = m.groupdict() + for k in mapping: + if g[k] is not None: + return k + +arch = get_field(m, arch_mapping) +platform = get_field(m, platform_mapping) +libc = get_field(m, libc_mapping) +call_abi = get_field(m, call_abi_mapping) +gcc_version = get_field(m, gcc_version_mapping) +cxx_abi = get_field(m, cxx_abi_mapping) + +def r(x): + x = x.replace("blank_call_abi", "") + x = x.replace("blank_gcc", "") + x = x.replace("blank_cxx_abi", "") + x = x.replace("blank_libc", "") + return x + +def p(x): + # These contain characters that can't be easily represented as + # capture group names, unfortunately: + os_remapping = { + 'darwin': 'apple-darwin14', + 'windows': 'w64-mingw32', + 'freebsd': 'unknown-freebsd11.1', + } + x = r(x) + if x: + for k in os_remapping: + x = x.replace(k, os_remapping[k]) + return '-' + x + return x + +# If the user passes in a GCC version (like 8.2.0) use that to force a +# "-gcc8" tag at the end of the triplet, but only if it has otherwise +# not been specified +if gcc_version == "blank_gcc": + if len(sys.argv) == 3: + gcc_version = { + "4": "gcc4", + "5": "gcc4", + "6": "gcc4", + "7": "gcc7", + "8": "gcc8", + }[sys.argv[2][0]] + + +print(arch+p(platform)+p(libc)+r(call_abi)+p(gcc_version)+p(cxx_abi)) + +# Testing suite: +# triplets="i686-w64-mingw32 x86_64-pc-linux-musl arm-linux-musleabihf x86_64-linux-gnu arm-linux-gnueabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1" +# for t in $triplets; do +# if [[ $(./normalize_triplet.py "$t") != "$t" ]]; then +# echo "ERROR: Failed test on $t" +# fi +# done diff --git a/contrib/windows/appveyor_build.sh b/contrib/windows/appveyor_build.sh index fa29774a4e63b..b18780e927842 100755 --- a/contrib/windows/appveyor_build.sh +++ b/contrib/windows/appveyor_build.sh @@ -45,7 +45,6 @@ if [ "$ARCH" = x86_64 ]; then echo 'USE_BLAS64 = 1' >> Make.user echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas64_' >> Make.user echo 'LIBBLASNAME = libopenblas64_' >> Make.user - echo 'BINARYBUILDER_TRIPLET = x86_64-w64-mingw32' >> Make.user else bits=32 archsuffix=86 @@ -53,7 +52,6 @@ else echo "override MARCH = pentium4" >> Make.user echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas' >> Make.user echo 'LIBBLASNAME = libopenblas' >> Make.user - echo 'BINARYBUILDER_TRIPLET = i686-w64-mingw32' >> Make.user fi echo "override JULIA_CPU_TARGET=generic;native" >> Make.user @@ -198,6 +196,7 @@ if [ -n "$USEMSVC" ]; then else # Use BinaryBuilder echo 'USE_BINARYBUILDER_LLVM = 1' >> Make.user + echo 'USE_BINARYBUILDER_OPENBLAS = 1' >> Make.user echo 'BINARYBUILDER_LLVM_ASSERTS = 1' >> Make.user echo 'override DEP_LIBS += llvm openlibm' >> Make.user export CCACHE_DIR=/cygdrive/c/ccache diff --git a/deps/Makefile b/deps/Makefile index a759218f6ca03..ba0bb685181db 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -12,6 +12,7 @@ include $(SRCDIR)/Versions.make include $(JULIAHOME)/Make.inc include $(SRCDIR)/tools/common.mk include $(SRCDIR)/tools/git-external.mk +include $(SRCDIR)/tools/bb-install.mk # Special comments: # diff --git a/deps/blas.mk b/deps/blas.mk index c025d4ac26e39..e88d676fd22c2 100644 --- a/deps/blas.mk +++ b/deps/blas.mk @@ -92,6 +92,8 @@ endif # Do not overwrite the "-j" flag OPENBLAS_BUILD_OPTS += MAKE_NB_JOBS=0 +ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1) + $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-configured: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/source-extracted perl -i -ple 's/^\s*(EXTRALIB\s*\+=\s*-lSystemStubs)\s*$$/# $$1/g' $(dir $<)/Makefile.system echo 1 > $@ @@ -194,3 +196,21 @@ configure-lapack: extract-lapack compile-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled fastcheck-lapack: check-lapack check-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/build-checked + +else # USE_BINARYBUILDER_OPENBLAS + + +OPENBLAS_BB_URL_BASE := https://github.com/JuliaPackaging/Yggdrasil/releases/download/OpenBLAS-v$(OPENBLAS_VER)-$(OPENBLAS_BB_REL) +OPENBLAS_BB_NAME := OpenBLAS.v$(OPENBLAS_VER)-$(OPENBLAS_BB_REL) + +$(eval $(call bb-install,openblas,OPENBLAS,true)) +get-lapack: get-openblas +extract-lapack: extract-openblas +configure-lapack: configure-openblas +compile-lapack: compile-openblas +fastcheck-lapack: fastcheck-openblas +check-lapack: check-openblas +clean-lapack: clean-openblas +distclean-lapack: distclean-openblas +install-lapack: install-openblas +endif diff --git a/deps/llvm.mk b/deps/llvm.mk index bfef4d51abdec..c516b577f6269 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -513,37 +513,12 @@ ifeq ($(USE_POLLY),1) endif endif else # USE_BINARYBUILDER_LLVM -LLVM_BB_URL_BASE := https://github.com/staticfloat/LLVMBuilder/releases/download +LLVM_BB_URL_BASE := https://github.com/staticfloat/LLVMBuilder/releases/download/v$(LLVM_VER)-$(LLVM_BB_REL) ifneq ($(BINARYBUILDER_LLVM_ASSERTS), 1) -LLVM_BB_NAME := LLVM +LLVM_BB_NAME := LLVM.v$(LLVM_VER) else -LLVM_BB_NAME := LLVM.asserts +LLVM_BB_NAME := LLVM.asserts.v$(LLVM_VER) endif -LLVM_BB_NAME := $(LLVM_BB_NAME).v$(LLVM_VER) -LLVM_BB_URL := $(LLVM_BB_URL_BASE)/v$(LLVM_VER)-$(LLVM_BB_REL)/$(LLVM_BB_NAME).$(BINARYBUILDER_TRIPLET).tar.gz - -$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL): - mkdir -p $@ - -$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz: | $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL) - $(JLDOWNLOAD) $@ $(LLVM_BB_URL) - -$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/build-compiled: | $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz - echo 1 > $@ - -$(eval $(call staged-install,llvm,llvm-$$(LLVM_VER)-$$(LLVM_BB_REL),,,,)) - -#Override provision of stage tarball -$(build_staging)/llvm-$(LLVM_VER)-$(LLVM_BB_REL).tgz: $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz | $(build_staging) - cp $< $@ - -clean-llvm: -distclean-llvm: -get-llvm: $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz -extract-llvm: -configure-llvm: -compile-llvm: -fastcheck-llvm: -check-llvm: +$(eval $(call bb-install,llvm,LLVM,false)) endif # USE_BINARYBUILDER_LLVM diff --git a/deps/openblas.version b/deps/openblas.version index fd259abbf7dc4..889bae7c4c860 100644 --- a/deps/openblas.version +++ b/deps/openblas.version @@ -1,2 +1,4 @@ OPENBLAS_BRANCH=v0.3.3 OPENBLAS_SHA1=fd8d1868a126bb9f12bbc43b36ee30d1ba943fbb +OPENBLAS_VER=0.3.3 +OPENBLAS_BB_REL=0 diff --git a/deps/tools/bb-install.mk b/deps/tools/bb-install.mk new file mode 100644 index 0000000000000..4737a9d7693af --- /dev/null +++ b/deps/tools/bb-install.mk @@ -0,0 +1,33 @@ +define bb-install +# If the user has signified that this is a GCC-multiversioned tarball, then generate the proper tarball +ifeq ($(3),true) +$(2)_BB_TRIPLET := $(shell python $(call cygpath_w,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE)) $(lastword $(shell $(FC) --version | head -1))) +else +$(2)_BB_TRIPLET := $(shell python $(call cygpath_w,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE))) +endif +$(2)_BB_URL := $$($(2)_BB_URL_BASE)/$$($(2)_BB_NAME).$$($(2)_BB_TRIPLET).tar.gz + +$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME): + mkdir -p $$@ + +$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz: | $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME) + $$(JLDOWNLOAD) $$@ $$($(2)_BB_URL) + +$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/build-compiled: | $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz + echo 1 > $$@ + +$$(eval $$(call staged-install,$(1),$(1)-$$$$($(2)_BB_NAME),,,,)) + +#Override provision of stage tarball +$$(build_staging)/$(1)-$$($(2)_BB_NAME).tgz: $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz | $$(build_staging) + cp $$< $$@ + +clean-$(1): +distclean-$(1): +get-$(1): $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz +extract-$(1): +configure-$(1): +compile-$(1): +fastcheck-$(1): +check-$(1): +endef