diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index c2f6411496..b923795dc0 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -60,6 +60,13 @@ conda list --show-channel-urls # FIX Added to deal with Anancoda SSL verification issues during conda builds conda config --set ssl_verify False +# FIXME: for now, force the building of all packages so they are built on a +# machine with a single CUDA version, then have the gpu/build.sh script simply +# install. This should eliminate a mismatch between different CUDA versions on +# cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat. +BUILD_LIBCUML=1 +BUILD_CUML=1 + ################################################################################ # BUILD - Conda package builds (conda deps: libcuml <- cuml) ################################################################################ @@ -75,6 +82,7 @@ else gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml --dirty --no-remove-work-dir mkdir -p ${CONDA_BLD_DIR}/libcuml/work cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcuml/work + rm -rf ${CONDA_BLD_DIR}/work fi fi @@ -84,7 +92,10 @@ if [ "$BUILD_CUML" == '1' ]; then gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cuml --python=${PYTHON} else gpuci_logger "PROJECT FLASH: Build conda pkg for cuml" - gpuci_conda_retry build --croot ${CONDA_BLD_DIR} -c ci/artifacts/cuml/cpu/.conda-bld/ --dirty --no-remove-work-dir conda/recipes/cuml --python=${PYTHON} + gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c $CONDA_BLD_DIR --dirty --no-remove-work-dir --python=${PYTHON} + mkdir -p ${CONDA_BLD_DIR}/cuml/work + cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/cuml/work + rm -rf ${CONDA_BLD_DIR}/work fi fi diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index c9dc8d4d09..efcff5b9c2 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -181,7 +181,6 @@ else chrpath -d libcuml++.so patchelf --replace-needed `patchelf --print-needed libcuml++.so | grep faiss` libfaiss.so libcuml++.so - gpuci_logger "GoogleTest for libcuml" cd $LIBCUML_BUILD_DIR chrpath -d ./test/ml patchelf --replace-needed `patchelf --print-needed ./test/ml | grep faiss` libfaiss.so ./test/ml @@ -192,18 +191,36 @@ else CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install gpuci_logger "Installing $CONDA_FILE" gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE" - + + # FIXME: Project FLASH only builds for python version 3.7 which is the one used in + # the CUDA 11.0 job, need to change all versions to project flash + if [ "$py_ver" == "3.7" ];then + gpuci_logger "Using Project FLASH to install cuml python" + CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "cuml*.tar.bz2"` + CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension + CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install + echo "Installing $CONDA_FILE" + gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE" + + else + gpuci_logger "Building cuml python in gpu job" + "$WORKSPACE/build.sh" -v cuml --codecov + fi + gpuci_logger "Install the main version of dask and distributed" set -x pip install "git+https://github.com/dask/distributed.git@2021.11.2" --upgrade --no-deps pip install "git+https://github.com/dask/dask.git@2021.11.2" --upgrade --no-deps set +x - - gpuci_logger "Building cuml" - "$WORKSPACE/build.sh" -v cuml --codecov - + gpuci_logger "Python pytest for cuml" cd $WORKSPACE/python + + # When installing cuml with project flash, we need to delete all folders except + # cuml/test since we are not building cython extensions in place + if [ "$py_ver" == "3.7" ];then + find ./cuml -mindepth 1 ! -regex '^./cuml/test\(/.*\)?' -delete + fi pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=cuml/test/dask --ignore=cuml/raft --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term