Use Project Flash to build cuml python package for CUDA 11.0 job

ci/cpu/build.sh

@@ -60,6 +60,13 @@ conda list --show-channel-urls
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
+# FIXME: for now, force the building of all packages so they are built on a
+# machine with a single CUDA version, then have the gpu/build.sh script simply
+# install. This should eliminate a mismatch between different CUDA versions on
+# cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat.
+BUILD_LIBCUML=1
+BUILD_CUML=1
+
 ################################################################################
 # BUILD - Conda package builds (conda deps: libcuml <- cuml)
 ################################################################################
@@ -75,6 +82,7 @@ else
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml --dirty --no-remove-work-dir
     mkdir -p ${CONDA_BLD_DIR}/libcuml/work
     cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcuml/work
+    rm -rf ${CONDA_BLD_DIR}/work
   fi
 fi
 
@@ -84,7 +92,10 @@ if [ "$BUILD_CUML" == '1' ]; then
     gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cuml --python=${PYTHON}
   else
     gpuci_logger "PROJECT FLASH: Build conda pkg for cuml"
-    gpuci_conda_retry build --croot ${CONDA_BLD_DIR} -c ci/artifacts/cuml/cpu/.conda-bld/ --dirty --no-remove-work-dir conda/recipes/cuml --python=${PYTHON}
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c $CONDA_BLD_DIR --dirty --no-remove-work-dir --python=${PYTHON}
+    mkdir -p ${CONDA_BLD_DIR}/cuml/work
+    cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/cuml/work
+    rm -rf ${CONDA_BLD_DIR}/work
   fi
 fi
 

ci/gpu/build.sh

@@ -181,7 +181,6 @@ else
     chrpath -d libcuml++.so
     patchelf --replace-needed `patchelf --print-needed libcuml++.so | grep faiss` libfaiss.so libcuml++.so
 
-    gpuci_logger "GoogleTest for libcuml"
     cd $LIBCUML_BUILD_DIR
     chrpath -d ./test/ml
     patchelf --replace-needed `patchelf --print-needed ./test/ml | grep faiss` libfaiss.so ./test/ml
@@ -192,18 +191,36 @@ else
     CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
     gpuci_logger "Installing $CONDA_FILE"
     gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"
-
+
+    # FIXME: Project FLASH only builds for python version 3.7 which is the one used in 
+    # the CUDA 11.0 job, need to change all versions to project flash 
+    if [ "$py_ver" == "3.7" ];then
+        gpuci_logger "Using Project FLASH to install cuml python"
+        CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "cuml*.tar.bz2"`
+        CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
+        CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
+        echo "Installing $CONDA_FILE"
+        gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"
+
+    else
+        gpuci_logger "Building cuml python in gpu job"
+        "$WORKSPACE/build.sh" -v cuml --codecov   
+    fi
+
     gpuci_logger "Install the main version of dask and distributed"
     set -x
     pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
     pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
     set +x
-
-    gpuci_logger "Building cuml"
-    "$WORKSPACE/build.sh" -v cuml --codecov
-
+
     gpuci_logger "Python pytest for cuml"
     cd $WORKSPACE/python
+
+    # When installing cuml with project flash, we need to delete all folders except
+    # cuml/test since we are not building cython extensions in place 
+    if [ "$py_ver" == "3.7" ];then
+        find ./cuml -mindepth 1 ! -regex '^./cuml/test\(/.*\)?' -delete
+    fi
 
     pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=cuml/test/dask --ignore=cuml/raft --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term