Merge branch 'branch-23.08' into cugraph-nx

rapidsai · Jul 27, 2023 · a237ea0 · a237ea0
2 parents 2b52f14 + e4eb22b
commit a237ea0
Show file tree

Hide file tree

Showing 198 changed files with 11,299 additions and 3,383 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -68,27 +68,20 @@ jobs:
       sha: ${{ inputs.sha }}
   wheel-build-pylibcugraph:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      package-name: pylibcugraph
-      package-dir: python/pylibcugraph
-
-      # Note that this approach to cloning repos obviates any modification to
-      # the CMake variables in get_cumlprims_mg.cmake since CMake will just use
-      # the clone as is.
+      script: ci/build_wheel_pylibcugraph.sh
       extra-repo: rapidsai/cugraph-ops
       extra-repo-sha: branch-23.08
       extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
-
-      skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/"
   wheel-publish-pylibcugraph:
     needs: wheel-build-pylibcugraph
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -98,28 +91,20 @@ jobs:
   wheel-build-cugraph:
     needs: wheel-publish-pylibcugraph
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      package-name: cugraph
-      package-dir: python/cugraph
-
-      # Note that this approach to cloning repos obviates any modification to
-      # the CMake variables in get_cumlprims_mg.cmake since CMake will just use
-      # the clone as is.
+      script: ci/build_wheel_cugraph.sh
       extra-repo: rapidsai/cugraph-ops
       extra-repo-sha: branch-23.08
       extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
-
-      before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse"
-      skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/"
   wheel-publish-cugraph:
     needs: wheel-build-cugraph
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -78,47 +78,34 @@ jobs:
   wheel-build-pylibcugraph:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
     with:
       build_type: pull-request
-      package-name: pylibcugraph
-      package-dir: python/pylibcugraph
+      script: ci/build_wheel_pylibcugraph.sh
       extra-repo: rapidsai/cugraph-ops
       extra-repo-sha: branch-23.08
       extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
-      skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/"
   wheel-tests-pylibcugraph:
     needs: wheel-build-pylibcugraph
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
     with:
       build_type: pull-request
-      package-name: pylibcugraph
-      test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests"
-      test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py"
+      script: ci/test_wheel_pylibcugraph.sh
   wheel-build-cugraph:
     needs: wheel-tests-pylibcugraph
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
     with:
       build_type: pull-request
-      package-name: cugraph
-      package-dir: python/cugraph
+      script: ci/build_wheel_cugraph.sh
       extra-repo: rapidsai/cugraph-ops
       extra-repo-sha: branch-23.08
       extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
-      before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse"
-      skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/"
   wheel-tests-cugraph:
     needs: wheel-build-cugraph
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
     with:
       build_type: pull-request
-      package-name: cugraph
-      # Always want to test against latest dask/distributed.
-      test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
-      # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests.
-      test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
-      test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests"
-      test-smoketest: "python ci/wheel_smoke_test_cugraph.py"
+      script: ci/test_wheel_cugraph.sh
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -32,24 +32,19 @@ jobs:
       sha: ${{ inputs.sha }}
   wheel-tests-pylibcugraph:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      package-name: pylibcugraph
-      test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests"
+      script: ci/test_wheel_pylibcugraph.sh
   wheel-tests-cugraph:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      package-name: cugraph
-      # Always want to test against latest dask/distributed.
-      test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
-      test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
-      test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests"
+      script: ci/test_wheel_cugraph.sh
diff --git a/build.sh b/build.sh
@@ -62,7 +62,7 @@ HELP="$0 [<target> ...] [<flag> ...]
  and <flag> is:
    -v                         - verbose build mode
    -g                         - build for debug
-   -n                         - do not install after a successful build
+   -n                         - do not install after a successful build (does not affect Python packages)
    --pydevelop                - use setup.py develop instead of install
    --allgpuarch               - build for all supported GPU architectures
    --skip_cpp_tests           - do not build the SG test binaries as part of the libcugraph and libcugraph_etl targets
@@ -106,7 +106,7 @@ BUILD_CPP_MG_TESTS=OFF
 BUILD_ALL_GPU_ARCH=0
 BUILD_WITH_CUGRAPHOPS=ON
 CMAKE_GENERATOR_OPTION="-G Ninja"
-PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps ."
+PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps"
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if PREFIX is not set, check CONDA_PREFIX, but there is no fallback
@@ -180,6 +180,12 @@ if hasArg --pydevelop; then
     PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps -e ."
 fi
 
+# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
+SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}"
+  if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUGRAPH_CPP"* ]]; then
+      SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_CUGRAPH_CPP=ON"
+  fi
+
 # If clean or uninstall targets given, run them prior to any other steps
 if hasArg uninstall; then
     if [[ "$INSTALL_PREFIX" != "" ]]; then
@@ -299,21 +305,9 @@ if buildAll || hasArg pylibcugraph; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/pylibcugraph
     else
-        cd ${REPODIR}/python/pylibcugraph
-        # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
-        # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
-        CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}
-        python setup.py build_ext \
-	       --inplace \
-	       -- \
-	       -DFIND_CUGRAPH_CPP=ON \
-	       -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \
-               -Dcugraph_ROOT=${LIBCUGRAPH_BUILD_DIR} \
-	       -- \
-	       -j${PARALLEL_LEVEL:-1}
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        SKBUILD_CONFIGURE_OPTIONS="${SKBUILD_EXTRA_CMAKE_ARGS} -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS}" \
+            SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL}" \
+            python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/pylibcugraph
     fi
 fi
 
@@ -322,22 +316,9 @@ if buildAll || hasArg cugraph; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph
     else
-        cd ${REPODIR}/python/cugraph
-        # FIXME: this needs to eventually reference the pylibcugraph build
-        # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
-        # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
-        CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}
-        python setup.py build_ext \
-	       --inplace \
-	       -- \
-	       -DFIND_CUGRAPH_CPP=ON \
-	       -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \
-               -Dcugraph_ROOT=${LIBCUGRAPH_BUILD_DIR} \
-	       -- \
-	       -j${PARALLEL_LEVEL:-1}
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        SKBUILD_CONFIGURE_OPTIONS="${SKBUILD_EXTRA_CMAKE_ARGS} -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS}" \
+            SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL}" \
+            python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph
     fi
 fi
 
@@ -346,12 +327,8 @@ if hasArg cugraph-service; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph-service
     else
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            cd ${REPODIR}/python/cugraph-service/client
-            python ${PYTHON_ARGS_FOR_INSTALL}
-            cd ${REPODIR}/python/cugraph-service/server
-            python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-service/client
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-service/server
     fi
 fi
 
@@ -360,10 +337,7 @@ if hasArg cugraph-pyg; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph-pyg
     else
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            cd ${REPODIR}/python/cugraph-pyg
-            python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-pyg
     fi
 fi
 
@@ -372,10 +346,7 @@ if hasArg cugraph-dgl; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph-dgl
     else
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            cd ${REPODIR}/python/cugraph-dgl
-            python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-dgl
     fi
 fi
 

diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -26,28 +26,36 @@ rapids-mamba-retry mambabuild \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/cugraph
 
-rapids-mamba-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  conda/recipes/cugraph-service
-
-rapids-mamba-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  --channel pyg \
-  --channel pytorch \
-  --channel pytorch-nightly \
-  conda/recipes/cugraph-pyg
-
-rapids-mamba-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  --channel dglteam \
-  --channel pytorch \
-  --channel pytorch-nightly \
-  conda/recipes/cugraph-dgl
+RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
+
+if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
+  # Only one CUDA configuration is needed, so we choose CUDA 11 arbitrarily.
+  # Nothing in the cugraph-service packages is CUDA-specific.
+  rapids-mamba-retry mambabuild \
+    --no-test \
+    --channel "${CPP_CHANNEL}" \
+    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+    conda/recipes/cugraph-service
+
+  # Only CUDA 11 is supported right now due to PyTorch requirement.
+  rapids-mamba-retry mambabuild \
+    --no-test \
+    --channel "${CPP_CHANNEL}" \
+    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+    --channel pyg \
+    --channel pytorch \
+    --channel pytorch-nightly \
+    conda/recipes/cugraph-pyg
+
+  # Only CUDA 11 is supported right now due to PyTorch requirement.
+  rapids-mamba-retry mambabuild \
+    --no-test \
+    --channel "${CPP_CHANNEL}" \
+    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+    --channel dglteam \
+    --channel pytorch \
+    --channel pytorch-nightly \
+    conda/recipes/cugraph-dgl
+fi
 
 rapids-upload-conda-to-s3 python
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name=$1
+package_dir=$2
+
+source rapids-configure-sccache
+source rapids-date-string
+
+# Use gha-tools rapids-pip-wheel-version to generate wheel version then
+# update the necessary files
+version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})"
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+bash ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}"
+echo "The package name and/or version was modified in the package source. The git diff is:"
+git diff
+
+cd "${package_dir}"
+
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph
+python -m pip install --no-deps ./local-pylibcugraph/pylibcugraph*.whl
+
+export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
+
+./ci/build_wheel.sh cugraph python/cugraph
diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
+
+./ci/build_wheel.sh pylibcugraph python/pylibcugraph