Distribute libcuspatial wheels

.github/workflows/build.yaml

@@ -66,7 +66,30 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build-libcuspatial:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_libcuspatial.sh
+  wheel-publish-libcuspatial:
+    needs: wheel-build-libcuspatial
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libcuspatial
+      package-type: cpp
   wheel-build-cuspatial:
+    needs: wheel-publish-libcuspatial
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:

.github/workflows/pr.yaml

@@ -21,6 +21,7 @@ jobs:
       - conda-notebook-tests
       - docs-build
       - wheel-build-cuspatial
+      - wheel-build-libcuspatial
       - wheel-tests-cuspatial
       - wheel-build-cuproj
       - wheel-tests-cuproj
@@ -83,10 +84,19 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/build_docs.sh"
-  wheel-build-cuspatial:
+  wheel-build-libcuspatial:
     needs: checks
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+      build_type: pull-request
+      script: ci/build_wheel_libcuspatial.sh
+  wheel-build-cuspatial:
+    needs: [checks, wheel-build-libcuspatial]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_cuspatial.sh

build.sh

@@ -124,11 +124,6 @@ if hasArg benchmarks; then
     BUILD_BENCHMARKS=ON
 fi
 
-# Append `-DFIND_CUSPATIAL_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
-if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUSPATIAL_CPP"* ]]; then
-    EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUSPATIAL_CPP=ON"
-fi
-
 # Append `-DFIND_CUPROJ_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
 if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUPROJ_CPP"* ]]; then
     EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUPROJ_CPP=ON"

ci/build_wheel.sh

@@ -7,6 +7,25 @@ package_name=$1
 package_dir=$2
 package_type=$3
 
+# The 'libcuspatial' wheel should package 'libcuspatial.so', and all others
+# should exclude it (they dynamically load it if they need it).
+#
+# Capturing that here in argument-parsing to allow this build_wheel.sh
+# script to be re-used by all wheel builds in the project.
+case "${package_dir}" in
+  python/libcuspatial)
+    EXCLUDE_ARGS=(
+      --exclude "libcudf.so"
+    )
+  ;;
+  *)
+    EXCLUDE_ARGS=(
+      --exclude "libcudf.so"
+      --exclude "libcuspatial.so"
+    )
+  ;;
+esac
+
 source rapids-configure-sccache
 source rapids-date-string
 
@@ -20,7 +39,7 @@ python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
 
 mkdir -p final_dist
 python -m auditwheel repair \
-    --exclude libcudf.so \
+    "${EXCLUDE_ARGS[@]}" \
     -w final_dist \
     dist/*
 

ci/build_wheel_cuspatial.sh

@@ -3,4 +3,15 @@
 
 set -euo pipefail
 
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# Downloads libcuspatial wheel from this current build,
+# then ensures 'cuspatial' wheel builds always use the 'libcuspatial' just built in the same CI run.
+#
+# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints
+# are used when creating the isolated build environment.
+RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcuspatial_dist
+echo "libcuspatial-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcuspatial_dist/libcuspatial_*.whl)" > /tmp/constraints.txt
+export PIP_CONSTRAINT="/tmp/constraints.txt"
+
 ci/build_wheel.sh cuspatial python/cuspatial python

ci/build_wheel_libcuspatial.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+ci/build_wheel.sh libcuspatial python/libcuspatial cpp

ci/test_wheel_cuproj.sh

@@ -13,12 +13,14 @@ DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-de
 # Download the cuproj and cuspatial built in the previous step
 RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
 RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install \
   --no-binary 'fiona' \
   "$(echo ./dist/cuspatial*.whl)" \
   "$(echo ./dist/cuproj*.whl)[test]" \
+  "$(echo ./dist/libcuspatial*.whl)" \
   'fiona>=1.8.19,<1.9'
 
 rapids-logger "pytest cuproj"

ci/test_wheel_cuspatial.sh

@@ -10,13 +10,15 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 apt update
 DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev
 
-# Download the cuspatial built in the previous step
+# Download the cuspatial and libcuspatial built in the previous step
 RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install \
   --no-binary 'fiona' \
   "$(echo ./dist/cuspatial*.whl)[test]" \
+  "$(echo ./dist/libcuspatial*.whl)" \
   'fiona>=1.8.19,<1.9'
 
 rapids-logger "pytest cuspatial"

cpp/cmake/thirdparty/get_ranger.cmake

@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,8 +26,6 @@ function(find_and_configure_ranger)
     rapids_cpm_find(
       ranger 00.01.00
       GLOBAL_TARGETS "${global_targets}"
-      BUILD_EXPORT_SET cuspatial-exports
-      INSTALL_EXPORT_SET cuspatial-exports
       CPM_ARGS
       GIT_REPOSITORY https://github.com/harrism/ranger.git
       GIT_TAG main

dependencies.yaml

@@ -18,6 +18,7 @@ files:
       - depends_on_cuml
       - depends_on_cupy
       - depends_on_libcudf
+      - depends_on_libcuspatial
       - depends_on_librmm
       - rapids_build_skbuild
       - run_python_cuspatial
@@ -30,11 +31,13 @@ files:
     output: none
     includes:
       - cuda_version
+      - depends_on_libcuspatial
       - test_libcuspatial
   test_python:
     output: none
     includes:
       - cuda_version
+      - depends_on_libcuspatial
       - py_version
       - test_python_cuspatial
       - test_python_cuproj
@@ -44,6 +47,7 @@ files:
     includes:
       - cuda_version
       - depends_on_cuml
+      - depends_on_libcuspatial
       - test_notebooks
       - notebooks
       - py_version
@@ -57,9 +61,35 @@ files:
     output: none
     includes:
       - cuda_version
+      - depends_on_libcuspatial
       - docs
       - py_version
       - test_cuspatial
+  py_build_libcuspatial:
+    output: [pyproject]
+    pyproject_dir: python/libcuspatial
+    extras:
+      table: build-system
+    includes:
+      - rapids_build_skbuild
+  py_rapids_build_libcuspatial:
+    output: [pyproject]
+    pyproject_dir: python/libcuspatial
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes:
+      - build_cpp
+      - build_wheels
+      - depends_on_libcudf
+      - depends_on_librmm
+  py_run_libcuspatial:
+    output: [pyproject]
+    pyproject_dir: python/libcuspatial
+    extras:
+      table: project
+    includes:
+      - depends_on_libcudf
   py_build_cuspatial:
     output: [pyproject]
     pyproject_dir: python/cuspatial
@@ -80,6 +110,7 @@ files:
       - build_python
       - build_wheels
       - depends_on_libcudf
+      - depends_on_libcuspatial
       - depends_on_librmm
   py_run_cuspatial:
     output: [pyproject]
@@ -89,6 +120,7 @@ files:
     includes:
       - depends_on_cudf
       - depends_on_libcudf
+      - depends_on_libcuspatial
       - depends_on_rmm
       - run_python_cuspatial
   py_test_cuspatial:
@@ -540,16 +572,39 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - libcuspatial==24.10.*,>=0.0.0a0
           - libcuspatial-tests==24.10.*,>=0.0.0a0
   test_cuspatial:
     common:
       - output_types: conda
         packages:
-          - libcuspatial==24.10.*,>=0.0.0a0
           - cuspatial==24.10.*,>=0.0.0a0
           - cuproj==24.10.*,>=0.0.0a0
 
+  depends_on_libcuspatial:
+    common:
+      - output_types: conda
+        packages:
+          - &libcuspatial_unsuffixed libcuspatial==24.10.*,>=0.0.0a0
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+              cuda_suffixed: "true"
+            packages:
+              - libcuspatial-cu12==24.10.*,>=0.0.0a0
+          - matrix:
+              cuda: "11.*"
+              cuda_suffixed: "true"
+            packages:
+              - libcuspatial-cu11==24.10.*,>=0.0.0a0
+          - {matrix: null, packages: [*libcuspatial_unsuffixed]}
+
   depends_on_librmm:
     common:
       - output_types: conda

python/cuspatial/CMakeLists.txt

@@ -23,34 +23,9 @@ project(
   VERSION "${RAPIDS_VERSION}"
   LANGUAGES CXX CUDA)
 
-option(FIND_CUSPATIAL_CPP "Search for existing cuspatial C++ installations before defaulting to local files"
-       OFF)
-
-# If the user requested it we attempt to find cuspatial.
-if(FIND_CUSPATIAL_CPP)
-  find_package(cuspatial "${RAPIDS_VERSION}")
-else()
-  set(cuspatial_FOUND OFF)
-endif()
-
-if(NOT cuspatial_FOUND)
-  set(BUILD_TESTS OFF)
-  set(BUILD_BENCHMARKS OFF)
-  set(CUDA_STATIC_RUNTIME ON)
-  set(CUSPATIAL_USE_CUDF_STATIC ON)
-  set(CUSPATIAL_EXCLUDE_CUDF_FROM_ALL ON)
-
-  add_subdirectory(../../cpp cuspatial-cpp EXCLUDE_FROM_ALL)
-
-  set(cython_lib_dir cuspatial)
-  install(TARGETS cuspatial DESTINATION ${cython_lib_dir})
-endif()
+find_package(cuspatial "${RAPIDS_VERSION}" REQUIRED)
 
 include(rapids-cython-core)
 rapids_cython_init()
 
 add_subdirectory(cuspatial/_lib)
-
-if(DEFINED cython_lib_dir)
-  rapids_cython_add_rpath_entries(TARGET cuspatial PATHS "${cython_lib_dir}")
-endif()

python/cuspatial/cuspatial/__init__.py

@@ -1,8 +1,16 @@
 # Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
-# load cudf before any other imports, to be sure libcudf's symbols are found
-# in the libcudf.so from the libcudf wheel (if one is installed)
-import cudf
+# If libcuspatial was installed as a wheel, we must request it to load the library
+# symbols. Otherwise, we assume that the library was installed in a system path that ld
+# can find.
+try:
+    import libcuspatial
+except ModuleNotFoundError:
+    pass
+else:
+    libcuspatial.load_library()
+    del libcuspatial
+
 
 from ._version import __git_commit__, __version__
 from .core.geodataframe import GeoDataFrame

python/cuspatial/pyproject.toml

@@ -33,6 +33,7 @@ dependencies = [
     "cudf==24.10.*,>=0.0.0a0",
     "geopandas>=0.11.0",
     "libcudf==24.10.*,>=0.0.0a0",
+    "libcuspatial==24.10.*,>=0.0.0a0",
     "numpy>=1.23,<2.0a0",
     "rmm==24.10.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -137,6 +138,7 @@ requires = [
     "cudf==24.10.*,>=0.0.0a0",
     "cython>=3.0.0",
     "libcudf==24.10.*,>=0.0.0a0",
+    "libcuspatial==24.10.*,>=0.0.0a0",
     "librmm==24.10.*,>=0.0.0a0",
     "ninja",
     "rmm==24.10.*,>=0.0.0a0",