From 3727dcb6a28218349bb97e1aae96cdb29e67dbd3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 29 Aug 2024 10:42:50 -0700 Subject: [PATCH 01/15] initial library setup --- ci/build_wheel_libcuspatial.sh | 6 ++ python/libcuspatial/CMakeLists.txt | 49 ++++++++++++++ python/libcuspatial/LICENSE | 1 + python/libcuspatial/README.md | 1 + python/libcuspatial/libcuspatial/__init__.py | 16 +++++ python/libcuspatial/libcuspatial/_version.py | 32 ++++++++++ python/libcuspatial/libcuspatial/load.py | 51 +++++++++++++++ python/libcuspatial/pyproject.toml | 67 ++++++++++++++++++++ 8 files changed, 223 insertions(+) create mode 100755 ci/build_wheel_libcuspatial.sh create mode 100644 python/libcuspatial/CMakeLists.txt create mode 120000 python/libcuspatial/LICENSE create mode 120000 python/libcuspatial/README.md create mode 100644 python/libcuspatial/libcuspatial/__init__.py create mode 100644 python/libcuspatial/libcuspatial/_version.py create mode 100644 python/libcuspatial/libcuspatial/load.py create mode 100644 python/libcuspatial/pyproject.toml diff --git a/ci/build_wheel_libcuspatial.sh b/ci/build_wheel_libcuspatial.sh new file mode 100755 index 000000000..01e699a21 --- /dev/null +++ b/ci/build_wheel_libcuspatial.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +ci/build_wheel.sh cuspatial python/libcuspatial cpp diff --git a/python/libcuspatial/CMakeLists.txt b/python/libcuspatial/CMakeLists.txt new file mode 100644 index 000000000..ac1eb3387 --- /dev/null +++ b/python/libcuspatial/CMakeLists.txt @@ -0,0 +1,49 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libcuspatial-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +# Check if cuspatial is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python cuspatial package. +find_package(cuspatial "${RAPIDS_VERSION}") +if(cuspatial_FOUND) + return() +endif() + +unset(cuspatial_FOUND) + +# # Find Python early so that later commands can use it +# find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) + +# set options that only have 1 correct values in the context of libcuspatial wheel builds +set(BUILD_SHARED_LIBS ON) +set(BUILD_TESTS OFF) +set(BUILD_BENCHMARKS OFF) +set(CUDA_ENABLE_LINEINFO OFF) +set(CUDA_STATIC_RUNTIME OFF) +set(CUSPATIAL_USE_CUDF_STATIC OFF) +set(CUSPATIAL_EXCLUDE_CUDF_FROM_ALL ON) + +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + +add_subdirectory(../../cpp cuspatial-cpp) +install(TARGETS cuspatial DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) diff --git a/python/libcuspatial/LICENSE b/python/libcuspatial/LICENSE new file mode 120000 index 000000000..30cff7403 --- /dev/null +++ b/python/libcuspatial/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcuspatial/README.md b/python/libcuspatial/README.md new file mode 120000 index 000000000..fe8400541 --- /dev/null +++ b/python/libcuspatial/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libcuspatial/libcuspatial/__init__.py b/python/libcuspatial/libcuspatial/__init__.py new file mode 100644 index 000000000..49b02bb14 --- /dev/null +++ b/python/libcuspatial/libcuspatial/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcuspatial._version import __git_commit__, __version__ +from libcuspatial.load import load_library diff --git a/python/libcuspatial/libcuspatial/_version.py b/python/libcuspatial/libcuspatial/_version.py new file mode 100644 index 000000000..826e84a67 --- /dev/null +++ b/python/libcuspatial/libcuspatial/_version.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcuspatial/libcuspatial/load.py b/python/libcuspatial/libcuspatial/load.py new file mode 100644 index 000000000..f75eca26f --- /dev/null +++ b/python/libcuspatial/libcuspatial/load.py @@ -0,0 +1,51 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os +import libcudf + + +def load_library(): + # libcudf must be loaded before libcuspatial because libcuspatial references its symbols + libcudf.load_library() + + # Dynamically load libcudf.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + libcuspatial_lib = None + try: + libcuspatial_lib = ctypes.CDLL("libcuspatial.so", ctypes.RTLD_GLOBAL) + except OSError: + # If neither of these directories contain the library, we assume we are in an + # environment where the C++ library is already installed somewhere else and the + # CMake build of the libcudf Python package was a no-op. + # + # Note that this approach won't work for real editable installs of the libcudf package. + # scikit-build-core has limited support for importlib.resources so there isn't a clean + # way to support that case yet. + for lib_dir in ("lib", "lib64"): + if os.path.isfile( + lib := os.path.join( + os.path.dirname(__file__), lib_dir, "libcuspatial.so" + ) + ): + libcuspatial_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) + break + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcudf was loaded from. + return libcuspatial_lib diff --git a/python/libcuspatial/pyproject.toml b/python/libcuspatial/pyproject.toml new file mode 100644 index 000000000..3c4f306c2 --- /dev/null +++ b/python/libcuspatial/pyproject.toml @@ -0,0 +1,67 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcuspatial" +dynamic = ["version"] +description = "cuSpatial: GPU-Accelerated Spatial and Trajectory Data Management and Analytics Library (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.10" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] + +[project.urls] +Homepage = "https://github.com/rapidsai/cuspatial" + +[project.entry-points."cmake.prefix"] +libcuspatial = "libcuspatial" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcuspatial"] +wheel.install-dir = "libcuspatial" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcuspatial/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" From c37508f6d47d9e9ee8fca9c7d8a548a8816a3f1f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 29 Aug 2024 11:04:58 -0700 Subject: [PATCH 02/15] dependencies.yaml updates --- dependencies.yaml | 58 ++++++++++++++++++++++-- python/libcuspatial/libcuspatial/load.py | 1 + python/libcuspatial/pyproject.toml | 10 ++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 80822b43d..588d2910d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -18,6 +18,7 @@ files: - depends_on_cuml - depends_on_cupy - depends_on_libcudf + - depends_on_libcuspatial - depends_on_librmm - rapids_build_skbuild - run_python_cuspatial @@ -30,11 +31,13 @@ files: output: none includes: - cuda_version + - depends_on_libcuspatial - test_libcuspatial test_python: output: none includes: - cuda_version + - depends_on_libcuspatial - py_version - test_python_cuspatial - test_python_cuproj @@ -44,6 +47,7 @@ files: includes: - cuda_version - depends_on_cuml + - depends_on_libcuspatial - test_notebooks - notebooks - py_version @@ -57,9 +61,35 @@ files: output: none includes: - cuda_version + - depends_on_libcuspatial - docs - py_version - test_cuspatial + py_build_libcuspatial: + output: [pyproject] + pyproject_dir: python/libcuspatial + extras: + table: build-system + includes: + - rapids_build_skbuild + py_rapids_build_libcuspatial: + output: [pyproject] + pyproject_dir: python/libcuspatial + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_cpp + - build_wheels + - depends_on_libcudf + - depends_on_librmm + py_run_libcuspatial: + output: [pyproject] + pyproject_dir: python/libcuspatial + extras: + table: project + includes: + - depends_on_libcudf py_build_cuspatial: output: [pyproject] pyproject_dir: python/cuspatial @@ -134,7 +164,6 @@ files: includes: - test_python_cuproj - depends_on_cuspatial - - test_cuspatial channels: - rapidsai @@ -542,16 +571,39 @@ dependencies: common: - output_types: conda packages: - - libcuspatial==24.10.*,>=0.0.0a0 - libcuspatial-tests==24.10.*,>=0.0.0a0 test_cuspatial: common: - output_types: conda packages: - - libcuspatial==24.10.*,>=0.0.0a0 - cuspatial==24.10.*,>=0.0.0a0 - cuproj==24.10.*,>=0.0.0a0 + depends_on_libcuspatial: + common: + - output_types: conda + packages: + - &libcuspatial_unsuffixed libcuspatial==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcuspatial-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcuspatial-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*libcuspatial_unsuffixed]} + depends_on_librmm: common: - output_types: conda diff --git a/python/libcuspatial/libcuspatial/load.py b/python/libcuspatial/libcuspatial/load.py index f75eca26f..117be5cb1 100644 --- a/python/libcuspatial/libcuspatial/load.py +++ b/python/libcuspatial/libcuspatial/load.py @@ -15,6 +15,7 @@ import ctypes import os + import libcudf diff --git a/python/libcuspatial/pyproject.toml b/python/libcuspatial/pyproject.toml index 3c4f306c2..ab64d4cb6 100644 --- a/python/libcuspatial/pyproject.toml +++ b/python/libcuspatial/pyproject.toml @@ -38,6 +38,9 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] +dependencies = [ + "libcudf==24.10.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/cuspatial" @@ -65,3 +68,10 @@ regex = "(?P.*)" build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "libcudf==24.10.*,>=0.0.0a0", + "librmm==24.10.*,>=0.0.0a0", + "ninja", + "wheel", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 59f14f19b69f7b8b291839d482f7a67f5eeccccd Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 29 Aug 2024 11:47:14 -0700 Subject: [PATCH 03/15] update build scripts, cuspatial cmake --- .github/workflows/build.yaml | 21 ++++++++++++++++++ .github/workflows/pr.yaml | 9 +++++++- ci/build_wheel_cuspatial.sh | 11 ++++++++++ ci/test_wheel_cuproj.sh | 2 ++ ci/test_wheel_cuspatial.sh | 4 +++- dependencies.yaml | 2 ++ python/cuspatial/CMakeLists.txt | 27 +----------------------- python/cuspatial/pyproject.toml | 2 ++ python/libcuspatial/libcuspatial/load.py | 25 ++++++++++++---------- 9 files changed, 64 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1f3fd7922..dd2179635 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,7 +66,28 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-libcuspatial: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcuspatial.sh + wheel-publish-libcuspatial: + needs: wheel-build-libcuspatial + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcuspatial + package-type: cpp wheel-build-cuspatial: + needs: wheel-publish-libcuspatial secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ecf4206c7..7ecffc108 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -83,10 +83,17 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cuspatial: + wheel-build-libcuspatial: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: pull-request + script: ci/build_wheel_libcuspatial.sh + wheel-build-cuspatial: + needs: [checks, wheel-build-libcuspatial] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_cuspatial.sh diff --git a/ci/build_wheel_cuspatial.sh b/ci/build_wheel_cuspatial.sh index 171bfa6a4..e581a3e76 100755 --- a/ci/build_wheel_cuspatial.sh +++ b/ci/build_wheel_cuspatial.sh @@ -3,4 +3,15 @@ set -euo pipefail +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# Downloads libcuspatial wheel from this current build, +# then ensures 'cuspatial' wheel builds always use the 'libcuspatial' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcuspatial_dist +echo "libcuspatial-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcuspatial_dist/libcuspatial_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" + ci/build_wheel.sh cuspatial python/cuspatial python diff --git a/ci/test_wheel_cuproj.sh b/ci/test_wheel_cuproj.sh index 4144e8dab..4752fe417 100755 --- a/ci/test_wheel_cuproj.sh +++ b/ci/test_wheel_cuproj.sh @@ -13,12 +13,14 @@ DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-de # Download the cuproj and cuspatial built in the previous step RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ --no-binary 'fiona' \ "$(echo ./dist/cuspatial*.whl)" \ "$(echo ./dist/cuproj*.whl)[test]" \ + "$(echo ./dist/libcuspatial*.whl)" \ 'fiona>=1.8.19,<1.9' rapids-logger "pytest cuproj" diff --git a/ci/test_wheel_cuspatial.sh b/ci/test_wheel_cuspatial.sh index 8faa4e8d3..7e16b9626 100755 --- a/ci/test_wheel_cuspatial.sh +++ b/ci/test_wheel_cuspatial.sh @@ -10,13 +10,15 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev -# Download the cuspatial built in the previous step +# Download the cuspatial and libcuspatial built in the previous step RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="libcuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ --no-binary 'fiona' \ "$(echo ./dist/cuspatial*.whl)[test]" \ + "$(echo ./dist/libcuspatial*.whl)" \ 'fiona>=1.8.19,<1.9' rapids-logger "pytest cuspatial" diff --git a/dependencies.yaml b/dependencies.yaml index 588d2910d..b8f5797b0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -110,6 +110,7 @@ files: - build_python - build_wheels - depends_on_libcudf + - depends_on_libcuspatial - depends_on_librmm py_run_cuspatial: output: [pyproject] @@ -119,6 +120,7 @@ files: includes: - depends_on_cudf - depends_on_libcudf + - depends_on_libcuspatial - depends_on_rmm - run_python_cuspatial py_test_cuspatial: diff --git a/python/cuspatial/CMakeLists.txt b/python/cuspatial/CMakeLists.txt index 28fcf467d..c3dc00c8a 100644 --- a/python/cuspatial/CMakeLists.txt +++ b/python/cuspatial/CMakeLists.txt @@ -23,34 +23,9 @@ project( VERSION "${RAPIDS_VERSION}" LANGUAGES CXX CUDA) -option(FIND_CUSPATIAL_CPP "Search for existing cuspatial C++ installations before defaulting to local files" - OFF) - -# If the user requested it we attempt to find cuspatial. -if(FIND_CUSPATIAL_CPP) - find_package(cuspatial "${RAPIDS_VERSION}") -else() - set(cuspatial_FOUND OFF) -endif() - -if(NOT cuspatial_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_BENCHMARKS OFF) - set(CUDA_STATIC_RUNTIME ON) - set(CUSPATIAL_USE_CUDF_STATIC ON) - set(CUSPATIAL_EXCLUDE_CUDF_FROM_ALL ON) - - add_subdirectory(../../cpp cuspatial-cpp EXCLUDE_FROM_ALL) - - set(cython_lib_dir cuspatial) - install(TARGETS cuspatial DESTINATION ${cython_lib_dir}) -endif() +find_package(cuspatial "${RAPIDS_VERSION}" REQUIRED) include(rapids-cython-core) rapids_cython_init() add_subdirectory(cuspatial/_lib) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cuspatial PATHS "${cython_lib_dir}") -endif() diff --git a/python/cuspatial/pyproject.toml b/python/cuspatial/pyproject.toml index 25ea28bfd..f8aba3c4b 100644 --- a/python/cuspatial/pyproject.toml +++ b/python/cuspatial/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "cudf==24.10.*,>=0.0.0a0", "geopandas>=0.11.0", "libcudf==24.10.*,>=0.0.0a0", + "libcuspatial==24.10.*,>=0.0.0a0", "numpy>=1.23,<2.0a0", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -137,6 +138,7 @@ requires = [ "cudf==24.10.*,>=0.0.0a0", "cython>=3.0.0", "libcudf==24.10.*,>=0.0.0a0", + "libcuspatial==24.10.*,>=0.0.0a0", "librmm==24.10.*,>=0.0.0a0", "ninja", "rmm==24.10.*,>=0.0.0a0", diff --git a/python/libcuspatial/libcuspatial/load.py b/python/libcuspatial/libcuspatial/load.py index 117be5cb1..285628187 100644 --- a/python/libcuspatial/libcuspatial/load.py +++ b/python/libcuspatial/libcuspatial/load.py @@ -20,23 +20,26 @@ def load_library(): - # libcudf must be loaded before libcuspatial because libcuspatial references its symbols + # libcudf must be loaded before libcuspatial because libcuspatial + # references its symbols libcudf.load_library() - # Dynamically load libcudf.so. Prefer a system library if one is present to - # avoid clobbering symbols that other packages might expect, but if no - # other library is present use the one in the wheel. + # Dynamically load libcuspatial.so. Prefer a system library if one is + # present to avoid clobbering symbols that other packages might expect, + # but if no other library is present use the one in the wheel. libcuspatial_lib = None try: libcuspatial_lib = ctypes.CDLL("libcuspatial.so", ctypes.RTLD_GLOBAL) except OSError: - # If neither of these directories contain the library, we assume we are in an - # environment where the C++ library is already installed somewhere else and the - # CMake build of the libcudf Python package was a no-op. + # If neither of these directories contain the library, we assume we are + # in an environment where the C++ library is already installed + # somewhere else and the CMake build of the libcuspatial Python package + # was a no-op. # - # Note that this approach won't work for real editable installs of the libcudf package. - # scikit-build-core has limited support for importlib.resources so there isn't a clean - # way to support that case yet. + # Note that this approach won't work for real editable installs of the + # libcuspatial package. scikit-build-core has limited support for + # importlib.resources so there isn't a clean way to support that case + # yet. for lib_dir in ("lib", "lib64"): if os.path.isfile( lib := os.path.join( @@ -48,5 +51,5 @@ def load_library(): # The caller almost never needs to do anything with this library, but no # harm in offering the option since this object at least provides a handle - # to inspect where libcudf was loaded from. + # to inspect where the library was loaded from. return libcuspatial_lib From ad7cd4e43d4a35adac2e7876e30d5954e7bee82d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 29 Aug 2024 12:02:17 -0700 Subject: [PATCH 04/15] add VERSION --- python/libcuspatial/libcuspatial/VERSION | 1 + 1 file changed, 1 insertion(+) create mode 120000 python/libcuspatial/libcuspatial/VERSION diff --git a/python/libcuspatial/libcuspatial/VERSION b/python/libcuspatial/libcuspatial/VERSION new file mode 120000 index 000000000..d62dc733e --- /dev/null +++ b/python/libcuspatial/libcuspatial/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file From d6eba422e98ef0ba7d1587ec565df5f89172c613 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 11:28:36 -0700 Subject: [PATCH 05/15] do not mark ranger for export --- cpp/cmake/thirdparty/get_ranger.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_ranger.cmake b/cpp/cmake/thirdparty/get_ranger.cmake index 563e4120a..cccd71d32 100644 --- a/cpp/cmake/thirdparty/get_ranger.cmake +++ b/cpp/cmake/thirdparty/get_ranger.cmake @@ -26,8 +26,6 @@ function(find_and_configure_ranger) rapids_cpm_find( ranger 00.01.00 GLOBAL_TARGETS "${global_targets}" - BUILD_EXPORT_SET cuspatial-exports - INSTALL_EXPORT_SET cuspatial-exports CPM_ARGS GIT_REPOSITORY https://github.com/harrism/ranger.git GIT_TAG main @@ -35,6 +33,7 @@ function(find_and_configure_ranger) OPTIONS "BUILD_TESTS OFF" FIND_PACKAGE_ARGUMENTS "${find_package_args}" ) + endfunction() find_and_configure_ranger() From 8a39de517d0cb68622df1a88f76c7d1311f299c0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 12:13:41 -0700 Subject: [PATCH 06/15] add library-loading, fix auditwheel exclusions --- ci/build_wheel.sh | 21 ++++++++++++++++++++- cpp/cmake/thirdparty/get_ranger.cmake | 2 +- python/cuspatial/cuspatial/__init__.py | 14 +++++++++++--- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 14802741d..5684f4ad8 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -7,6 +7,25 @@ package_name=$1 package_dir=$2 package_type=$3 +# The 'libcuspatial' wheel should package 'libcuspatial.so', and all others +# should exclude it (they dynamically link load it if they need it). +# +# Capturing that here in argument-parsing to allow this build_wheel.sh +# script to be re-used by all wheel builds in the project. +case "${package_dir}" in + python/libcuspatial) + EXCLUDE_ARGS=( + --exclude "libcudf.so" + ) + ;; + *) + EXCLUDE_ARGS=( + --exclude "libcudf.so" + --exclude "libcuspatial.so" + ) + ;; +esac + source rapids-configure-sccache source rapids-date-string @@ -20,7 +39,7 @@ python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair \ - --exclude libcudf.so \ + "${EXCLUDE_ARGS[@]}" \ -w final_dist \ dist/* diff --git a/cpp/cmake/thirdparty/get_ranger.cmake b/cpp/cmake/thirdparty/get_ranger.cmake index cccd71d32..6441b607d 100644 --- a/cpp/cmake/thirdparty/get_ranger.cmake +++ b/cpp/cmake/thirdparty/get_ranger.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/cuspatial/cuspatial/__init__.py b/python/cuspatial/cuspatial/__init__.py index 14a5fcc4d..89c193819 100644 --- a/python/cuspatial/cuspatial/__init__.py +++ b/python/cuspatial/cuspatial/__init__.py @@ -1,8 +1,16 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. -# load cudf before any other imports, to be sure libcudf's symbols are found -# in the libcudf.so from the libcudf wheel (if one is installed) -import cudf +# If libcuspatial was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libcuspatial +except ModuleNotFoundError: + pass +else: + libcuspatial.load_library() + del libcuspatial + from ._version import __git_commit__, __version__ from .core.geodataframe import GeoDataFrame From 8a09b3a2c1b92ff39695b3df861cd640388df27c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 12:22:24 -0700 Subject: [PATCH 07/15] revert unrelated changes, remove now-unnecessary logic in build.sh --- build.sh | 5 ----- ci/build_wheel.sh | 2 +- cpp/cmake/thirdparty/get_ranger.cmake | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/build.sh b/build.sh index ffe97da9a..77b8fa3eb 100755 --- a/build.sh +++ b/build.sh @@ -124,11 +124,6 @@ if hasArg benchmarks; then BUILD_BENCHMARKS=ON fi -# Append `-DFIND_CUSPATIAL_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUSPATIAL_CPP"* ]]; then - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUSPATIAL_CPP=ON" -fi - # Append `-DFIND_CUPROJ_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUPROJ_CPP"* ]]; then EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_CUPROJ_CPP=ON" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 5684f4ad8..75328d272 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -8,7 +8,7 @@ package_dir=$2 package_type=$3 # The 'libcuspatial' wheel should package 'libcuspatial.so', and all others -# should exclude it (they dynamically link load it if they need it). +# should exclude it (they dynamically load it if they need it). # # Capturing that here in argument-parsing to allow this build_wheel.sh # script to be re-used by all wheel builds in the project. diff --git a/cpp/cmake/thirdparty/get_ranger.cmake b/cpp/cmake/thirdparty/get_ranger.cmake index 6441b607d..c57131f3a 100644 --- a/cpp/cmake/thirdparty/get_ranger.cmake +++ b/cpp/cmake/thirdparty/get_ranger.cmake @@ -33,7 +33,6 @@ function(find_and_configure_ranger) OPTIONS "BUILD_TESTS OFF" FIND_PACKAGE_ARGUMENTS "${find_package_args}" ) - endfunction() find_and_configure_ranger() From 2021ac69be6317880d7f90a7a3a58910b697ff30 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 12:29:57 -0700 Subject: [PATCH 08/15] add job to pr-builder --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7ecffc108..2e1d7020f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,6 +21,7 @@ jobs: - conda-notebook-tests - docs-build - wheel-build-cuspatial + - wheel-build-libcuspatial - wheel-tests-cuspatial - wheel-build-cuproj - wheel-tests-cuproj From 2d78d79c73bfc1bba4fc399c84f09e05b521331a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 13:14:47 -0700 Subject: [PATCH 09/15] fix build script, revert unrelated change --- ci/build_wheel_libcuspatial.sh | 2 +- dependencies.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_libcuspatial.sh b/ci/build_wheel_libcuspatial.sh index 01e699a21..ea62af1df 100755 --- a/ci/build_wheel_libcuspatial.sh +++ b/ci/build_wheel_libcuspatial.sh @@ -3,4 +3,4 @@ set -euo pipefail -ci/build_wheel.sh cuspatial python/libcuspatial cpp +ci/build_wheel.sh libcuspatial python/libcuspatial cpp diff --git a/dependencies.yaml b/dependencies.yaml index df0c869af..463683232 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -165,6 +165,7 @@ files: includes: - test_python_cuproj - depends_on_cuspatial + - test_cuspatial channels: - rapidsai From 815c82af21267a8282b83f4be5041c2ef6da4d70 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 13:50:13 -0700 Subject: [PATCH 10/15] remove commented-out debugging code --- python/libcuspatial/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/libcuspatial/CMakeLists.txt b/python/libcuspatial/CMakeLists.txt index ac1eb3387..c102c2b2c 100644 --- a/python/libcuspatial/CMakeLists.txt +++ b/python/libcuspatial/CMakeLists.txt @@ -31,10 +31,7 @@ endif() unset(cuspatial_FOUND) -# # Find Python early so that later commands can use it -# find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) - -# set options that only have 1 correct values in the context of libcuspatial wheel builds +# set options for building libcuspatial set(BUILD_SHARED_LIBS ON) set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) From 4539088a853852781a513be99ab24105401ec677 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 14:52:33 -0700 Subject: [PATCH 11/15] reduce number of builds --- .github/workflows/build.yaml | 1 + .github/workflows/pr.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dd2179635..5d725f8c0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -70,6 +70,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2e1d7020f..0e234bfb9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -89,6 +89,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: ci/build_wheel_libcuspatial.sh wheel-build-cuspatial: From fe85ee2c398d83b2be0e454c29e6b8261e31ef2a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 30 Aug 2024 15:16:40 -0700 Subject: [PATCH 12/15] add comment on matrix filter --- .github/workflows/build.yaml | 1 + .github/workflows/pr.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5d725f8c0..d86122f5a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -70,6 +70,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: + # build for every combination of arch and CUDA version, but only for the latest Python matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0e234bfb9..94617d953 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -89,6 +89,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: + # build for every combination of arch and CUDA version, but only for the latest Python matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: ci/build_wheel_libcuspatial.sh From 6856666967fe1a962b0162fdd63dacf10dc15c8a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Sep 2024 06:42:33 -0700 Subject: [PATCH 13/15] fix copyright --- ci/build_wheel_libcuspatial.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_libcuspatial.sh b/ci/build_wheel_libcuspatial.sh index ea62af1df..742a58caa 100755 --- a/ci/build_wheel_libcuspatial.sh +++ b/ci/build_wheel_libcuspatial.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. set -euo pipefail From 1d00c91f4e72a0984883d403f13753e22750c86e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Sep 2024 08:19:28 -0700 Subject: [PATCH 14/15] statically link cudart --- python/libcuspatial/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libcuspatial/CMakeLists.txt b/python/libcuspatial/CMakeLists.txt index c102c2b2c..8a16eec49 100644 --- a/python/libcuspatial/CMakeLists.txt +++ b/python/libcuspatial/CMakeLists.txt @@ -36,7 +36,7 @@ set(BUILD_SHARED_LIBS ON) set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) set(CUDA_ENABLE_LINEINFO OFF) -set(CUDA_STATIC_RUNTIME OFF) +set(CUDA_STATIC_RUNTIME ON) set(CUSPATIAL_USE_CUDF_STATIC OFF) set(CUSPATIAL_EXCLUDE_CUDF_FROM_ALL ON) From fb307ba4ac1e0170e0e8d623b70cd913a4bbb077 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 4 Sep 2024 09:33:22 -0700 Subject: [PATCH 15/15] remove unnecessary extra install() --- python/libcuspatial/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/libcuspatial/CMakeLists.txt b/python/libcuspatial/CMakeLists.txt index 8a16eec49..86c3cd1ef 100644 --- a/python/libcuspatial/CMakeLists.txt +++ b/python/libcuspatial/CMakeLists.txt @@ -35,12 +35,8 @@ unset(cuspatial_FOUND) set(BUILD_SHARED_LIBS ON) set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) -set(CUDA_ENABLE_LINEINFO OFF) set(CUDA_STATIC_RUNTIME ON) set(CUSPATIAL_USE_CUDF_STATIC OFF) set(CUSPATIAL_EXCLUDE_CUDF_FROM_ALL ON) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) - add_subdirectory(../../cpp cuspatial-cpp) -install(TARGETS cuspatial DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})