From b43a32efeb6861a5f1095aca747f8ee5994472b4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 29 Aug 2024 10:55:44 -0500 Subject: [PATCH] wheels: dynamically load `libcudf.so` from `libcudf` wheel (#1447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contributes to https://github.com/rapidsai/build-planning/issues/33. Proposes the following for `cuspatial` wheels: * add build and runtime dependencies on `libcudf` wheels * stop vendoring copies of `libcudf.so`, `libnvcomp.so`, `libnvcomp_bitcomp.so`, and `libnvcomp_gdeflate.so` - *(load `libcudf.so` dynamically at runtime instead)* And other related changes for development/CI: * combine all `pip install` calls into 1 in wheel-testing scripts - *like https://github.com/rapidsai/cudf/pull/16575* - *to improve the chance that packaging issues are discovered in CI* * `dependencies.yaml` changes: - more use of YAML anchors = less duplication - use dedicated `depends_on_librmm` and `depends_on_libcudf` groups * explicitly pass a package type to `gha-tools` wheel uploading/downloading scripts ## Notes for Reviewers ### Benefits of these changes Unblocks CI in this repo (ref: https://github.com/rapidsai/cuspatial/pull/1444#issuecomment-2308151120, https://github.com/rapidsai/cuspatial/pull/1441#issuecomment-2308149200). Reduces wheel sizes for `cuspatial` wheels by about 125MB 😁 | wheel | size (before) | size (this PR) | |:-----------:|-------------:|---------------:| | `cuspatial` | 146.0M | 21M | | `cuproj ` | 0.9M | 0.9M | |**TOTAL** | **146.9M** | **21.9M** | *NOTES: size = compressed, "before" = 2024-08-21 nightlies (https://github.com/rapidsai/cuspatial/commit/c60bd4dfc25e62d68cdf520789c6c8bfc577724d), CUDA = 12, Python = 3.11*
how I calculated those (click me) ```shell # note: 2024-08-21 because that was the most recent date with # successfully-built cuspatial nightlies # docker run \ --rm \ -v $(pwd):/opt/work:ro \ -w /opt/work \ --network host \ --env RAPIDS_NIGHTLY_DATE=2024-08-21 \ --env RAPIDS_NIGHTLY_SHA=c60bd4d \ --env RAPIDS_PR_NUMBER=1447 \ --env RAPIDS_PY_CUDA_SUFFIX=cu12 \ --env RAPIDS_REPOSITORY=rapidsai/cuspatial \ --env WHEEL_DIR_BEFORE=/tmp/wheels-before \ --env WHEEL_DIR_AFTER=/tmp/wheels-after \ -it rapidsai/ci-wheel:cuda12.5.1-rockylinux8-py3.11 \ bash mkdir -p "${WHEEL_DIR_BEFORE}" mkdir -p "${WHEEL_DIR_AFTER}" py_projects=( cuspatial cuproj ) for project in "${py_projects[@]}"; do # before RAPIDS_BUILD_TYPE=nightly \ RAPIDS_PY_WHEEL_NAME="${project}_${RAPIDS_PY_CUDA_SUFFIX}" \ RAPIDS_REF_NAME="branch-24.10" \ RAPIDS_SHA=${RAPIDS_NIGHTLY_SHA} \ rapids-download-wheels-from-s3 python "${WHEEL_DIR_BEFORE}" # after RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_PY_WHEEL_NAME="${project}_${RAPIDS_PY_CUDA_SUFFIX}" \ RAPIDS_REF_NAME="pull-request/${RAPIDS_PR_NUMBER}" \ rapids-download-wheels-from-s3 python "${WHEEL_DIR_AFTER}" done du -sh ${WHEEL_DIR_BEFORE}/* du -sh ${WHEEL_DIR_BEFORE} du -sh ${WHEEL_DIR_AFTER}/* du -sh ${WHEEL_DIR_AFTER} ```
Reduces the amount of additional work required to start shipping `libcuspatial` wheels. ### Background This is part of ongoing work towards packaging `libcuspatial` as a wheel. relevant prior work: * packaging `libcudf` wheels: https://github.com/rapidsai/cudf/pull/15483 * consolidating `pip install` calls in CI scripts for `cudf`: https://github.com/rapidsai/cudf/pull/16575 * `cudf` dropping its Arrow library dependency: https://github.com/rapidsai/cudf/pull/16640 ### How I tested this Confirmed in local builds and CI logs that `cudf` is being *found*, not *built*, in `cuspatial` builds. ```text -- CPM: Using local package cudf@24.10.0 ``` ([build link](https://github.com/rapidsai/cuspatial/actions/runs/10602971716/job/29386288614?pr=1447#step:9:23472)) Built `cuspatial` wheels locally and ran all the unit tests, without issue. # Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cuspatial/pull/1447 --- .github/workflows/build.yaml | 2 + ci/build_wheel.sh | 8 ++- ci/build_wheel_cuproj.sh | 4 +- ci/build_wheel_cuspatial.sh | 6 +- ci/test_wheel_cuproj.sh | 16 +++-- ci/test_wheel_cuspatial.sh | 12 ++-- dependencies.yaml | 69 ++++++++++++++++--- python/cuspatial/CMakeLists.txt | 10 +-- .../cmake/Modules/WheelHelpers.cmake | 59 ---------------- python/cuspatial/cuspatial/__init__.py | 6 +- python/cuspatial/pyproject.toml | 4 ++ 11 files changed, 100 insertions(+), 96 deletions(-) delete mode 100644 python/cuspatial/cmake/Modules/WheelHelpers.cmake diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4ecbbab45..1f3fd7922 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -85,6 +85,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cuspatial + package-type: python wheel-build-cuproj: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 @@ -104,3 +105,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cuproj + package-type: python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 5e6eb944e..14802741d 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +package_type=$3 source rapids-configure-sccache source rapids-date-string @@ -18,6 +19,9 @@ cd "${package_dir}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* +python -m auditwheel repair \ + --exclude libcudf.so \ + -w final_dist \ + dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist diff --git a/ci/build_wheel_cuproj.sh b/ci/build_wheel_cuproj.sh index 961f4e363..6001e44e9 100755 --- a/ci/build_wheel_cuproj.sh +++ b/ci/build_wheel_cuproj.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail -ci/build_wheel.sh cuproj python/cuproj +ci/build_wheel.sh cuproj python/cuproj python diff --git a/ci/build_wheel_cuspatial.sh b/ci/build_wheel_cuspatial.sh index 932665e27..171bfa6a4 100755 --- a/ci/build_wheel_cuspatial.sh +++ b/ci/build_wheel_cuspatial.sh @@ -1,8 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail -export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" - -ci/build_wheel.sh cuspatial python/cuspatial +ci/build_wheel.sh cuspatial python/cuspatial python diff --git a/ci/test_wheel_cuproj.sh b/ci/test_wheel_cuproj.sh index 9451190e0..4144e8dab 100755 --- a/ci/test_wheel_cuproj.sh +++ b/ci/test_wheel_cuproj.sh @@ -5,19 +5,21 @@ set -eou pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# Install additional dependencies +# install build dependencies for fiona apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev -python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9' -# Download the cuspatial built in the previous step -RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cuspatial-dep -python -m pip install --no-deps ./local-cuspatial-dep/cuspatial*.whl +# Download the cuproj and cuspatial built in the previous step +RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cuproj*.whl)[test] +python -m pip install \ + --no-binary 'fiona' \ + "$(echo ./dist/cuspatial*.whl)" \ + "$(echo ./dist/cuproj*.whl)[test]" \ + 'fiona>=1.8.19,<1.9' rapids-logger "pytest cuproj" pushd python/cuproj/cuproj diff --git a/ci/test_wheel_cuspatial.sh b/ci/test_wheel_cuspatial.sh index aa36681bc..8faa4e8d3 100755 --- a/ci/test_wheel_cuspatial.sh +++ b/ci/test_wheel_cuspatial.sh @@ -5,15 +5,19 @@ set -eou pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# Install additional dependencies +# install build dependencies for fiona apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev -python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9' + +# Download the cuspatial built in the previous step +RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cuspatial*.whl)[test] +python -m pip install \ + --no-binary 'fiona' \ + "$(echo ./dist/cuspatial*.whl)[test]" \ + 'fiona>=1.8.19,<1.9' rapids-logger "pytest cuspatial" pushd python/cuspatial/cuspatial diff --git a/dependencies.yaml b/dependencies.yaml index 3b4bee329..80822b43d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -17,6 +17,8 @@ files: - depends_on_cudf - depends_on_cuml - depends_on_cupy + - depends_on_libcudf + - depends_on_librmm - rapids_build_skbuild - run_python_cuspatial - test_libcuspatial @@ -77,14 +79,17 @@ files: - build_cpp - build_python - build_wheels + - depends_on_libcudf + - depends_on_librmm py_run_cuspatial: output: [pyproject] pyproject_dir: python/cuspatial extras: table: project includes: - - depends_on_rmm - depends_on_cudf + - depends_on_libcudf + - depends_on_rmm - run_python_cuspatial py_test_cuspatial: output: [pyproject] @@ -141,14 +146,12 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - ninja - - cmake>=3.26.4,!=3.30.0 + - &ninja ninja + - &cmake cmake>=3.26.4,!=3.30.0 - output_types: conda packages: - c-compiler - cxx-compiler - - libcudf==24.10.*,>=0.0.0a0 - - librmm==24.10.*,>=0.0.0a0 - proj - sqlite specific: @@ -184,13 +187,13 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - ninja - - cmake>=3.26.4,!=3.30.0 + - *ninja + - *cmake - output_types: conda packages: - c-compiler - cxx-compiler - - librmm==24.10.*,>=0.0.0a0 + - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0 - proj - sqlite specific: @@ -443,6 +446,31 @@ dependencies: - pylibcudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_unsuffixed]} + depends_on_libcudf: + common: + - output_types: conda + packages: + - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcudf-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcudf-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*libcudf_unsuffixed]} + depends_on_cuml: common: - output_types: conda @@ -523,3 +551,28 @@ dependencies: - libcuspatial==24.10.*,>=0.0.0a0 - cuspatial==24.10.*,>=0.0.0a0 - cuproj==24.10.*,>=0.0.0a0 + + depends_on_librmm: + common: + - output_types: conda + packages: + - *librmm_unsuffixed + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} diff --git a/python/cuspatial/CMakeLists.txt b/python/cuspatial/CMakeLists.txt index e3678c311..28fcf467d 100644 --- a/python/cuspatial/CMakeLists.txt +++ b/python/cuspatial/CMakeLists.txt @@ -43,15 +43,7 @@ if(NOT cuspatial_FOUND) add_subdirectory(../../cpp cuspatial-cpp EXCLUDE_FROM_ALL) set(cython_lib_dir cuspatial) - include(cmake/Modules/WheelHelpers.cmake) - # TODO: This install is currently overzealous. We should only install the libraries that are - # downloaded by CPM during the build, not libraries that were found on the system. However, in - # practice this would only be a problem if libcudf was not found but some of the - # dependencies were, and we have no real use cases where that happens. - install_aliased_imported_targets( - TARGETS cuspatial arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} - ) + install(TARGETS cuspatial DESTINATION ${cython_lib_dir}) endif() include(rapids-cython-core) diff --git a/python/cuspatial/cmake/Modules/WheelHelpers.cmake b/python/cuspatial/cmake/Modules/WheelHelpers.cmake deleted file mode 100644 index 278d6751c..000000000 --- a/python/cuspatial/cmake/Modules/WheelHelpers.cmake +++ /dev/null @@ -1,59 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= -include_guard(GLOBAL) - -# Making libraries available inside wheels by installing the associated targets. -function(install_aliased_imported_targets) - list(APPEND CMAKE_MESSAGE_CONTEXT "install_aliased_imported_targets") - - set(options "") - set(one_value "DESTINATION") - set(multi_value "TARGETS") - cmake_parse_arguments(_ "${options}" "${one_value}" "${multi_value}" ${ARGN}) - - message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__DESTINATION}'") - - foreach(target IN LISTS __TARGETS) - - if(NOT TARGET ${target}) - message(VERBOSE "No target named ${target}") - continue() - endif() - - get_target_property(alias_target ${target} ALIASED_TARGET) - if(alias_target) - set(target ${alias_target}) - endif() - - get_target_property(is_imported ${target} IMPORTED) - if(NOT is_imported) - # If the target isn't imported, install it into the wheel - install(TARGETS ${target} DESTINATION ${__DESTINATION}) - message(VERBOSE "install(TARGETS ${target} DESTINATION ${__DESTINATION})") - else() - # If the target is imported, make sure it's global - get_target_property(type ${target} TYPE) - if(${type} STREQUAL "UNKNOWN_LIBRARY") - install(FILES $ DESTINATION ${__DESTINATION}) - message(VERBOSE "install(FILES $ DESTINATION ${__DESTINATION})") - else() - install(IMPORTED_RUNTIME_ARTIFACTS ${target} DESTINATION ${__DESTINATION}) - message( - VERBOSE - "install(IMPORTED_RUNTIME_ARTIFACTS $ DESTINATION ${__DESTINATION})" - ) - endif() - endif() - endforeach() -endfunction() diff --git a/python/cuspatial/cuspatial/__init__.py b/python/cuspatial/cuspatial/__init__.py index 069b5454d..14a5fcc4d 100644 --- a/python/cuspatial/cuspatial/__init__.py +++ b/python/cuspatial/cuspatial/__init__.py @@ -1,4 +1,8 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +# load cudf before any other imports, to be sure libcudf's symbols are found +# in the libcudf.so from the libcudf wheel (if one is installed) +import cudf from ._version import __git_commit__, __version__ from .core.geodataframe import GeoDataFrame diff --git a/python/cuspatial/pyproject.toml b/python/cuspatial/pyproject.toml index 80f436ea6..25ea28bfd 100644 --- a/python/cuspatial/pyproject.toml +++ b/python/cuspatial/pyproject.toml @@ -32,6 +32,7 @@ requires-python = ">=3.10" dependencies = [ "cudf==24.10.*,>=0.0.0a0", "geopandas>=0.11.0", + "libcudf==24.10.*,>=0.0.0a0", "numpy>=1.23,<2.0a0", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -71,6 +72,7 @@ known_dask = [ known_rapids = [ "rmm", "cudf", + "libcudf", "pylibcudf", ] known_first_party = [ @@ -134,6 +136,8 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cudf==24.10.*,>=0.0.0a0", "cython>=3.0.0", + "libcudf==24.10.*,>=0.0.0a0", + "librmm==24.10.*,>=0.0.0a0", "ninja", "rmm==24.10.*,>=0.0.0a0", "wheel",