From 6b255bde409061267608511825b1aef40e5f443e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 29 Mar 2024 18:29:04 +0000 Subject: [PATCH 01/60] Add libraft package --- python/libraft/CMakeLists.txt | 77 ++++++++++++++++++++++++++++++ python/libraft/LICENSE | 1 + python/libraft/README.md | 1 + python/libraft/libraft/VERSION | 1 + python/libraft/libraft/__init__.py | 19 ++++++++ python/libraft/libraft/_version.py | 25 ++++++++++ python/libraft/pyproject.toml | 61 +++++++++++++++++++++++ 7 files changed, 185 insertions(+) create mode 100644 python/libraft/CMakeLists.txt create mode 120000 python/libraft/LICENSE create mode 120000 python/libraft/README.md create mode 100644 python/libraft/libraft/VERSION create mode 100644 python/libraft/libraft/__init__.py create mode 100644 python/libraft/libraft/_version.py create mode 100644 python/libraft/pyproject.toml diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt new file mode 100644 index 0000000000..7a2d77041d --- /dev/null +++ b/python/libraft/CMakeLists.txt @@ -0,0 +1,77 @@ +# ============================================================================= +# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +# We always need CUDA for pylibraft because the raft dependency brings in a header-only cuco +# dependency that enables CUDA unconditionally. +include(rapids-cuda) +rapids_cuda_init_architectures(pylibraft) + +project( + pylibraft + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" + ON +) + +# If the user requested it we attempt to find RAFT. +if(FIND_RAFT_CPP) + find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) + if(NOT TARGET raft::raft_lib) + message( + FATAL_ERROR + "Building against a preexisting libraft library requires the compiled libraft to have been built!" + ) + + endif() +else() + set(raft_FOUND OFF) +endif() + +include(rapids-cython-core) + +if(NOT raft_FOUND) + set(BUILD_TESTS OFF) + set(BUILD_PRIMS_BENCH OFF) + set(BUILD_ANN_BENCH OFF) + set(RAFT_COMPILE_LIBRARY ON) + set(CUDA_STATIC_RUNTIME ON) + + add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) + + # When building the C++ libraries from source we must copy libraft.so alongside the + # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft + # library, we shouldn't need this + set(cython_lib_dir pylibraft) + install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) +endif() + +rapids_cython_init() + +add_subdirectory(pylibraft/common) +add_subdirectory(pylibraft/distance) +add_subdirectory(pylibraft/matrix) +add_subdirectory(pylibraft/neighbors) +add_subdirectory(pylibraft/random) +add_subdirectory(pylibraft/cluster) + +if(DEFINED cython_lib_dir) + rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") +endif() diff --git a/python/libraft/LICENSE b/python/libraft/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libraft/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libraft/README.md b/python/libraft/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libraft/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION new file mode 100644 index 0000000000..0bff6981a3 --- /dev/null +++ b/python/libraft/libraft/VERSION @@ -0,0 +1 @@ +24.06.00 diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py new file mode 100644 index 0000000000..04aecd1c25 --- /dev/null +++ b/python/libraft/libraft/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file is simply used to make librmm a real package rather than a namespace +# package to work around https://github.com/scikit-build/scikit-build-core/issues/682. +# Since we have it, we may as well also set up some helpful metadata. +from libraft._version import __git_commit__, __version__ diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py new file mode 100644 index 0000000000..8027336e3c --- /dev/null +++ b/python/libraft/libraft/_version.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import importlib.resources + +__version__ = ( + importlib.resources.files("libraft") + .joinpath("VERSION") + .read_text() + .strip() +) +__git_commit__ = "" diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml new file mode 100644 index 0000000000..94abf8eef3 --- /dev/null +++ b/python/libraft/pyproject.toml @@ -0,0 +1,61 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] + +requires = [ + "cmake>=3.26.4", + "ninja", + "librmm==24.6.*", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "scikit_build_core.build" + +[project] +name = "libraft" +dynamic = ["version"] +description = "RAFT: Reusable Algorithms Functions and other Tools" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +classifiers = [ + "Intended Audience :: Developers", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] + +[project.urls] +Homepage = "https://github.com/rapidsai/raft" +Documentation = "https://docs.rapids.ai/api/raft/stable/" + +[project.entry-points."cmake.prefix"] +libraft = "libraft" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.exclude = ["*tests*"] +sdist.reproducible = true +wheel.packages = ["libraft"] +wheel.install-dir = "libraft" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libraft/VERSION" +regex = "(?P.*)" From 27aefecc8240cd93e1eff76c09ffc7eb835c0c81 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 29 Mar 2024 18:37:08 +0000 Subject: [PATCH 02/60] Update dependencies.yaml --- dependencies.yaml | 24 ++++++++++++++++++++++-- python/libraft/pyproject.toml | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 658b08421d..6d75cd7a8c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -8,6 +8,7 @@ files: includes: - build - build_pylibraft + - cython_build - cuda - cuda_version - develop @@ -27,6 +28,7 @@ files: arch: [x86_64, aarch64] includes: - build + - cython_build - cuda - cuda_version - develop @@ -58,6 +60,14 @@ files: - docs - py_version - test_pylibraft + py_build_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: build-system + includes: + - build + - librmm py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -66,6 +76,7 @@ files: includes: - build - build_pylibraft + - cython_build py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -90,6 +101,7 @@ files: table: build-system includes: - build + - cython_build py_run_raft_dask: output: pyproject pyproject_dir: python/raft-dask @@ -131,7 +143,6 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4 - - cython>=3.0.0 - ninja - output_types: [conda] packages: @@ -175,7 +186,16 @@ dependencies: packages: [nvcc_linux-64=11.2] - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - + cython_build: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cython>=3.0.0 + librmm: + common: + - output_types: [requirements, pyproject] + packages: + - librmm==24.6.* build_pylibraft: common: - output_types: [conda] diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index 94abf8eef3..dc664bba22 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -16,8 +16,8 @@ requires = [ "cmake>=3.26.4", - "ninja", "librmm==24.6.*", + "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "scikit_build_core.build" From 86c5c8317d162266d555b126c6f721f84f3cab04 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 29 Mar 2024 19:48:09 +0000 Subject: [PATCH 03/60] Get libraft building --- python/libraft/CMakeLists.txt | 61 ++++++----------------------------- 1 file changed, 9 insertions(+), 52 deletions(-) diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 7a2d77041d..722a811050 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -16,62 +16,19 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) include(../../rapids_config.cmake) -# We always need CUDA for pylibraft because the raft dependency brings in a header-only cuco -# dependency that enables CUDA unconditionally. include(rapids-cuda) -rapids_cuda_init_architectures(pylibraft) +rapids_cuda_init_architectures(libraft-python) project( - pylibraft + libraft-python VERSION "${RAPIDS_VERSION}" LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - ON -) - -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) - if(NOT TARGET raft::raft_lib) - message( - FATAL_ERROR - "Building against a preexisting libraft library requires the compiled libraft to have been built!" - ) - - endif() -else() - set(raft_FOUND OFF) -endif() - -include(rapids-cython-core) - -if(NOT raft_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(BUILD_ANN_BENCH OFF) - set(RAFT_COMPILE_LIBRARY ON) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - - # When building the C++ libraries from source we must copy libraft.so alongside the - # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft - # library, we shouldn't need this - set(cython_lib_dir pylibraft) - install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) -endif() - -rapids_cython_init() - -add_subdirectory(pylibraft/common) -add_subdirectory(pylibraft/distance) -add_subdirectory(pylibraft/matrix) -add_subdirectory(pylibraft/neighbors) -add_subdirectory(pylibraft/random) -add_subdirectory(pylibraft/cluster) +set(BUILD_TESTS OFF) +set(BUILD_PRIMS_BENCH OFF) +set(BUILD_ANN_BENCH OFF) +set(RAFT_COMPILE_LIBRARY ON) +set(CUDA_STATIC_RUNTIME ON) -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") -endif() +add_subdirectory(../../cpp raft-cpp) From 6fd0c64852878b308c564045a512e91972d734c7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 29 Mar 2024 19:50:29 +0000 Subject: [PATCH 04/60] Add libraft as dependency --- dependencies.yaml | 3 +++ python/pylibraft/pyproject.toml | 1 + python/raft-dask/pyproject.toml | 1 + 3 files changed, 5 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 6d75cd7a8c..4c020921ed 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -191,6 +191,9 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cython>=3.0.0 + - output_types: [requirements, pyproject] + packages: + - libraft==24.6.* librmm: common: - output_types: [requirements, pyproject] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 3e8ca0b6d3..a8781e61b8 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -18,6 +18,7 @@ requires = [ "cmake>=3.26.4", "cuda-python>=11.7.1,<12.0a0", "cython>=3.0.0", + "libraft==24.6.*", "ninja", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 815f6b277c..7065cccd73 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -18,6 +18,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libraft==24.6.*", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From d360ae76227366b2f0021cfe5cc89206b18d2d06 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 29 Mar 2024 19:57:01 +0000 Subject: [PATCH 05/60] Make librmm a runtime dependency of libraft --- dependencies.yaml | 11 +++++++++++ python/libraft/pyproject.toml | 3 +++ 2 files changed, 14 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 4c020921ed..717871a8bb 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -68,6 +68,17 @@ files: includes: - build - librmm + py_run_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: project + includes: + # This is really a build requirement for anything using libraft to build + # against, but is not required when _running_ with libraft. There isn't a + # great way to express that without separating libraft into libraft and + # libraft-dev packages, though. + - librmm py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index dc664bba22..9e867dadc8 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -37,6 +37,9 @@ classifiers = [ "Programming Language :: C++", "Environment :: GPU :: NVIDIA CUDA", ] +dependencies = [ + "librmm==24.6.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/raft" From 2aac72a6e7c52fe9953e7f46056d502c4473a971 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 30 Mar 2024 16:54:10 +0000 Subject: [PATCH 06/60] Change pylibraft to use libraft wheel --- python/libraft/pyproject.toml | 5 ++ python/pylibraft/CMakeLists.txt | 61 +++++++++---------- .../pylibraft/cluster/CMakeLists.txt | 5 +- .../pylibraft/pylibraft/common/CMakeLists.txt | 5 +- .../pylibraft/distance/CMakeLists.txt | 3 + .../pylibraft/pylibraft/matrix/CMakeLists.txt | 5 +- .../pylibraft/neighbors/CMakeLists.txt | 3 + .../pylibraft/neighbors/cagra/CMakeLists.txt | 5 +- .../neighbors/ivf_flat/CMakeLists.txt | 5 +- .../pylibraft/neighbors/ivf_pq/CMakeLists.txt | 5 +- .../pylibraft/pylibraft/random/CMakeLists.txt | 5 +- 11 files changed, 68 insertions(+), 39 deletions(-) diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index 9e867dadc8..cdf9a7a4f4 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -57,6 +57,11 @@ sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["libraft"] wheel.install-dir = "libraft" +# TODO: We don't want to install the static library because it dramatically +# increases the size. Unfortunately this also means we have to specify all the +# transitive dependencies pulled via CPM manually. Not sure yet if there is a +# good alternative. +#install.components = ["raft", "compiled", "hnswlib", "nvidiacutlass"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 7a2d77041d..5b96fc2a65 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,42 +27,43 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - ON -) +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) - if(NOT TARGET raft::raft_lib) - message( - FATAL_ERROR - "Building against a preexisting libraft library requires the compiled libraft to have been built!" - ) +# Register a library that is installed to site-packages for rpath linking +function(rapids_cython_register_site_packages_library target) + # TODO: Is this solution robust? Should we also check for other configs, or the default IMPORTED_LOCATION? + get_property(target_library TARGET "${target}" PROPERTY IMPORTED_LOCATION_RELEASE) + cmake_path(GET target_library PARENT_PATH library_dir) + find_package(Python REQUIRED) + # TODO: Should we support other install locations like user site? + cmake_path(IS_PREFIX Python_SITELIB "${library_dir}" library_is_in_site) + # We cannot support libraries that are not installed to site-packages + if(NOT library_is_in_site) + message(FATAL_ERROR "Library ${target} is not installed to site-packages. This is not supported.") endif() -else() - set(raft_FOUND OFF) -endif() -include(rapids-cython-core) + cmake_path(RELATIVE_PATH library_dir BASE_DIRECTORY "${Python_SITELIB}" + OUTPUT_VARIABLE library_relative_path) + set_property(GLOBAL PROPERTY "rapids_cython_registered_libraries_${target}" "${library_relative_path}") +endfunction() -if(NOT raft_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(BUILD_ANN_BENCH OFF) - set(RAFT_COMPILE_LIBRARY ON) - set(CUDA_STATIC_RUNTIME ON) +# TODO: We have to use raft_lib for this because we can't get the required +# location out of raft::raft or raft::compiled. Is there a better way? +rapids_cython_register_site_packages_library(raft::raft_lib) - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) +function(rapids_cython_link_to_site_packages_library target linked_target) + get_property(library_relative_path GLOBAL PROPERTY "rapids_cython_registered_libraries_${linked_target}") + if(NOT library_relative_path) + message(FATAL_ERROR "Library ${linked_target} is not registered as a site-packages library.") + endif() - # When building the C++ libraries from source we must copy libraft.so alongside the - # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft - # library, we shouldn't need this - set(cython_lib_dir pylibraft) - install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) -endif() + cmake_path(RELATIVE_PATH PROJECT_SOURCE_DIR BASE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" + OUTPUT_VARIABLE path_to_site) + set_property(TARGET "${target}" APPEND PROPERTY INSTALL_RPATH "\$ORIGIN/${path_to_site}/${library_relative_path}") +endfunction() +include(rapids-cython-core) rapids_cython_init() add_subdirectory(pylibraft/common) @@ -71,7 +72,3 @@ add_subdirectory(pylibraft/matrix) add_subdirectory(pylibraft/neighbors) add_subdirectory(pylibraft/random) add_subdirectory(pylibraft/cluster) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") -endif() diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 7d6e05d918..a6980dc9f1 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,3 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 6ce1dfe347..08bd7b4500 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,3 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index 2530e07a98..ed6c1b062b 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -22,3 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt index ffba10dea9..44bf455c90 100644 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ b/python/pylibraft/pylibraft/matrix/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,3 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt index 069038a0e8..17d84165da 100644 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt @@ -22,6 +22,9 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() add_subdirectory(cagra) add_subdirectory(ivf_flat) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt index 441bb0b311..798a4c6862 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,3 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt index 8f395faec9..027c1f8f58 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,3 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt index e3d721a6ea..f15cc5f620 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,3 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index fcc5ee6311..88fbc89280 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -25,3 +25,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ ) +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) +endforeach() From cea97463de9c8166d0b3e85dc550f16c34142f7c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 30 Mar 2024 16:56:17 +0000 Subject: [PATCH 07/60] Don't error --- python/pylibraft/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 5b96fc2a65..ccbdc7b27a 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -39,8 +39,13 @@ function(rapids_cython_register_site_packages_library target) # TODO: Should we support other install locations like user site? cmake_path(IS_PREFIX Python_SITELIB "${library_dir}" library_is_in_site) # We cannot support libraries that are not installed to site-packages + # TODO: We do need to still allow building in such cases. For now the easiest + # thing I can think of is just a status message here because a warning seems + # pretty loud for standard uses cases like conda or devcontainers where this + # won't link. if(NOT library_is_in_site) - message(FATAL_ERROR "Library ${target} is not installed to site-packages. This is not supported.") + message(STATUS "Library ${target} is not installed to site-packages. This is not supported.") + return() endif() cmake_path(RELATIVE_PATH library_dir BASE_DIRECTORY "${Python_SITELIB}" @@ -55,7 +60,8 @@ rapids_cython_register_site_packages_library(raft::raft_lib) function(rapids_cython_link_to_site_packages_library target linked_target) get_property(library_relative_path GLOBAL PROPERTY "rapids_cython_registered_libraries_${linked_target}") if(NOT library_relative_path) - message(FATAL_ERROR "Library ${linked_target} is not registered as a site-packages library.") + message(STATUS "Library ${linked_target} is not registered as a site-packages library, not updating RPATH of ${target}") + return() endif() cmake_path(RELATIVE_PATH PROJECT_SOURCE_DIR BASE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" From 834c252531931f01d5d488a6eba16d77b908d9a3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 30 Mar 2024 17:31:50 +0000 Subject: [PATCH 08/60] Fix name --- python/libraft/libraft/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py index 04aecd1c25..4001b09601 100644 --- a/python/libraft/libraft/__init__.py +++ b/python/libraft/libraft/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. # -# This file is simply used to make librmm a real package rather than a namespace +# This file is simply used to make libraft a real package rather than a namespace # package to work around https://github.com/scikit-build/scikit-build-core/issues/682. # Since we have it, we may as well also set up some helpful metadata. from libraft._version import __git_commit__, __version__ From 23769372b0e87c81b8bf2326762eb5d8edc91169 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 18:57:46 +0000 Subject: [PATCH 09/60] Remove RPATH hacking in favor of dynamic loading. --- python/libraft/libraft/__init__.py | 17 ++++++-- python/libraft/pyproject.toml | 10 +++-- python/pylibraft/CMakeLists.txt | 40 ------------------- python/pylibraft/pylibraft/__init__.py | 10 ++++- .../pylibraft/cluster/CMakeLists.txt | 5 +-- .../pylibraft/pylibraft/common/CMakeLists.txt | 5 +-- .../pylibraft/distance/CMakeLists.txt | 3 -- .../pylibraft/pylibraft/matrix/CMakeLists.txt | 5 +-- .../pylibraft/neighbors/CMakeLists.txt | 3 -- .../pylibraft/neighbors/cagra/CMakeLists.txt | 5 +-- .../neighbors/ivf_flat/CMakeLists.txt | 5 +-- .../pylibraft/neighbors/ivf_pq/CMakeLists.txt | 5 +-- .../pylibraft/pylibraft/random/CMakeLists.txt | 5 +-- 13 files changed, 36 insertions(+), 82 deletions(-) diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py index 4001b09601..43a1183911 100644 --- a/python/libraft/libraft/__init__.py +++ b/python/libraft/libraft/__init__.py @@ -13,7 +13,18 @@ # limitations under the License. # -# This file is simply used to make libraft a real package rather than a namespace -# package to work around https://github.com/scikit-build/scikit-build-core/issues/682. -# Since we have it, we may as well also set up some helpful metadata. +# Dynamically load libraft.so. Prefer a system library if one is present to avoid clobbering symbols that other packages might expect, but if no other library is present use the one in the wheel. +import ctypes +import os + from libraft._version import __git_commit__, __version__ + +try: + libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) +except OSError: + libraft_lib = ctypes.CDLL( + # TODO: Do we always know it will be lib64? Should we consider finding a way for + # CMake to export the path for us to find here? + os.path.join(os.path.dirname(__file__), "lib64", "libraft.so"), + ctypes.RTLD_GLOBAL, + ) diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index cdf9a7a4f4..bb383b9297 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -58,10 +58,12 @@ sdist.reproducible = true wheel.packages = ["libraft"] wheel.install-dir = "libraft" # TODO: We don't want to install the static library because it dramatically -# increases the size. Unfortunately this also means we have to specify all the -# transitive dependencies pulled via CPM manually. Not sure yet if there is a -# good alternative. -#install.components = ["raft", "compiled", "hnswlib", "nvidiacutlass"] +# increases the size. Unfortunately we also need to install a number of pieces +# beyond the contents of the compiled component, including (but not limited to) +# the transitive dependencies pulled via CPM manually. There doesn't seem to be +# a good solution for this other than adding an option to the CPP CMake to not +# create the static library. +#install.components = ["compiled"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index ccbdc7b27a..bba5549f62 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -29,46 +29,6 @@ project( find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) -# Register a library that is installed to site-packages for rpath linking -function(rapids_cython_register_site_packages_library target) - # TODO: Is this solution robust? Should we also check for other configs, or the default IMPORTED_LOCATION? - get_property(target_library TARGET "${target}" PROPERTY IMPORTED_LOCATION_RELEASE) - cmake_path(GET target_library PARENT_PATH library_dir) - - find_package(Python REQUIRED) - # TODO: Should we support other install locations like user site? - cmake_path(IS_PREFIX Python_SITELIB "${library_dir}" library_is_in_site) - # We cannot support libraries that are not installed to site-packages - # TODO: We do need to still allow building in such cases. For now the easiest - # thing I can think of is just a status message here because a warning seems - # pretty loud for standard uses cases like conda or devcontainers where this - # won't link. - if(NOT library_is_in_site) - message(STATUS "Library ${target} is not installed to site-packages. This is not supported.") - return() - endif() - - cmake_path(RELATIVE_PATH library_dir BASE_DIRECTORY "${Python_SITELIB}" - OUTPUT_VARIABLE library_relative_path) - set_property(GLOBAL PROPERTY "rapids_cython_registered_libraries_${target}" "${library_relative_path}") -endfunction() - -# TODO: We have to use raft_lib for this because we can't get the required -# location out of raft::raft or raft::compiled. Is there a better way? -rapids_cython_register_site_packages_library(raft::raft_lib) - -function(rapids_cython_link_to_site_packages_library target linked_target) - get_property(library_relative_path GLOBAL PROPERTY "rapids_cython_registered_libraries_${linked_target}") - if(NOT library_relative_path) - message(STATUS "Library ${linked_target} is not registered as a site-packages library, not updating RPATH of ${target}") - return() - endif() - - cmake_path(RELATIVE_PATH PROJECT_SOURCE_DIR BASE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" - OUTPUT_VARIABLE path_to_site) - set_property(TARGET "${target}" APPEND PROPERTY INSTALL_RPATH "\$ORIGIN/${path_to_site}/${library_relative_path}") -endfunction() - include(rapids-cython-core) rapids_cython_init() diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index 3b67a5f951..30e48fa37d 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,3 +14,11 @@ # from pylibraft._version import __git_commit__, __version__ + +# Importing the libraft wheel loads libraft symbols at runtime. If the wheel is not present, then libraft was installed in a different way. +try: + import libraft + + del libraft +except ModuleNotFoundError: + pass diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index a6980dc9f1..7d6e05d918 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,6 +22,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 08bd7b4500..6ce1dfe347 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,6 +22,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index ed6c1b062b..2530e07a98 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -22,6 +22,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt index 44bf455c90..ffba10dea9 100644 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ b/python/pylibraft/pylibraft/matrix/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,6 +22,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt index 17d84165da..069038a0e8 100644 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt @@ -22,9 +22,6 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() add_subdirectory(cagra) add_subdirectory(ivf_flat) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt index 798a4c6862..441bb0b311 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,6 +22,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt index 027c1f8f58..8f395faec9 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,6 +22,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt index f15cc5f620..e3d721a6ea 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -22,6 +22,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index 88fbc89280..fcc5ee6311 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -25,6 +25,3 @@ rapids_cython_create_modules( SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ ) -foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - rapids_cython_link_to_site_packages_library(${target} raft::raft_lib) -endforeach() From c252e354e1637c12da318e7f156860f3fc6831f4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 18:59:33 +0000 Subject: [PATCH 10/60] Remove now extraneous ASSOCIATED_TARGETS --- python/pylibraft/pylibraft/cluster/CMakeLists.txt | 4 ++-- python/pylibraft/pylibraft/common/CMakeLists.txt | 4 ++-- python/pylibraft/pylibraft/distance/CMakeLists.txt | 2 +- python/pylibraft/pylibraft/matrix/CMakeLists.txt | 4 ++-- python/pylibraft/pylibraft/neighbors/CMakeLists.txt | 2 +- python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt | 4 ++-- python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt | 4 ++-- python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt | 4 ++-- python/pylibraft/pylibraft/random/CMakeLists.txt | 4 ++-- python/raft-dask/raft_dask/common/CMakeLists.txt | 4 ++-- python/raft-dask/raft_dask/include_test/CMakeLists.txt | 4 ++-- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 7d6e05d918..06a639436a 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cluster_ ) diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 6ce1dfe347..d1c1acb3aa 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ ) diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index 2530e07a98..ffcef45c32 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX distance_ ) diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt index ffba10dea9..07d35325a5 100644 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ b/python/pylibraft/pylibraft/matrix/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX matrix_ ) diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt index 069038a0e8..2a954183d3 100644 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt @@ -20,7 +20,7 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ ) add_subdirectory(cagra) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt index 441bb0b311..2df03c7b0b 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_cagra_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt index 8f395faec9..f50051ba23 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfflat_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt index e3d721a6ea..e57798fcc6 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfpq_ ) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index fcc5ee6311..7d61855111 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ ) diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 3798b5ac4b..49dee15d8f 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -15,6 +15,6 @@ set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft raft::distributed) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX ) diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index e588ce1d1e..8475bcaa93 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -15,6 +15,6 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX ) From bd57f69d9611f453591c4e52376bfcc0509ad92c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 31 Mar 2024 23:55:08 +0000 Subject: [PATCH 11/60] Make loading the library opt-in rather than automatic on import --- python/libraft/libraft/__init__.py | 15 +---------- python/libraft/libraft/_version.py | 2 +- python/libraft/libraft/load.py | 37 ++++++++++++++++++++++++++ python/pylibraft/pylibraft/__init__.py | 8 +++--- 4 files changed, 44 insertions(+), 18 deletions(-) create mode 100644 python/libraft/libraft/load.py diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py index 43a1183911..2ba8e06d56 100644 --- a/python/libraft/libraft/__init__.py +++ b/python/libraft/libraft/__init__.py @@ -13,18 +13,5 @@ # limitations under the License. # -# Dynamically load libraft.so. Prefer a system library if one is present to avoid clobbering symbols that other packages might expect, but if no other library is present use the one in the wheel. -import ctypes -import os - from libraft._version import __git_commit__, __version__ - -try: - libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) -except OSError: - libraft_lib = ctypes.CDLL( - # TODO: Do we always know it will be lib64? Should we consider finding a way for - # CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib64", "libraft.so"), - ctypes.RTLD_GLOBAL, - ) +from libraft.load import load_library diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py index 8027336e3c..3e3792a85c 100644 --- a/python/libraft/libraft/_version.py +++ b/python/libraft/libraft/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py new file mode 100644 index 0000000000..317a68e9e4 --- /dev/null +++ b/python/libraft/libraft/load.py @@ -0,0 +1,37 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + + +def load_library(): + # Dynamically load libraft.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) + except OSError: + libraft_lib = ctypes.CDLL( + # TODO: Do we always know it will be lib64? Should we consider + # finding a way for CMake to export the path for us to find here? + os.path.join(os.path.dirname(__file__), "lib64", "libraft.so"), + ctypes.RTLD_GLOBAL, + ) + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libraft was loaded from. + return libraft_lib diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index 30e48fa37d..8aac8f93da 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -15,10 +15,12 @@ from pylibraft._version import __git_commit__, __version__ -# Importing the libraft wheel loads libraft symbols at runtime. If the wheel is not present, then libraft was installed in a different way. +# If libraft was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. try: import libraft - - del libraft except ModuleNotFoundError: pass +else: + libraft.load_library() + del libraft From 3cf64c6786c5f13f10a0780e91e5d58a4df0beae Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 01:57:09 +0000 Subject: [PATCH 12/60] Support disabling the static library build --- cpp/CMakeLists.txt | 74 +++++++++++++++++++++++------------ python/libraft/CMakeLists.txt | 1 + python/libraft/pyproject.toml | 7 ---- 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cbae4bfb3f..9e45313803 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -75,9 +75,11 @@ if((BUILD_TESTS ) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() -option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" +option(RAFT_COMPILE_LIBRARY "Enable building raft library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) +option(RAFT_COMPILE_DYNAMIC_ONLY "Only build the static library and skip the +static library. Has no effect if RAFT_COMPILE_LIBRARY is OFF" OFF) if(BUILD_CPU_ONLY) set(BUILD_SHARED_LIBS OFF) @@ -582,17 +584,25 @@ if(RAFT_COMPILE_LIBRARY) ) add_library(raft_lib SHARED $) - add_library(raft_lib_static STATIC $) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + add_library(raft_lib_static STATIC $) + endif() + + set(raft_lib_targets raft_lib) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_lib_targets raft_lib_static) + endif() set_target_properties( - raft_lib raft_lib_static + ${raft_lib_targets} PROPERTIES OUTPUT_NAME raft BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) - foreach(target raft_lib raft_lib_static raft_objs) + list(APPEND raft_lib_targets raft_objs) + foreach(target IN LISTS raft_lib_targets) target_link_libraries( ${target} PUBLIC raft::raft @@ -617,21 +627,23 @@ target_link_libraries(raft_compiled INTERFACE raft::raft $ + ) endif() -target_link_libraries( - raft_compiled_static INTERFACE raft::raft $ -) - # ################################################################################################## # * raft_distributed ------------------------------------------------------------------------------- add_library(raft_distributed INTERFACE) @@ -670,8 +682,12 @@ install( EXPORT raft-exports ) +set(raft_compiled_install_targets raft_compiled) +if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_compiled_install_targets raft_compiled_static) +endif() install( - TARGETS raft_compiled raft_compiled_static + TARGETS ${raft_compiled_install_targets} DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -684,12 +700,14 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - install( - TARGETS raft_lib_static - DESTINATION ${lib_dir} - COMPONENT compiled-static - EXPORT raft-compiled-static-lib-exports - ) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + install( + TARGETS raft_lib_static + DESTINATION ${lib_dir} + COMPONENT compiled-static + EXPORT raft-compiled-static-lib-exports + ) + endif() install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -760,8 +778,12 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - list(APPEND raft_components compiled compiled-static) - list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) + list(APPEND raft_components compiled) + list(APPEND raft_export_sets raft-compiled-lib-exports) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_components compiled-static) + list(APPEND raft_export_sets raft-compiled-static-lib-exports) + endif() endif() string( diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index 722a811050..df48e2a4f6 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -29,6 +29,7 @@ set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) set(BUILD_ANN_BENCH OFF) set(RAFT_COMPILE_LIBRARY ON) +set(RAFT_COMPILE_DYNAMIC_ONLY ON) set(CUDA_STATIC_RUNTIME ON) add_subdirectory(../../cpp raft-cpp) diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index bb383b9297..9e867dadc8 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -57,13 +57,6 @@ sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["libraft"] wheel.install-dir = "libraft" -# TODO: We don't want to install the static library because it dramatically -# increases the size. Unfortunately we also need to install a number of pieces -# beyond the contents of the compiled component, including (but not limited to) -# the transitive dependencies pulled via CPM manually. There doesn't seem to be -# a good solution for this other than adding an option to the CPP CMake to not -# create the static library. -#install.components = ["compiled"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 97b26921f4acea2ba67077c8ddaf2e9464010018 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:12:19 +0000 Subject: [PATCH 13/60] Add build in CI --- .github/workflows/pr.yaml | 11 ++++++++- ci/build_wheel.sh | 5 ++-- ci/build_wheel_cpp.sh | 48 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100755 ci/build_wheel_cpp.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c2d9556859..e463f4f1ba 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,6 +19,7 @@ jobs: - conda-python-build - conda-python-tests - docs-build + - wheel-build-cpp - wheel-build-pylibraft - wheel-tests-pylibraft - wheel-build-raft-dask @@ -74,10 +75,18 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-pylibraft: + wheel-build-cpp: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + build_type: pull-request + script: ci/build_wheel_cpp.sh + wheel-build-pylibraft: + needs: wheel-build-cpp + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 5d06e46303..17ca6d2e21 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail @@ -14,6 +14,7 @@ version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 ./dist # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -53,7 +54,7 @@ fi cd "${package_dir}" # Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="${PWD}/dist" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh new file mode 100755 index 0000000000..b5b5756a06 --- /dev/null +++ b/ci/build_wheel_cpp.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libraft" +package_dir="python/libraft" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + +cd "${package_dir}" + +# Hardcode the output dir +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist From e075ab8d5b2da7115adfbde79e918df2fdbac3c5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:12:47 +0000 Subject: [PATCH 14/60] WAR dependencies --- dependencies.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 717871a8bb..c49f287d6d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -207,7 +207,8 @@ dependencies: - libraft==24.6.* librmm: common: - - output_types: [requirements, pyproject] + #- output_types: [requirements, pyproject] + - output_types: [pyproject] packages: - librmm==24.6.* build_pylibraft: From ec178d53fd0490ab332f4d343f23f1d3ce7e1e49 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:22:56 +0000 Subject: [PATCH 15/60] Combine Python wheel building steps --- .github/workflows/pr.yaml | 14 ++---- ci/build_wheel.sh | 62 -------------------------- ci/build_wheel_pylibraft.sh | 9 ---- ci/build_wheel_python.sh | 89 +++++++++++++++++++++++++++++++++++++ ci/build_wheel_raft_dask.sh | 9 ---- 5 files changed, 92 insertions(+), 91 deletions(-) delete mode 100755 ci/build_wheel.sh delete mode 100755 ci/build_wheel_pylibraft.sh create mode 100755 ci/build_wheel_python.sh delete mode 100755 ci/build_wheel_raft_dask.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e463f4f1ba..9e5f1cd9df 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,9 +20,8 @@ jobs: - conda-python-tests - docs-build - wheel-build-cpp - - wheel-build-pylibraft + - wheel-build-python - wheel-tests-pylibraft - - wheel-build-raft-dask - wheel-tests-raft-dask - devcontainer secrets: inherit @@ -83,13 +82,13 @@ jobs: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: ci/build_wheel_cpp.sh - wheel-build-pylibraft: + wheel-build-python: needs: wheel-build-cpp secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request - script: ci/build_wheel_pylibraft.sh + script: ci/build_wheel_python.sh wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit @@ -97,13 +96,6 @@ jobs: with: build_type: pull-request script: ci/test_wheel_pylibraft.sh - wheel-build-raft-dask: - needs: wheel-tests-pylibraft - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 - with: - build_type: pull-request - script: "ci/build_wheel_raft_dask.sh" wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh deleted file mode 100755 index 17ca6d2e21..0000000000 --- a/ci/build_wheel.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_name=$1 -package_dir=$2 -underscore_package_name=$(echo "${package_name}" | tr "-" "_") - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 ./dist - -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${underscore_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -if [[ ${package_name} == "raft-dask" ]]; then - sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} -else - sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -fi - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi - -cd "${package_dir}" - -# Hardcode the output dir -PIP_FIND_LINKS="${PWD}/dist" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* - -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh deleted file mode 100755 index ec30a28b92..0000000000 --- a/ci/build_wheel_pylibraft.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -set -euo pipefail - -# Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" - -ci/build_wheel.sh pylibraft python/pylibraft diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh new file mode 100755 index 0000000000..e846eedd08 --- /dev/null +++ b/ci/build_wheel_python.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 ./libraft_dist + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +echo "${version}" > VERSION +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + + +############################################### +# Build pylibraft + +package_name="pylibraft" +package_dir="python/pylibraft" + +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi + +pushd "${package_dir}" + +PIP_FIND_LINKS="${PWD}/libraft_dist" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check + +mkdir -p pylibraft_final_dist +python -m auditwheel repair -w pylibraft_final_dist pylibraft_dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 pylibraft_final_dist + +popd + + +############################################### +# Build raft-dask + +package_name=raft-dask +package_dir=python/raft-dask +underscore_package_name=$(echo "${package_name}" | tr "-" "_") + +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${underscore_package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} + +pushd "${package_dir}" + +PIP_FIND_LINKS="${PWD}/pylibraft_dist;${PWD}/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check + +mkdir -p raft_dask_final_dist +python -m auditwheel repair -w raft_dask_final_dist raft_dask_dist/* + +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 raft_dask_final_dist + +popd diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh deleted file mode 100755 index 5ae12303d0..0000000000 --- a/ci/build_wheel_raft_dask.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -set -euo pipefail - -# Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" - -ci/build_wheel.sh raft-dask python/raft-dask From 070c17157127ad5a3dbd7e9a6a2540b4a3dc17b0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:29:08 +0000 Subject: [PATCH 16/60] Temporarily disable raft-dask builds since the package hasn't been updated yet --- ci/build_wheel_python.sh | 58 ++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index e846eedd08..efebf9cd85 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -58,32 +58,32 @@ RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wh popd -############################################### -# Build raft-dask - -package_name=raft-dask -package_dir=python/raft-dask -underscore_package_name=$(echo "${package_name}" | tr "-" "_") - -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${underscore_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} - -pushd "${package_dir}" - -PIP_FIND_LINKS="${PWD}/pylibraft_dist;${PWD}/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check - -mkdir -p raft_dask_final_dist -python -m auditwheel repair -w raft_dask_final_dist raft_dask_dist/* - -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 raft_dask_final_dist - -popd +################################################ +## Build raft-dask +# +#package_name=raft-dask +#package_dir=python/raft-dask +#underscore_package_name=$(echo "${package_name}" | tr "-" "_") +# +#pyproject_file="${package_dir}/pyproject.toml" +#version_file="${package_dir}/${underscore_package_name}/_version.py" +# +#sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +#sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} +# +#sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +#sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +#sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +#sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} +#sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} +# +#pushd "${package_dir}" +# +#PIP_FIND_LINKS="${PWD}/pylibraft_dist;${PWD}/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check +# +#mkdir -p raft_dask_final_dist +#python -m auditwheel repair -w raft_dask_final_dist raft_dask_dist/* +# +#RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 raft_dask_final_dist +# +#popd From 2764c9fc84d19a76a29e57314eaffede704e12f4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:35:04 +0000 Subject: [PATCH 17/60] Pull rmm from the PR --- ci/build_wheel_python.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index efebf9cd85..2d703700a6 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -30,6 +30,13 @@ fi ############################################### # Build pylibraft +set -euo pipefail + +# TODO: Generalize gha tool for getting conda artifacts to wheels. +artifact_name=$(RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + package_name="pylibraft" package_dir="python/pylibraft" @@ -48,7 +55,7 @@ fi pushd "${package_dir}" -PIP_FIND_LINKS="${PWD}/libraft_dist" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="${PWD}/libraft_dist;${librmm_wheelhouse}" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check mkdir -p pylibraft_final_dist python -m auditwheel repair -w pylibraft_final_dist pylibraft_dist/* From 833507c475fe956562eec0dda456bbb597a35051 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:36:34 +0000 Subject: [PATCH 18/60] Fix GHA --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9e5f1cd9df..5d0368e3f7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -90,14 +90,14 @@ jobs: build_type: pull-request script: ci/build_wheel_python.sh wheel-tests-pylibraft: - needs: wheel-build-pylibraft + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: - needs: wheel-build-raft-dask + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: From 0fb2aae5e32f85dd1788c5dacfd251ccae46bea7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:41:37 +0000 Subject: [PATCH 19/60] Disambiguate dists --- ci/build_wheel_python.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 2d703700a6..6005878592 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -10,7 +10,7 @@ version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 ./libraft_dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 /tmp/libraft_dist # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -55,7 +55,7 @@ fi pushd "${package_dir}" -PIP_FIND_LINKS="${PWD}/libraft_dist;${librmm_wheelhouse}" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="/tmp/libraft_dist;${librmm_wheelhouse}" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check mkdir -p pylibraft_final_dist python -m auditwheel repair -w pylibraft_final_dist pylibraft_dist/* @@ -86,7 +86,7 @@ popd # #pushd "${package_dir}" # -#PIP_FIND_LINKS="${PWD}/pylibraft_dist;${PWD}/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check +#PIP_FIND_LINKS="../pylibraft/pylibraft_dist;/tmp/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check # #mkdir -p raft_dask_final_dist #python -m auditwheel repair -w raft_dask_final_dist raft_dask_dist/* From f256bd1cd921c05071125827dece8b11743d03b0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 21:52:59 +0000 Subject: [PATCH 20/60] Download librmm for the libraft build --- ci/build_wheel_cpp.sh | 7 +++++-- ci/build_wheel_python.sh | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index b5b5756a06..e0e68f1c47 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -3,6 +3,10 @@ set -euo pipefail +artifact_name=$(RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + package_name="libraft" package_dir="python/libraft" @@ -39,8 +43,7 @@ sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyp cd "${package_dir}" -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="${librmm_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 6005878592..f92248ef0e 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -30,8 +30,6 @@ fi ############################################### # Build pylibraft -set -euo pipefail - # TODO: Generalize gha tool for getting conda artifacts to wheels. artifact_name=$(RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) From 078a284a555b086b1fd441f8afe3eee91dd9fb3f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 22:05:56 +0000 Subject: [PATCH 21/60] Fix rmm artifact name --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index e0e68f1c47..1ee571ab63 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -3,7 +3,7 @@ set -euo pipefail -artifact_name=$(RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index f92248ef0e..d29fc3d11d 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -31,7 +31,7 @@ fi # Build pylibraft # TODO: Generalize gha tool for getting conda artifacts to wheels. -artifact_name=$(RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") From d4fd16f196132b51ab9858e140d21a27109ba110 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 22:13:08 +0000 Subject: [PATCH 22/60] Bug --- ci/build_wheel_cpp.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 1ee571ab63..4045982ec3 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -3,10 +3,6 @@ set -euo pipefail -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") - package_name="libraft" package_dir="python/libraft" @@ -18,6 +14,10 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" From 02149789e0dbfde7486d354538759d09b3226728 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Apr 2024 23:20:00 +0000 Subject: [PATCH 23/60] Fix variable --- ci/build_wheel_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index d29fc3d11d..d45309cfa8 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -31,7 +31,7 @@ fi # Build pylibraft # TODO: Generalize gha tool for getting conda artifacts to wheels. -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") From 6908580e48b30afed1a2247626f5d6659316270e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 00:28:07 +0000 Subject: [PATCH 24/60] Fix find links --- ci/build_wheel_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index d45309cfa8..435243b7f4 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -53,7 +53,7 @@ fi pushd "${package_dir}" -PIP_FIND_LINKS="/tmp/libraft_dist;${librmm_wheelhouse}" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check +PIP_FIND_LINKS="/tmp/libraft_dist ${librmm_wheelhouse}" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check mkdir -p pylibraft_final_dist python -m auditwheel repair -w pylibraft_final_dist pylibraft_dist/* @@ -84,7 +84,7 @@ popd # #pushd "${package_dir}" # -#PIP_FIND_LINKS="../pylibraft/pylibraft_dist;/tmp/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check +#PIP_FIND_LINKS="../pylibraft/pylibraft_dist /tmp/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check # #mkdir -p raft_dask_final_dist #python -m auditwheel repair -w raft_dask_final_dist raft_dask_dist/* From fd02fc3a8b8fdda233a99e8e311e8607c49fe2e6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 01:34:47 +0000 Subject: [PATCH 25/60] Make wheel py3 --- python/libraft/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml index 9e867dadc8..c5a76e363a 100644 --- a/python/libraft/pyproject.toml +++ b/python/libraft/pyproject.toml @@ -57,6 +57,7 @@ sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["libraft"] wheel.install-dir = "libraft" +wheel.py-api = "py3" [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From b2f58a3314c848a72053543b54742ab37cddc690 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 05:05:06 +0000 Subject: [PATCH 26/60] Fix sed --- ci/build_wheel_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 435243b7f4..7511bf5705 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -45,7 +45,7 @@ sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFF sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/libraft==(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} From 1f73ba786a920e09fd80386c85112d66407b02f6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 14:03:14 +0000 Subject: [PATCH 27/60] Exclude libraft.so from auditwheel --- ci/build_wheel_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 7511bf5705..525cb04842 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -56,7 +56,7 @@ pushd "${package_dir}" PIP_FIND_LINKS="/tmp/libraft_dist ${librmm_wheelhouse}" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check mkdir -p pylibraft_final_dist -python -m auditwheel repair -w pylibraft_final_dist pylibraft_dist/* +python -m auditwheel repair -w pylibraft_final_dist --exclude libraft.so pylibraft_dist/* RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 pylibraft_final_dist @@ -87,7 +87,7 @@ popd #PIP_FIND_LINKS="../pylibraft/pylibraft_dist /tmp/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check # #mkdir -p raft_dask_final_dist -#python -m auditwheel repair -w raft_dask_final_dist raft_dask_dist/* +#python -m auditwheel repair -w raft_dask_final_dist --exclude libraft.so raft_dask_dist/* # #RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 raft_dask_final_dist # From e871cbcc42ed74e0e7df3f8c6cda831f5e763a95 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 15:15:32 +0000 Subject: [PATCH 28/60] Make libraft a runtime dependency of the wheel --- dependencies.yaml | 7 ++++++- python/pylibraft/pyproject.toml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index c49f287d6d..d510fea1c2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -462,10 +462,15 @@ dependencies: - matrix: {cuda: "12.*"} packages: - *rmm_cu12 + - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - *rmm_cu11 - - {matrix: null, packages: [*rmm_conda]} + - libraft-cu12==24.6.* + - matrix: null + packages: + - *rmm_conda + - libraft==24.6.* run_raft_dask: common: - output_types: [conda, pyproject] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index a8781e61b8..bf98d9b9c8 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -37,6 +37,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", + "libraft==24.6.*", "numpy>=1.23,<2.0a0", "rmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 4b882551ecdf14149686a7b2c53b509446145385 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 16:46:41 +0000 Subject: [PATCH 29/60] Also download libraft wheel --- ci/test_wheel_pylibraft.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index b38f5a690b..632b640a27 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -5,9 +5,10 @@ set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/pylibraft*.whl)[test] +python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist python -m pytest ./python/pylibraft/pylibraft/test From d15c098c2547a39bfc43280ea5d248dae6cf988d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 17:49:27 +0000 Subject: [PATCH 30/60] Also add the rmm wheelhouse --- ci/test_wheel_pylibraft.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 632b640a27..a24266a0a1 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -8,7 +8,11 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) +commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) +librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") + # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist +python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} python -m pytest ./python/pylibraft/pylibraft/test From 2cccc763038101752153e3bdd5a76c6d8a6f0bfe Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 19:30:33 +0000 Subject: [PATCH 31/60] Specify py version for libraft download --- ci/test_wheel_pylibraft.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index a24266a0a1..94e9ce91f5 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -5,7 +5,7 @@ set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" PYTHON_VERSION="3.11" rapids-download-wheels-from-s3 ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) From fcbf48be442dd7f480dd0ee7af7ab13651f69e00 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 21:13:33 +0000 Subject: [PATCH 32/60] Specify version correctly... --- ci/test_wheel_pylibraft.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 94e9ce91f5..8d5da9ccfd 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -5,7 +5,7 @@ set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" PYTHON_VERSION="3.11" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) From ace582d05e331a13b9ad0235254636b91fe56db5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 23:11:50 +0000 Subject: [PATCH 33/60] Update to use latest tools --- ci/build_wheel_cpp.sh | 11 +++++++---- ci/build_wheel_python.sh | 14 ++++++++------ ci/test_wheel_pylibraft.sh | 13 ++++++++----- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 4045982ec3..6e0457f35e 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -6,6 +6,11 @@ set -euo pipefail package_name="libraft" package_dir="python/libraft" +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools + export PATH="/tmp/gha-tools/tools:${PATH}" +fi + source rapids-configure-sccache source rapids-date-string @@ -14,9 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm PYTHON_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") +librmm_wheelhouse=$(get-pr-wheel-artifact.sh rmm 1512 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -48,4 +51,4 @@ PIP_FIND_LINKS="${librmm_wheelhouse}" python -m pip wheel . -w dist -vvv --no-de mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 525cb04842..1654c5b9c3 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -3,6 +3,11 @@ set -euo pipefail +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools + export PATH="/tmp/gha-tools/tools:${PATH}" +fi + source rapids-configure-sccache source rapids-date-string @@ -10,7 +15,7 @@ version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 /tmp/libraft_dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -30,10 +35,7 @@ fi ############################################### # Build pylibraft -# TODO: Generalize gha tool for getting conda artifacts to wheels. -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") +librmm_wheelhouse=$(get-pr-wheel-artifact.sh rmm 1512 cpp) package_name="pylibraft" package_dir="python/pylibraft" @@ -58,7 +60,7 @@ PIP_FIND_LINKS="/tmp/libraft_dist ${librmm_wheelhouse}" python -m pip wheel . -w mkdir -p pylibraft_final_dist python -m auditwheel repair -w pylibraft_final_dist --exclude libraft.so pylibraft_dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 pylibraft_final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python pylibraft_final_dist popd diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 8d5da9ccfd..e9520673a1 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -3,14 +3,17 @@ set -euo pipefail +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools + export PATH="/tmp/gha-tools/tools:${PATH}" +fi + mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_VERSION="3.11" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -artifact_name=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_REPOSITORY=rmm RAPIDS_PY_VERSION="3.11" rapids-package-name wheel_python) -commit=$(git ls-remote https://github.com/rapidsai/rmm.git refs/heads/pull-request/1512 | cut -c1-7) -librmm_wheelhouse=$(rapids-get-artifact "ci/rmm/pull-request/1512/${commit}/${artifact_name}") +librmm_wheelhouse=$(get-pr-wheel-artifact.sh rmm 1512 cpp) # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} From 3fb9ce7be26fec764447e845ad2377991ec55300 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 23:26:48 +0000 Subject: [PATCH 34/60] typo --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel_pylibraft.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 6e0457f35e..50e25f8c2a 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,7 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -librmm_wheelhouse=$(get-pr-wheel-artifact.sh rmm 1512 cpp) +librmm_wheelhouse=$(rapids-get-pr-wheel-artifact.sh rmm 1512 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 1654c5b9c3..6ab25630b4 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -35,7 +35,7 @@ fi ############################################### # Build pylibraft -librmm_wheelhouse=$(get-pr-wheel-artifact.sh rmm 1512 cpp) +librmm_wheelhouse=$(rapids-get-pr-wheel-artifact.sh rmm 1512 cpp) package_name="pylibraft" package_dir="python/pylibraft" diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index e9520673a1..8ce8158c28 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -librmm_wheelhouse=$(get-pr-wheel-artifact.sh rmm 1512 cpp) +librmm_wheelhouse=$(rapids-get-pr-wheel-artifact.sh rmm 1512 cpp) # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} From 3e77844c7b5f448be34c1bb0bb59a199318c443d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 23:32:58 +0000 Subject: [PATCH 35/60] Get rid of extension --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel_pylibraft.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 50e25f8c2a..e6916e8852 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,7 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -librmm_wheelhouse=$(rapids-get-pr-wheel-artifact.sh rmm 1512 cpp) +librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 6ab25630b4..215413d6b3 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -35,7 +35,7 @@ fi ############################################### # Build pylibraft -librmm_wheelhouse=$(rapids-get-pr-wheel-artifact.sh rmm 1512 cpp) +librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) package_name="pylibraft" package_dir="python/pylibraft" diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 8ce8158c28..7780abc15f 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -librmm_wheelhouse=$(rapids-get-pr-wheel-artifact.sh rmm 1512 cpp) +librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} From 3ab6d6a9a48492cc622993018d93aa34d8530ce4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 23:49:01 +0000 Subject: [PATCH 36/60] Specify the wheel name to get --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel_pylibraft.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index e6916e8852..849ff8558e 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,7 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +RAPIDS_PY_WHEEL_NAME="librmm${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 215413d6b3..e3a562a873 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -35,7 +35,7 @@ fi ############################################### # Build pylibraft -librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +RAPIDS_PY_WHEEL_NAME="librmm${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) package_name="pylibraft" package_dir="python/pylibraft" diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 7780abc15f..2b20fbe253 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +RAPIDS_PY_WHEEL_NAME="librmm${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} From 63a657dbc9ee0335caebb6a8973aadb90405287d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 2 Apr 2024 23:50:21 +0000 Subject: [PATCH 37/60] Add missing underscore --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel_pylibraft.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 849ff8558e..0ad5464f2d 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,7 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="librmm${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index e3a562a873..4246043886 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -35,7 +35,7 @@ fi ############################################### # Build pylibraft -RAPIDS_PY_WHEEL_NAME="librmm${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) package_name="pylibraft" package_dir="python/pylibraft" diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 2b20fbe253..a835f921df 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -RAPIDS_PY_WHEEL_NAME="librmm${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} From 0aa6711d955bd3a00153d7bf52fa0f4c72777c8b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 Apr 2024 00:17:45 +0000 Subject: [PATCH 38/60] Specify var in command --- ci/build_wheel_cpp.sh | 4 ++-- ci/build_wheel_python.sh | 2 +- ci/test_wheel_pylibraft.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 0ad5464f2d..c4e0b2ecf5 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -8,8 +8,8 @@ package_dir="python/libraft" if [[ ! -d "/tmp/gha-tools" ]]; then git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools - export PATH="/tmp/gha-tools/tools:${PATH}" fi +export PATH="/tmp/gha-tools/tools:${PATH}" source rapids-configure-sccache source rapids-date-string @@ -19,7 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 4246043886..fe5419f603 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -35,7 +35,7 @@ fi ############################################### # Build pylibraft -RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) package_name="pylibraft" package_dir="python/pylibraft" diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index a835f921df..20d21ac94f 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" librmm_wheelhouse=$(rapids-get-pr-wheel-artifact rmm 1512 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} From 006c3e542c2b3cc9e0fc63519ade65d17a4b83ab Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 Apr 2024 00:26:15 +0000 Subject: [PATCH 39/60] Move all exports out of conditional --- ci/build_wheel_python.sh | 2 +- ci/test_wheel_pylibraft.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index fe5419f603..00d64d401f 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -5,8 +5,8 @@ set -euo pipefail if [[ ! -d "/tmp/gha-tools" ]]; then git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools - export PATH="/tmp/gha-tools/tools:${PATH}" fi +export PATH="/tmp/gha-tools/tools:${PATH}" source rapids-configure-sccache source rapids-date-string diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 20d21ac94f..39f20bf525 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -5,8 +5,8 @@ set -euo pipefail if [[ ! -d "/tmp/gha-tools" ]]; then git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools - export PATH="/tmp/gha-tools/tools:${PATH}" fi +export PATH="/tmp/gha-tools/tools:${PATH}" mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" From a53fe64c6d25e129aaa6a12feecd789b583bf2b7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 11 Apr 2024 23:35:58 +0000 Subject: [PATCH 40/60] Updates for raft-dask --- ci/build_wheel_python.sh | 56 +++++++++++++------------- dependencies.yaml | 10 ++++- python/raft-dask/CMakeLists.txt | 29 +------------ python/raft-dask/pyproject.toml | 1 + python/raft-dask/raft_dask/__init__.py | 12 +++++- 5 files changed, 49 insertions(+), 59 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 00d64d401f..1e1d8858a3 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -66,31 +66,31 @@ popd ################################################ -## Build raft-dask -# -#package_name=raft-dask -#package_dir=python/raft-dask -#underscore_package_name=$(echo "${package_name}" | tr "-" "_") -# -#pyproject_file="${package_dir}/pyproject.toml" -#version_file="${package_dir}/${underscore_package_name}/_version.py" -# -#sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -#sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} -# -#sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -#sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -#sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -#sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} -#sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} -# -#pushd "${package_dir}" -# -#PIP_FIND_LINKS="../pylibraft/pylibraft_dist /tmp/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check -# -#mkdir -p raft_dask_final_dist -#python -m auditwheel repair -w raft_dask_final_dist --exclude libraft.so raft_dask_dist/* -# -#RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 raft_dask_final_dist -# -#popd +# Build raft-dask + +package_name="raft-dask" +package_dir="python/raft-dask" +underscore_package_name=$(echo "${package_name}" | tr "-" "_") + +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${underscore_package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} +sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} + +pushd "${package_dir}" + +PIP_FIND_LINKS="../pylibraft/pylibraft_dist /tmp/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check + +mkdir -p raft_dask_final_dist +python -m auditwheel repair -w raft_dask_final_dist --exclude libraft.so raft_dask_dist/* + +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 raft_dask_final_dist + +popd diff --git a/dependencies.yaml b/dependencies.yaml index d510fea1c2..63b9154a77 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -466,7 +466,7 @@ dependencies: - matrix: {cuda: "11.*"} packages: - *rmm_cu11 - - libraft-cu12==24.6.* + - libraft-cu11==24.6.* - matrix: null packages: - *rmm_conda @@ -502,11 +502,17 @@ dependencies: packages: - &pylibraft_cu12 pylibraft-cu12==24.6.* - &ucx_py_cu12 ucx-py-cu12==0.38.* + - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - &pylibraft_cu11 pylibraft-cu11==24.6.* - &ucx_py_cu11 ucx-py-cu11==0.38.* - - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} + - libraft-cu11==24.6.* + - matrix: null + packages: + - *pylibraft_conda + - *ucx_py_conda + - libraft==24.6.* test_python_common: common: - output_types: [conda, requirements, pyproject] diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 58e5ae8104..fbb07bfc4b 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -24,34 +24,7 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - OFF -) - -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) -else() - set(raft_FOUND OFF) -endif() - -if(NOT raft_FOUND) - find_package(ucx REQUIRED) - - # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all - # library compilation and we don't need to install anything here. - set(BUILD_TESTS OFF) - set(BUILD_ANN_BENCH OFF) - set(BUILD_PRIMS_BENCH OFF) - set(RAFT_COMPILE_LIBRARIES OFF) - set(RAFT_COMPILE_DIST_LIBRARY OFF) - set(RAFT_COMPILE_NN_LIBRARY OFF) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) - find_package(NCCL REQUIRED) -endif() +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 7065cccd73..d846580529 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -36,6 +36,7 @@ requires-python = ">=3.9" dependencies = [ "dask-cuda==24.6.*", "joblib>=0.11", + "libraft==24.6.*", "numba>=0.57", "numpy>=1.23,<2.0a0", "pylibraft==24.6.*", diff --git a/python/raft-dask/raft_dask/__init__.py b/python/raft-dask/raft_dask/__init__.py index fbbaee4118..8e4daaf88c 100644 --- a/python/raft-dask/raft_dask/__init__.py +++ b/python/raft-dask/raft_dask/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,3 +14,13 @@ # from raft_dask._version import __git_commit__, __version__ + +# If libraft was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft From 643f700adb191295cf105bc0b7acd0776cdf063d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 11 Apr 2024 23:39:39 +0000 Subject: [PATCH 41/60] More updates --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- ci/test_wheel_pylibraft.sh | 2 +- ci/test_wheel_raft_dask.sh | 15 ++++++++++----- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index c4e0b2ecf5..986a0977bc 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -19,7 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 1e1d8858a3..3b1f417b18 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -35,7 +35,7 @@ fi ############################################### # Build pylibraft -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) package_name="pylibraft" package_dir="python/pylibraft" diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 39f20bf525..1f92c70b66 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1512 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 76bb62e859..d5f3bb5358 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -3,15 +3,20 @@ set -euo pipefail +if [[ ! -d "/tmp/gha-tools" ]]; then + git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools +fi +export PATH="/tmp/gha-tools/tools:${PATH}" + mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -# Download the pylibraft built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep -python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/raft_dask*.whl)[test] +python -m pip install $(echo ./dist/raft_dask*.whl)[test] --find-links dist/ --find-links ${librmm_wheelhouse} python -m pytest ./python/raft-dask/raft_dask/test From c36b69c84ecc78c6bf271c834cae45d8b773aa50 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 12 Apr 2024 00:47:05 +0000 Subject: [PATCH 42/60] Fix dependency links --- ci/build_wheel_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 3b1f417b18..ec4399e3ae 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -55,7 +55,7 @@ fi pushd "${package_dir}" -PIP_FIND_LINKS="/tmp/libraft_dist ${librmm_wheelhouse}" python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check +python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check --find-links /tmp/libraft_dist --find-links ${librmm_wheelhouse} mkdir -p pylibraft_final_dist python -m auditwheel repair -w pylibraft_final_dist --exclude libraft.so pylibraft_dist/* @@ -86,7 +86,7 @@ sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} pushd "${package_dir}" -PIP_FIND_LINKS="../pylibraft/pylibraft_dist /tmp/libraft_dist" python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check +python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check --find-links /tmp/libraft_dist --find-links ${librmm_wheelhouse} --find-links ../pylibraft/pylibraft_dist mkdir -p raft_dask_final_dist python -m auditwheel repair -w raft_dask_final_dist --exclude libraft.so raft_dask_dist/* From a3fa4fd3f0127731515d21408ce7a652f1e318bb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 01:07:33 +0000 Subject: [PATCH 43/60] Remove unnecessary clones --- ci/build_wheel_cpp.sh | 5 ----- ci/build_wheel_python.sh | 5 ----- 2 files changed, 10 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 986a0977bc..17bad4f162 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -6,11 +6,6 @@ set -euo pipefail package_name="libraft" package_dir="python/libraft" -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - source rapids-configure-sccache source rapids-date-string diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index ec4399e3ae..fc184938e5 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -3,11 +3,6 @@ set -euo pipefail -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - source rapids-configure-sccache source rapids-date-string From 110c26fa8f8f5e3f50f2778a07e4b2231caa7591 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 01:20:51 +0000 Subject: [PATCH 44/60] Simplify wheel building script --- ci/build_wheel_python.sh | 93 ++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 52 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index fc184938e5..06b6a5d184 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -12,11 +12,16 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) + +wheelhouses=("/tmp/libraft_dist" "${librmm_wheelhouse}") + # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" echo "${version}" > VERSION + # For nightlies we want to ensure that we're pulling in alphas as well. The # easiest way to do so is to augment the spec with a constraint containing a # min alpha version that doesn't affect the version bounds but does allow usage @@ -26,66 +31,50 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi +build_wheel () { + local package_name="${1}" + local version_package_name=$(echo "${package_name}" | tr "-" "_") -############################################### -# Build pylibraft - -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - -package_name="pylibraft" -package_dir="python/pylibraft" - -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/libraft==(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi - -pushd "${package_dir}" - -python -m pip wheel . -w pylibraft_dist -vvv --no-deps --disable-pip-version-check --find-links /tmp/libraft_dist --find-links ${librmm_wheelhouse} - -mkdir -p pylibraft_final_dist -python -m auditwheel repair -w pylibraft_final_dist --exclude libraft.so pylibraft_dist/* - -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python pylibraft_final_dist - -popd - + local package_dir="python/${package_name}" + local pyproject_file="${package_dir}/pyproject.toml" + local version_file="${package_dir}/${version_package_name}/_version.py" -################################################ -# Build raft-dask + sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} -package_name="raft-dask" -package_dir="python/raft-dask" -underscore_package_name=$(echo "${package_name}" | tr "-" "_") + for dep in rmm libraft pylibraft ucx-py; do + sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + done -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${underscore_package_name}/_version.py" + # dask-cuda & rapids-dask-dependency don't get a suffix, but they do get an alpha spec. + for dep in dask-cuda rapids-dask-dependency; do + sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} + done -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} + fi -sed -r -i "s/libraft(.*)\"/libraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} + pushd "${package_dir}" -pushd "${package_dir}" + local find_links="" + # Iterate over the array + for wheelhouse in "${wheelhouses[@]}"; do + find_links+="--find-links ${wheelhouse} " + done + + python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check ${find_links} -python -m pip wheel . -w raft_dask_dist -vvv --no-deps --disable-pip-version-check --find-links /tmp/libraft_dist --find-links ${librmm_wheelhouse} --find-links ../pylibraft/pylibraft_dist + mkdir -p final_dist + python -m auditwheel repair -w final_dist --exclude libraft.so dist/* -mkdir -p raft_dask_final_dist -python -m auditwheel repair -w raft_dask_final_dist --exclude libraft.so raft_dask_dist/* + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 raft_dask_final_dist + # Append ${PWD}/final_dist to the list of wheelhouses for the next package + wheelhouses+=("${PWD}/final_dist") + popd +} -popd +build_wheel pylibraft +build_wheel raft-dask From a358b52bd894b51f233f28724190e579aa643198 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 01:35:16 +0000 Subject: [PATCH 45/60] Remove more clones --- ci/test_wheel_pylibraft.sh | 6 ------ ci/test_wheel_raft_dask.sh | 6 ------ 2 files changed, 12 deletions(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 1f92c70b66..a8315d4c38 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -3,12 +3,6 @@ set -euo pipefail -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index d5f3bb5358..0288402967 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -3,12 +3,6 @@ set -euo pipefail -if [[ ! -d "/tmp/gha-tools" ]]; then - git clone https://github.com/msarahan/gha-tools.git -b get-pr-wheel-artifact /tmp/gha-tools -fi -export PATH="/tmp/gha-tools/tools:${PATH}" - -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist From 888ed93878d2de5e467be6a6fba57a701cae8826 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 15:19:40 +0000 Subject: [PATCH 46/60] More pip conda alignment --- ci/build_wheel_cpp.sh | 10 +++++----- ci/build_wheel_python.sh | 41 ++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 17bad4f162..951398dd6d 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -14,9 +14,6 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - -# This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" @@ -41,9 +38,12 @@ sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyp cd "${package_dir}" -PIP_FIND_LINKS="${librmm_wheelhouse}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${librmm_wheelhouse}" mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist +# TODO: Remove RAPIDS_PY_WHEEL_NAME once gha-tools includes the cuda version in the artifact name. +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 06b6a5d184..9cee4e53f0 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -10,11 +10,6 @@ version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist - -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - -wheelhouses=("/tmp/libraft_dist" "${librmm_wheelhouse}") # This is the version of the suffix with a preceding hyphen. It's used # everywhere except in the final wheel name. @@ -31,6 +26,22 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi +# TODO: Remove RAPIDS_PY_WHEEL_NAME once gha-tools includes the cuda version in the artifact name. +# TODO: Remove the final argument once rapids-download-wheels-from-s3 is updated to construct the directory. +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +PYTHON_WHEELHOUSE="${PWD}/dist/" +PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" +WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${LIBRAFT_WHEELHOUSE}" "${librmm_wheelhouse}") +mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" + +FIND_LINKS="" +# Iterate over the array +for wheelhouse in "${WHEELHOUSES[@]}"; do + FIND_LINKS+="--find-links ${wheelhouse} " +done + + build_wheel () { local package_name="${1}" local version_package_name=$(echo "${package_name}" | tr "-" "_") @@ -58,23 +69,13 @@ build_wheel () { pushd "${package_dir}" - local find_links="" - # Iterate over the array - for wheelhouse in "${wheelhouses[@]}"; do - find_links+="--find-links ${wheelhouse} " - done - - python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check ${find_links} - - mkdir -p final_dist - python -m auditwheel repair -w final_dist --exclude libraft.so dist/* - - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist - - # Append ${PWD}/final_dist to the list of wheelhouses for the next package - wheelhouses+=("${PWD}/final_dist") + python -m pip wheel . -w "${PYTHON_WHEELHOUSE}" -vvv --no-deps --disable-pip-version-check ${FIND_LINKS} + python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so "${PYTHON_WHEELHOUSE}"/* popd } build_wheel pylibraft build_wheel raft-dask + +# TODO: Remove RAPIDS_PY_WHEEL_NAME once gha-tools includes the cuda version in the artifact name. +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" From e2e0902fd3d3c8592decec88cda829a9bae386e0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 15:22:02 +0000 Subject: [PATCH 47/60] Standardize test scripts --- ci/test_wheel_pylibraft.sh | 10 +++++----- ci/test_wheel_raft_dask.sh | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index a8315d4c38..0f29658b16 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -4,12 +4,12 @@ set -euo pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/pylibraft*.whl)[test] --find-links ./dist --find-links ${librmm_wheelhouse} +python -m pip install "pylibraft[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} python -m pytest ./python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 0288402967..a37b776fef 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -4,13 +4,12 @@ set -euo pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist -RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/raft_dask*.whl)[test] --find-links dist/ --find-links ${librmm_wheelhouse} +python -m pip install "raft_dask[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} python -m pytest ./python/raft-dask/raft_dask/test From 9bb484bfc4225f2199708e477c4384c3367596d7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 15:26:44 +0000 Subject: [PATCH 48/60] Centralize testing logic --- ci/build_wheel_python.sh | 4 ++-- ci/test_wheel.sh | 19 +++++++++++++++++++ ci/test_wheel_pylibraft.sh | 11 +---------- ci/test_wheel_raft_dask.sh | 11 +---------- 4 files changed, 23 insertions(+), 22 deletions(-) create mode 100755 ci/test_wheel.sh diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 9cee4e53f0..5db7896977 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -44,11 +44,11 @@ done build_wheel () { local package_name="${1}" - local version_package_name=$(echo "${package_name}" | tr "-" "_") + local underscore_package_name=$(echo "${package_name}" | tr "-" "_") local package_dir="python/${package_name}" local pyproject_file="${package_dir}/pyproject.toml" - local version_file="${package_dir}/${version_package_name}/_version.py" + local version_file="${package_dir}/${underscore_package_name}/_version.py" sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100755 index 0000000000..e10fe2a276 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="${1}" +package_dir="python/${package_name}" +underscore_package_name=$(echo "${package_name}" | tr "-" "_") + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" + +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) + +python -m pip install "${package_name}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} + +python -m pytest ${package_dir}/${underscore_package_name}/test diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index 0f29658b16..720c690f7c 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -3,13 +3,4 @@ set -euo pipefail -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -WHEELHOUSE="${PWD}/dist/" -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" - -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - -python -m pip install "pylibraft[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} - -python -m pytest ./python/pylibraft/pylibraft/test +./ci/test_wheel.sh pylibraft diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index a37b776fef..b7a516ec18 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -3,13 +3,4 @@ set -euo pipefail -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -WHEELHOUSE="${PWD}/dist/" -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" - -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - -python -m pip install "raft_dask[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} - -python -m pytest ./python/raft-dask/raft_dask/test +./ci/test_wheel.sh raft-dask From d94ce97056e7609bf3766485eb9000ad3bf7a19c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 15:44:15 +0000 Subject: [PATCH 49/60] Fix rmm name --- ci/build_wheel_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 951398dd6d..40f3871d5a 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -38,7 +38,7 @@ sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyp cd "${package_dir}" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${librmm_wheelhouse}" From 0435e6b7216d81b5d53482f0f8477f784e1b0acb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 15:59:08 +0000 Subject: [PATCH 50/60] Add suffix to install command --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e10fe2a276..ac6a69e0e8 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -14,6 +14,6 @@ RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 p librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) -python -m pip install "${package_name}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} +python -m pip install "${package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} python -m pytest ${package_dir}/${underscore_package_name}/test From efafdb6eb5915d81f39e6d14fbc46b687cdb63ec Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 17 Apr 2024 17:24:37 +0000 Subject: [PATCH 51/60] Only audit once and fix varname --- ci/build_wheel_python.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 5db7896977..270fc05fb8 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -32,7 +32,7 @@ CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) PYTHON_WHEELHOUSE="${PWD}/dist/" PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" -WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${LIBRAFT_WHEELHOUSE}" "${librmm_wheelhouse}") +WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${CPP_WHEELHOUSE}" "${librmm_wheelhouse}") mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" FIND_LINKS="" @@ -70,12 +70,11 @@ build_wheel () { pushd "${package_dir}" python -m pip wheel . -w "${PYTHON_WHEELHOUSE}" -vvv --no-deps --disable-pip-version-check ${FIND_LINKS} - python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so "${PYTHON_WHEELHOUSE}"/* popd } build_wheel pylibraft build_wheel raft-dask -# TODO: Remove RAPIDS_PY_WHEEL_NAME once gha-tools includes the cuda version in the artifact name. +python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so "${PYTHON_WHEELHOUSE}"/* RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" From 0f509321f86ac9946c141d6650021f7a346d2c28 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 18 Apr 2024 04:19:47 +0000 Subject: [PATCH 52/60] Make VERSION a symlink --- python/libraft/libraft/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 120000 python/libraft/libraft/VERSION diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION deleted file mode 100644 index 0bff6981a3..0000000000 --- a/python/libraft/libraft/VERSION +++ /dev/null @@ -1 +0,0 @@ -24.06.00 diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libraft/libraft/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file From de8419302d6793922c664ca3894416d1ddafefc7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 May 2024 18:53:00 +0000 Subject: [PATCH 53/60] Stop pulling rmm from PR and add package name back to artifacts --- ci/build_wheel_cpp.sh | 7 ++----- ci/build_wheel_python.sh | 9 +++------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 40f3871d5a..0e0417dc35 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -38,12 +38,9 @@ sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyp cd "${package_dir}" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${librmm_wheelhouse}" +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* -# TODO: Remove RAPIDS_PY_WHEEL_NAME once gha-tools includes the cuda version in the artifact name. -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 270fc05fb8..46250c161f 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -26,13 +26,10 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi -# TODO: Remove RAPIDS_PY_WHEEL_NAME once gha-tools includes the cuda version in the artifact name. -# TODO: Remove the final argument once rapids-download-wheels-from-s3 is updated to construct the directory. -CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist) -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist) PYTHON_WHEELHOUSE="${PWD}/dist/" PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" -WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${CPP_WHEELHOUSE}" "${librmm_wheelhouse}") +WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${CPP_WHEELHOUSE}") mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" FIND_LINKS="" @@ -77,4 +74,4 @@ build_wheel pylibraft build_wheel raft-dask python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so "${PYTHON_WHEELHOUSE}"/* -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" From f3336ba8f70974c617aff0c70c5fab146c23bc9f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 May 2024 19:52:36 +0000 Subject: [PATCH 54/60] Update build.yaml --- .github/workflows/build.yaml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c8837afba7..d99d5d28e5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,37 +67,40 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} - wheel-build-pylibraft: + wheel-build-cpp: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_pylibraft.sh - wheel-publish-pylibraft: - needs: wheel-build-pylibraft + script: ci/build_wheel_cpp.sh + wheel-build-python: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: pylibraft - wheel-build-raft-dask: - needs: wheel-publish-pylibraft + script: ci/build_wheel_python.sh + wheel-publish-cpp: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_raft_dask.sh - wheel-publish-raft-dask: - needs: wheel-build-raft-dask + package-name: raft + package-type: cpp + wheel-publish-python: + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: @@ -105,4 +108,5 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: raft_dask + package-name: raft + package-type: python From bcc71070140662854a56b900d3e3b7cc87a8f08a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 May 2024 19:54:28 +0000 Subject: [PATCH 55/60] Minor CMake cleanup --- cpp/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9e45313803..5012a0423f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -584,12 +584,10 @@ if(RAFT_COMPILE_LIBRARY) ) add_library(raft_lib SHARED $) - if(NOT RAFT_COMPILE_DYNAMIC_ONLY) - add_library(raft_lib_static STATIC $) - endif() set(raft_lib_targets raft_lib) if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + add_library(raft_lib_static STATIC $) list(APPEND raft_lib_targets raft_lib_static) endif() From 674e9ef2b8002e935f94745a57dc73538b7f6bc9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 May 2024 21:19:22 +0000 Subject: [PATCH 56/60] Update dependencies.yaml --- dependencies.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 63b9154a77..9fa728fbad 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -207,8 +207,7 @@ dependencies: - libraft==24.6.* librmm: common: - #- output_types: [requirements, pyproject] - - output_types: [pyproject] + - output_types: [requirements, pyproject] packages: - librmm==24.6.* build_pylibraft: From d012e761ff6377a2fe95bdf28f29098b48b09af9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 May 2024 21:25:38 +0000 Subject: [PATCH 57/60] Update CMake and loading for devcontainer use case --- python/libraft/CMakeLists.txt | 10 ++++++++++ python/libraft/libraft/load.py | 23 +++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt index df48e2a4f6..cc3b48d033 100644 --- a/python/libraft/CMakeLists.txt +++ b/python/libraft/CMakeLists.txt @@ -25,6 +25,16 @@ project( LANGUAGES CXX CUDA ) +# Check if raft is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python raft package. +find_package(raft "${RAPIDS_VERSION}") + +if(raft_FOUND) + return() +endif() + +unset(raft_FOUND) + set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) set(BUILD_ANN_BENCH OFF) diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py index 317a68e9e4..fb7bf35274 100644 --- a/python/libraft/libraft/load.py +++ b/python/libraft/libraft/load.py @@ -21,15 +21,26 @@ def load_library(): # Dynamically load libraft.so. Prefer a system library if one is present to # avoid clobbering symbols that other packages might expect, but if no # other library is present use the one in the wheel. + libraft_lib = None try: libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) except OSError: - libraft_lib = ctypes.CDLL( - # TODO: Do we always know it will be lib64? Should we consider - # finding a way for CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib64", "libraft.so"), - ctypes.RTLD_GLOBAL, - ) + # If neither of these directories contain the library, we assume we are + # in an environment where the C++ library is already installed + # somewhere else and the CMake build of the libraft Python + # package was a no-op. Note that this approach won't work for + # real editable installs of the libraft package, but that's not a use + # case I think we need to support. scikit-build-core has limited + # support for importlib.resources so there isn't a clean way to support + # that case yet. + for lib_dir in ("lib", "lib64"): + if os.path.isfile( + lib := os.path.join( + os.path.dirname(__file__), lib_dir, "libraft.so" + ) + ): + libraft_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) + break # The caller almost never needs to do anything with this library, but no # harm in offering the option since this object at least provides a handle From 23ad58166b988962e2547c951666e07a7b435591 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 May 2024 21:43:50 +0000 Subject: [PATCH 58/60] Update build.sh --- build.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/build.sh b/build.sh index da5efa5183..ee16bb4b1b 100755 --- a/build.sh +++ b/build.sh @@ -381,14 +381,6 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi -# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" -fi -# Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS -SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') - # If clean given, run it prior to any other steps if (( ${CLEAN} == 1 )); then # If the dirs to clean are mounted dirs in a container, the @@ -495,13 +487,13 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then - SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ + CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then - SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ + CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/raft-dask fi From 212fa0d5657106acff29984d61ccc27e4c9abe43 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 May 2024 22:14:46 +0000 Subject: [PATCH 59/60] Fix test script --- ci/test_wheel.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index ac6a69e0e8..601783eea7 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -9,11 +9,9 @@ underscore_package_name=$(echo "${package_name}" | tr "-" "_") RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" WHEELHOUSE="${PWD}/dist/" -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" -RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - -python -m pip install "${package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" --find-links ${librmm_wheelhouse} +python -m pip install "${package_name}-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" python -m pytest ${package_dir}/${underscore_package_name}/test From 6b3ab9f0632bab3961d4b8fc5fd0ba57d96300c4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 May 2024 23:39:18 +0000 Subject: [PATCH 60/60] Add matrixed dependencies --- dependencies.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index a603ca9d33..685d0a8fac 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -213,9 +213,23 @@ dependencies: - libraft==24.6.* librmm: common: - - output_types: [requirements, pyproject] + - output_types: requirements packages: - - librmm==24.6.* + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - librmm-cu12==24.6.* + - matrix: {cuda: "11.*"} + packages: + - librmm-cu11==24.6.* + - matrix: null + packages: + - librmm==24.6.* build_pylibraft: common: - output_types: [conda]