diff --git a/libcudacxx/include/cuda/__memory_resource/cuda_pinned_memory_resource.h b/libcudacxx/include/cuda/__memory_resource/cuda_pinned_memory_resource.h new file mode 100644 index 00000000000..a1045e2f1cc --- /dev/null +++ b/libcudacxx/include/cuda/__memory_resource/cuda_pinned_memory_resource.h @@ -0,0 +1,191 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA__MEMORY_RESOURCE_CUDA_PINNED_MEMORY_RESOURCE_H +#define _CUDA__MEMORY_RESOURCE_CUDA_PINNED_MEMORY_RESOURCE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if !defined(_CCCL_COMPILER_MSVC_2017) + +# if !defined(_CCCL_CUDA_COMPILER_NVCC) && !defined(_CCCL_CUDA_COMPILER_NVHPC) +# include +# endif // !_CCCL_CUDA_COMPILER_NVCC && !_CCCL_CUDA_COMPILER_NVHPC + +# include +# include +# include +# include +# include +# include + +# if _CCCL_STD_VER >= 2014 + +_LIBCUDACXX_BEGIN_NAMESPACE_CUDA_MR + +/** + * @brief `cuda_pinned_memory_resource` uses cudaMallocHost / cudaFreeHost for allocation/deallocation. + */ +class cuda_pinned_memory_resource +{ +private: + unsigned int __flags_ = cudaHostAllocDefault; + + static constexpr unsigned int __available_flags = + cudaHostAllocDefault | cudaHostAllocPortable | cudaHostAllocMapped | cudaHostAllocWriteCombined; + +public: + constexpr cuda_pinned_memory_resource(const unsigned int __flags = cudaHostAllocDefault) noexcept + : __flags_(__flags & __available_flags) + { + _LIBCUDACXX_ASSERT(__flags_ == __flags, "Unexpected flags passed to cuda_pinned_memory_resource"); + } + + /** + * @brief Allocate host memory of size at least \p __bytes. + * @param __bytes The size in bytes of the allocation. + * @param __alignment The requested alignment of the allocation. + * @throw cuda::cuda_error if allocation fails with a CUDA error. + * @return Pointer to the newly allocated memory + */ + void* allocate(const size_t __bytes, const size_t __alignment = default_cuda_malloc_host_alignment) const + { + // We need to ensure that the provided alignment matches the minimal provided alignment + if (!__is_valid_alignment(__alignment)) + { + _CUDA_VSTD_NOVERSION::__throw_bad_alloc(); + } + + void* __ptr{nullptr}; + _CCCL_TRY_CUDA_API(::cudaMallocHost, "Failed to allocate memory with cudaMallocHost.", &__ptr, __bytes, __flags_); + return __ptr; + } + + /** + * @brief Deallocate memory pointed to by \p __ptr. + * @param __ptr Pointer to be deallocated. Must have been allocated through a call to `allocate` + * @param __bytes The number of bytes that was passed to the `allocate` call that returned \p __ptr. + * @param __alignment The alignment that was passed to the `allocate` call that returned \p __ptr. + */ + void deallocate(void* __ptr, const size_t, const size_t __alignment = default_cuda_malloc_host_alignment) const + { + // We need to ensure that the provided alignment matches the minimal provided alignment + _LIBCUDACXX_ASSERT(__is_valid_alignment(__alignment), + "Invalid alignment passed to cuda_memory_resource::deallocate."); + _CCCL_ASSERT_CUDA_API(::cudaFreeHost, "cuda_pinned_memory_resource::deallocate failed", __ptr); + (void) __alignment; + } + + /** + * @brief Equality comparison with another cuda_pinned_memory_resource + * @return Whether both cuda_pinned_memory_resource were constructed with the same flags + */ + _LIBCUDACXX_NODISCARD_ATTRIBUTE constexpr bool operator==(cuda_pinned_memory_resource const& __other) const noexcept + { + return __flags_ == __other.__flags_; + } +# if _CCCL_STD_VER <= 2017 + /** + * @brief Equality comparison with another cuda_pinned_memory_resource + * @return Whether both cuda_pinned_memory_resource were constructed with different flags + */ + _LIBCUDACXX_NODISCARD_ATTRIBUTE constexpr bool operator!=(cuda_pinned_memory_resource const& __other) const noexcept + { + return __flags_ != __other.__flags_; + } +# endif // _CCCL_STD_VER <= 2017 + + /** + * @brief Equality comparison between a cuda_memory_resource and another resource + * @param __lhs The cuda_memory_resource + * @param __rhs The resource to compare to + * @return If the underlying types are equality comparable, returns the result of equality comparison of both + * resources. Otherwise, returns false. + */ + template + _LIBCUDACXX_NODISCARD_FRIEND auto operator==(cuda_pinned_memory_resource const& __lhs, _Resource const& __rhs) noexcept + _LIBCUDACXX_TRAILING_REQUIRES(bool)(__different_resource) + { + return resource_ref<>{const_cast(__lhs)} + == resource_ref<>{const_cast<_Resource&>(__rhs)}; + } +# if _CCCL_STD_VER <= 2017 + /** + * @copydoc cuda_pinned_memory_resource::operator<_Resource>==(cuda_pinned_memory_resource const&, _Resource const&) + */ + template + _LIBCUDACXX_NODISCARD_FRIEND auto operator==(_Resource const& __rhs, cuda_pinned_memory_resource const& __lhs) noexcept + _LIBCUDACXX_TRAILING_REQUIRES(bool)(__different_resource) + { + return resource_ref<>{const_cast(__lhs)} + == resource_ref<>{const_cast<_Resource&>(__rhs)}; + } + /** + * @copydoc cuda_pinned_memory_resource::operator<_Resource>==(cuda_pinned_memory_resource const&, _Resource const&) + */ + template + _LIBCUDACXX_NODISCARD_FRIEND auto operator!=(cuda_pinned_memory_resource const& __lhs, _Resource const& __rhs) noexcept + _LIBCUDACXX_TRAILING_REQUIRES(bool)(__different_resource) + { + return resource_ref<>{const_cast(__lhs)} + != resource_ref<>{const_cast<_Resource&>(__rhs)}; + } + /** + * @copydoc cuda_pinned_memory_resource::operator<_Resource>==(cuda_pinned_memory_resource const&, _Resource const&) + */ + template + _LIBCUDACXX_NODISCARD_FRIEND auto operator!=(_Resource const& __rhs, cuda_pinned_memory_resource const& __lhs) noexcept + _LIBCUDACXX_TRAILING_REQUIRES(bool)(__different_resource) + { + return resource_ref<>{const_cast(__lhs)} + != resource_ref<>{const_cast<_Resource&>(__rhs)}; + } +# endif // _CCCL_STD_VER <= 2017 + + /** + * @brief Enables the `pinned_memory` property + */ + friend constexpr void get_property(cuda_pinned_memory_resource const&, pinned_memory) noexcept {} + /** + * @brief Enables the `device_accessible` property + */ + friend constexpr void get_property(cuda_pinned_memory_resource const&, device_accessible) noexcept {} + /** + * @brief Enables the `host_accessible` property + */ + friend constexpr void get_property(cuda_pinned_memory_resource const&, host_accessible) noexcept {} + + /** + * @brief Checks whether the passed in alignment is valid + */ + static constexpr bool __is_valid_alignment(const size_t __alignment) noexcept + { + return __alignment <= default_cuda_malloc_host_alignment && (default_cuda_malloc_host_alignment % __alignment == 0); + } +}; +static_assert(resource_with, ""); +static_assert(resource_with, ""); +static_assert(resource_with, ""); + +_LIBCUDACXX_END_NAMESPACE_CUDA_MR + +# endif // _CCCL_STD_VER >= 2014 + +#endif // !_CCCL_COMPILER_MSVC_2017 + +#endif //_CUDA__MEMORY_RESOURCE_CUDA_PINNED_MEMORY_RESOURCE_H diff --git a/libcudacxx/include/cuda/__memory_resource/properties.h b/libcudacxx/include/cuda/__memory_resource/properties.h index bca25a4499e..4e485aae364 100644 --- a/libcudacxx/include/cuda/__memory_resource/properties.h +++ b/libcudacxx/include/cuda/__memory_resource/properties.h @@ -21,6 +21,8 @@ # pragma system_header #endif // no system header +#include + #if !defined(_CCCL_COMPILER_MSVC_2017) # if _CCCL_STD_VER >= 2014 @@ -32,6 +34,11 @@ _LIBCUDACXX_BEGIN_NAMESPACE_CUDA_MR */ _LIBCUDACXX_INLINE_VAR constexpr size_t default_cuda_malloc_alignment = 256; +/** + * @brief The default alignment by a cudaMallocHost{...} call + */ +_LIBCUDACXX_INLINE_VAR constexpr size_t default_cuda_malloc_host_alignment = alignof(_CUDA_VSTD::max_align_t); + /** * @brief The \c device_accessible property signals that the allocated memory is device accessible */ @@ -50,6 +57,12 @@ struct host_accessible struct managed_memory {}; +/** + * @brief The \c pinned_memory property signals that the allocated memory is not pageable. + */ +struct pinned_memory +{}; + _LIBCUDACXX_END_NAMESPACE_CUDA_MR # endif // _CCCL_STD_VER >= 2014 diff --git a/libcudacxx/include/cuda/memory_resource b/libcudacxx/include/cuda/memory_resource index 4a08a76d9b9..a138995aa5f 100644 --- a/libcudacxx/include/cuda/memory_resource +++ b/libcudacxx/include/cuda/memory_resource @@ -94,6 +94,7 @@ class resource_ref { #include #include +#include #include #include #include diff --git a/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/allocate.pass.cpp b/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/allocate.pass.cpp new file mode 100644 index 00000000000..d82490725b3 --- /dev/null +++ b/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/allocate.pass.cpp @@ -0,0 +1,98 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11 +// UNSUPPORTED: msvc-19.16 +// UNSUPPORTED: nvrtc + +#include +#include +#include +#include + +#include "test_macros.h" + +void ensure_pinned_host_ptr(void* ptr) +{ + assert(ptr != nullptr); + cudaPointerAttributes attributes; + cudaError_t status = cudaPointerGetAttributes(&attributes, ptr); + assert(status == cudaSuccess); + assert((attributes.type == cudaMemoryTypeHost) && (attributes.devicePointer != nullptr)); +} + +void test(const unsigned int flag) +{ + cuda::mr::cuda_pinned_memory_resource res{flag}; + + { // allocate / deallocate + auto* ptr = res.allocate(42); + static_assert(cuda::std::is_same::value, ""); + ensure_pinned_host_ptr(ptr); + + res.deallocate(ptr, 42); + } + + { // allocate / deallocate with alignment + auto* ptr = res.allocate(42, 4); + static_assert(cuda::std::is_same::value, ""); + ensure_pinned_host_ptr(ptr); + + res.deallocate(ptr, 42, 4); + } + +#ifndef TEST_HAS_NO_EXCEPTIONS + { // allocate with too small alignment + while (true) + { + try + { + auto* ptr = res.allocate(5, 42); + unused(ptr); + } + catch (const cuda::std::bad_alloc&) + { + break; + } + assert(false); + } + } + + { // allocate with non matching alignment + while (true) + { + try + { + auto* ptr = res.allocate(5, 1337); + unused(ptr); + } + catch (const cuda::std::bad_alloc&) + { + break; + } + assert(false); + } + } +#endif // TEST_HAS_NO_EXCEPTIONS +} + +void test() +{ + test(cudaHostAllocDefault); + test(cudaHostAllocPortable); + test(cudaHostAllocMapped); + test(cudaHostAllocWriteCombined); +} + +int main(int, char**) +{ + NV_IF_TARGET(NV_IS_HOST, test();) + return 0; +} diff --git a/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/equality.pass.cpp b/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/equality.pass.cpp new file mode 100644 index 00000000000..a75b0925ecd --- /dev/null +++ b/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/equality.pass.cpp @@ -0,0 +1,134 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11 +// UNSUPPORTED: msvc-19.16 +// UNSUPPORTED: nvrtc + +#include +#include +#include +#include + +enum class AccessibilityType +{ + Device, + Host, +}; + +template +struct resource +{ + void* allocate(size_t, size_t) + { + return nullptr; + } + void deallocate(void*, size_t, size_t) {} + + bool operator==(const resource&) const + { + return true; + } + bool operator!=(const resource& other) const + { + return false; + } + + template = 0> + friend void get_property(const resource&, cuda::mr::pinned_memory) noexcept + {} +}; +static_assert(cuda::mr::resource>, ""); +static_assert(!cuda::mr::resource_with, cuda::mr::pinned_memory>, ""); +static_assert(cuda::mr::resource>, ""); +static_assert(cuda::mr::resource_with, cuda::mr::pinned_memory>, ""); + +template +struct async_resource : public resource +{ + void* allocate_async(size_t, size_t, cuda::stream_ref) + { + return nullptr; + } + void deallocate_async(void*, size_t, size_t, cuda::stream_ref) {} +}; +static_assert(cuda::mr::async_resource>, ""); +static_assert(!cuda::mr::async_resource_with, cuda::mr::pinned_memory>, ""); +static_assert(cuda::mr::async_resource>, ""); +static_assert(cuda::mr::async_resource_with, cuda::mr::pinned_memory>, ""); + +void test() +{ + cuda::mr::cuda_pinned_memory_resource first{}; + { // comparison against a plain cuda_pinned_memory_resource + cuda::mr::cuda_pinned_memory_resource second{cudaHostAllocDefault}; + assert(first == second); + assert(!(first != second)); + } + + { // comparison against a plain cuda_pinned_memory_resource with a different flag set + cuda::mr::cuda_pinned_memory_resource second{cudaHostAllocPortable}; + assert(!(first == second)); + assert((first != second)); + } + + { // comparison against a cuda_pinned_memory_resource wrapped inside a resource_ref + cuda::mr::cuda_pinned_memory_resource second{}; + cuda::mr::resource_ref second_ref{second}; + assert(first == second_ref); + assert(!(first != second_ref)); + assert(second_ref == first); + assert(!(second_ref != first)); + } + + { // comparison against a cuda_pinned_memory_resource wrapped inside a resource_ref<> + cuda::mr::cuda_pinned_memory_resource second{}; + cuda::mr::resource_ref<> second_ref{second}; + assert(first == second_ref); + assert(!(first != second_ref)); + assert(second_ref == first); + assert(!(second_ref != first)); + } + + { // comparison against a different resource through resource_ref + resource host_resource{}; + resource device_resource{}; + assert(!(first == host_resource)); + assert(first != host_resource); + assert(!(first == device_resource)); + assert(first != device_resource); + + assert(!(host_resource == first)); + assert(host_resource != first); + assert(!(device_resource == first)); + assert(device_resource != first); + } + + { // comparison against a different resource through resource_ref + async_resource host_async_resource{}; + async_resource device_async_resource{}; + assert(!(first == host_async_resource)); + assert(first != host_async_resource); + assert(!(first == device_async_resource)); + assert(first != device_async_resource); + + assert(!(host_async_resource == first)); + assert(host_async_resource != first); + assert(!(device_async_resource == first)); + assert(device_async_resource != first); + } +} + +int main(int, char**) +{ + NV_IF_TARGET(NV_IS_HOST, test();) + return 0; +} diff --git a/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/traits.pass.cpp b/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/traits.pass.cpp new file mode 100644 index 00000000000..3909ac7238e --- /dev/null +++ b/libcudacxx/test/libcudacxx/cuda/memory_resource/cuda_pinned_memory_resource/traits.pass.cpp @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11 +// UNSUPPORTED: msvc-19.16 +// UNSUPPORTED: nvrtc + +#include +#include + +using resource = cuda::mr::cuda_pinned_memory_resource; +static_assert(!cuda::std::is_trivial::value, ""); +static_assert(!cuda::std::is_trivially_default_constructible::value, ""); +static_assert(cuda::std::is_trivially_copy_constructible::value, ""); +static_assert(cuda::std::is_trivially_move_constructible::value, ""); +static_assert(cuda::std::is_trivially_copy_assignable::value, ""); +static_assert(cuda::std::is_trivially_move_assignable::value, ""); +static_assert(cuda::std::is_trivially_destructible::value, ""); +static_assert(!cuda::std::is_empty::value, ""); + +int main(int, char**) +{ + return 0; +}