diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__config b/libcudacxx/include/cuda/std/detail/libcxx/include/__config index 54d4dbe628..13124e9653 100644 --- a/libcudacxx/include/cuda/std/detail/libcxx/include/__config +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__config @@ -1623,23 +1623,24 @@ typedef unsigned int char32_t; # endif #endif // defined(__APPLE__) -#if !defined(_LIBCUDACXX_HAS_NO_ALIGNED_ALLOCATION) && \ - (defined(_LIBCUDACXX_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) || \ - (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606)) -# define _LIBCUDACXX_HAS_NO_ALIGNED_ALLOCATION -#endif +# if !defined(_LIBCUDACXX_HAS_NO_ALIGNED_ALLOCATION) \ + && (defined(_LIBCUDACXX_HAS_NO_LIBRARY_ALIGNED_ALLOCATION) \ + || (!defined(__cpp_aligned_new) || __cpp_aligned_new < 201606)) \ + || defined(__cuda_std__) // FIXME: Properly handle aligned allocations +# define _LIBCUDACXX_HAS_NO_ALIGNED_ALLOCATION +# endif -#if defined(__APPLE__) || defined(__FreeBSD__) -#define _LIBCUDACXX_HAS_DEFAULTRUNELOCALE -#endif +# if defined(__APPLE__) || defined(__FreeBSD__) +# define _LIBCUDACXX_HAS_DEFAULTRUNELOCALE +# endif -#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__sun__) -#define _LIBCUDACXX_WCTYPE_IS_MASK -#endif +# if defined(__APPLE__) || defined(__FreeBSD__) || defined(__sun__) +# define _LIBCUDACXX_WCTYPE_IS_MASK +# endif -#if _CCCL_STD_VER <= 2017 || !defined(__cpp_char8_t) -#define _LIBCUDACXX_NO_HAS_CHAR8_T -#endif +# if _CCCL_STD_VER <= 2017 || !defined(__cpp_char8_t) +# define _LIBCUDACXX_NO_HAS_CHAR8_T +# endif // Deprecation macros. // @@ -2156,6 +2157,15 @@ extern "C" _LIBCUDACXX_FUNC_VIS void __sanitizer_annotate_contiguous_container( _CUDA_VSTD::__construct_at(_CUDA_VSTD::addressof(_LOCATION), __VA_ARGS__) #endif +// We can only expose constexpr allocations if the compiler supports it +# if defined(__cpp_constexpr_dynamic_alloc) && defined(__cpp_lib_constexpr_dynamic_alloc) && _CCCL_STD_VER >= 2020 \ + && !defined(_CCCL_COMPILER_NVRTC) +# define _CCCL_HAS_CONSTEXPR_ALLOCATION +# define _CCCL_CONSTEXPR_CXX20_ALLOCATION constexpr +# else // ^^^ __cpp_constexpr_dynamic_alloc ^^^ / vvv !__cpp_constexpr_dynamic_alloc vvv +# define _CCCL_CONSTEXPR_CXX20_ALLOCATION +# endif + // NVRTC has a bug that prevented the use of delegated constructors, as it did not accept execution space annotations. // This creates a whole lot of boilerplate that we can avoid through a macro (see nvbug3961621) # if defined(_CCCL_COMPILER_NVRTC) || (defined(_LIBCUDACXX_CUDACC_BELOW_11_3) && defined(_CCCL_COMPILER_CLANG)) diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__functional_base b/libcudacxx/include/cuda/std/detail/libcxx/include/__functional_base index 76e8e64217..2de26d2ec0 100644 --- a/libcudacxx/include/cuda/std/detail/libcxx/include/__functional_base +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__functional_base @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -37,93 +39,6 @@ _LIBCUDACXX_BEGIN_NAMESPACE_STD -// allocator_arg_t - -struct _LIBCUDACXX_TEMPLATE_VIS allocator_arg_t { explicit allocator_arg_t() = default; }; - -#if defined(_LIBCUDACXX_BUILDING_LIBRARY) -extern _LIBCUDACXX_EXPORTED_FROM_ABI const allocator_arg_t allocator_arg; -#else -/* _LIBCUDACXX_INLINE_VAR */ constexpr allocator_arg_t allocator_arg = allocator_arg_t(); -#endif - -// uses_allocator - -template -struct __has_allocator_type -{ -private: - struct __two {char __lx; char __lxx;}; - template _LIBCUDACXX_INLINE_VISIBILITY static __two __test(...); - template _LIBCUDACXX_INLINE_VISIBILITY static char __test(typename _Up::allocator_type* = 0); -public: - static const bool value = sizeof(__test<_Tp>(0)) == 1; -}; - -template ::value> -struct __uses_allocator - : public integral_constant::value> -{ -}; - -template -struct __uses_allocator<_Tp, _Alloc, false> - : public false_type -{ -}; - -template -struct _LIBCUDACXX_TEMPLATE_VIS uses_allocator - : public __uses_allocator<_Tp, _Alloc> -{ -}; - -#if _CCCL_STD_VER > 2014 -template -_LIBCUDACXX_INLINE_VAR constexpr size_t uses_allocator_v = uses_allocator<_Tp, _Alloc>::value; -#endif - -// allocator construction - -template -struct __uses_alloc_ctor_imp -{ - typedef _LIBCUDACXX_NODEBUG_TYPE __remove_cvref_t<_Alloc> _RawAlloc; - static const bool __ua = uses_allocator<_Tp, _RawAlloc>::value; - static const bool __ic = - is_constructible<_Tp, allocator_arg_t, _Alloc, _Args...>::value; - static const int value = __ua ? 2 - __ic : 0; -}; - -template -struct __uses_alloc_ctor - : integral_constant::value> - {}; - -template -inline _LIBCUDACXX_INLINE_VISIBILITY -void __user_alloc_construct_impl (integral_constant, _Tp *__storage, const _Allocator &, _Args &&... __args ) -{ - new (__storage) _Tp (_CUDA_VSTD::forward<_Args>(__args)...); -} - -// FIXME: This should have a version which takes a non-const alloc. -template -inline _LIBCUDACXX_INLINE_VISIBILITY -void __user_alloc_construct_impl (integral_constant, _Tp *__storage, const _Allocator &__a, _Args &&... __args ) -{ - new (__storage) _Tp (allocator_arg, __a, _CUDA_VSTD::forward<_Args>(__args)...); -} - -// FIXME: This should have a version which takes a non-const alloc. -template -inline _LIBCUDACXX_INLINE_VISIBILITY -void __user_alloc_construct_impl (integral_constant, _Tp *__storage, const _Allocator &__a, _Args &&... __args ) -{ - new (__storage) _Tp (_CUDA_VSTD::forward<_Args>(__args)..., __a); -} - _LIBCUDACXX_END_NAMESPACE_STD #include diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/align.h b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/align.h new file mode 100644 index 0000000000..5c669494a0 --- /dev/null +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/align.h @@ -0,0 +1,55 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX___MEMORY_ALIGN_H +#define _LIBCUDACXX___MEMORY_ALIGN_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +_CCCL_DIAG_PUSH +_CCCL_DIAG_SUPPRESS_MSVC(4146) // unary minus operator applied to unsigned type, result still unsigned + +_LIBCUDACXX_BEGIN_NAMESPACE_STD + +inline _LIBCUDACXX_INLINE_VISIBILITY void* align(size_t __alignment, size_t __size, void*& __ptr, size_t& __space) +{ + if (__space < __size) + { + return nullptr; + } + + const auto __intptr = reinterpret_cast(__ptr); + const auto __aligned = (__intptr - 1u + __alignment) & -__alignment; + const auto __diff = __aligned - __intptr; + if (__diff > (__space - __size)) + { + return nullptr; + } + __space -= __diff; + return __ptr = reinterpret_cast(__aligned); +} + +_LIBCUDACXX_END_NAMESPACE_STD + +_CCCL_DIAG_POP + +#endif // _LIBCUDACXX___MEMORY_ALIGN_H diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocate_at_least.h b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocate_at_least.h new file mode 100644 index 0000000000..469978163a --- /dev/null +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocate_at_least.h @@ -0,0 +1,80 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX___MEMORY_ALLOCATE_AT_LEAST_H +#define _LIBCUDACXX___MEMORY_ALLOCATE_AT_LEAST_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +_LIBCUDACXX_BEGIN_NAMESPACE_STD + +#if _CCCL_STD_VER >= 2023 +template +struct allocation_result +{ + _Pointer ptr; + size_t count; +}; +_LIBCUDACXX_CTAD_SUPPORTED_FOR_TYPE(allocation_result); + +template +_LIBCUDACXX_NODISCARD_ATTRIBUTE _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY constexpr allocation_result< + typename allocator_traits<_Alloc>::pointer> +allocate_at_least(_Alloc& __alloc, size_t __n) +{ + if constexpr (requires { __alloc.allocate_at_least(__n); }) + { + return __alloc.allocate_at_least(__n); + } + else + { + return {__alloc.allocate(__n), __n}; + } +} + +template +_LIBCUDACXX_NODISCARD_ATTRIBUTE _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY constexpr auto +__allocate_at_least(_Alloc& __alloc, size_t __n) +{ + return _CUDA_VSTD::allocate_at_least(__alloc, __n); +} +#else // ^^^ _CCCL_STD_VER >= 2023 ^^^ / vvv _CCCL_STD_VER < 2023 vvv +template +struct __allocation_result +{ + _Pointer ptr; + size_t count; +}; + +template +_LIBCUDACXX_NODISCARD_ATTRIBUTE _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY constexpr __allocation_result< + typename allocator_traits<_Alloc>::pointer> +__allocate_at_least(_Alloc& __alloc, size_t __n) +{ + return {__alloc.allocate(__n), __n}; +} + +#endif // _CCCL_STD_VER >= 2023 + +_LIBCUDACXX_END_NAMESPACE_STD + +#endif // _LIBCUDACXX___MEMORY_ALLOCATE_AT_LEAST_H diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocation_guard.h b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocation_guard.h new file mode 100644 index 0000000000..3d65c2ce8b --- /dev/null +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocation_guard.h @@ -0,0 +1,97 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX___MEMORY_ALLOCATION_GUARD_H +#define _LIBCUDACXX___MEMORY_ALLOCATION_GUARD_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +_LIBCUDACXX_BEGIN_NAMESPACE_STD + +// Helper class to allocate memory using an Allocator in an exception safe +// manner. +// +// The intended usage of this class is as follows: +// +// 0 +// 1 __allocation_guard guard(alloc, 10); +// 2 do_some_initialization_that_may_throw(guard.__get()); +// 3 save_allocated_pointer_in_a_noexcept_operation(guard.__release_ptr()); +// 4 +// +// If line (2) throws an exception during initialization of the memory, the +// guard's destructor will be called, and the memory will be released using +// Allocator deallocation. Otherwise, we release the memory from the guard on +// line (3) in an operation that can't throw -- after that, the guard is not +// responsible for the memory anymore. +// +// This is similar to a unique_ptr, except it's easier to use with a +// custom allocator. + +_CCCL_NV_DIAG_SUPPRESS(2659) // constexpr non-static member function will not be implicitly 'const' in C++14 + +template +struct __allocation_guard +{ + using _Pointer = typename allocator_traits<_Alloc>::pointer; + using _Size = typename allocator_traits<_Alloc>::size_type; + + template // we perform the allocator conversion inside the constructor + _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 explicit __allocation_guard( + _AllocT __alloc, _Size __n) + : __alloc_(_CUDA_VSTD::move(__alloc)) + , __n_(__n) + , __ptr_(allocator_traits<_Alloc>::allocate(__alloc_, __n_)) // initialization order is important + {} + + _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 ~__allocation_guard() noexcept + { + if (__ptr_ != nullptr) + { + allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __n_); + } + } + + _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX14 _Pointer __release_ptr() noexcept + { // not called __release() because it's a keyword in objective-c++ + _Pointer __tmp = __ptr_; + __ptr_ = nullptr; + return __tmp; + } + + _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY constexpr _Pointer __get() const noexcept + { + return __ptr_; + } + +private: + _Alloc __alloc_; + _Size __n_; + _Pointer __ptr_; +}; + +_CCCL_NV_DIAG_DEFAULT(2659) + +_LIBCUDACXX_END_NAMESPACE_STD + +#endif // _LIBCUDACXX___MEMORY_ALLOCATION_GUARD_H diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator.h b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator.h new file mode 100644 index 0000000000..de0ba61a8f --- /dev/null +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator.h @@ -0,0 +1,328 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX___MEMORY_ALLOCATOR_H +#define _LIBCUDACXX___MEMORY_ALLOCATOR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_CCCL_HAS_CONSTEXPR_ALLOCATION) && !defined(_CCCL_COMPILER_NVRTC) +# include +#endif // _CCCL_HAS_CONSTEXPR_ALLOCATION && !_CCCL_COMPILER_NVRTC + +_LIBCUDACXX_BEGIN_NAMESPACE_STD + +template +class allocator; + +#if _CCCL_STD_VER <= 2017 +// These specializations shouldn't be marked _LIBCUDACXX_DEPRECATED_IN_CXX17. +// Specializing allocator is deprecated, but not using it. +template <> +class _LIBCUDACXX_TEMPLATE_VIS allocator +{ +public: + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef void* pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const void* const_pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef void value_type; + + template + struct _LIBCUDACXX_DEPRECATED_IN_CXX17 rebind + { + typedef allocator<_Up> other; + }; +}; + +template <> +class _LIBCUDACXX_TEMPLATE_VIS allocator +{ +public: + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const void* pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const void* const_pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const void value_type; + + template + struct _LIBCUDACXX_DEPRECATED_IN_CXX17 rebind + { + typedef allocator<_Up> other; + }; +}; +#endif // _CCCL_STD_VER <= 2017 + +// This class provides a non-trivial default constructor to the class that derives from it +// if the condition is satisfied. +// +// The second template parameter exists to allow giving a unique type to __non_trivial_if, +// which makes it possible to avoid breaking the ABI when making this a base class of an +// existing class. Without that, imagine we have classes D1 and D2, both of which used to +// have no base classes, but which now derive from __non_trivial_if. The layout of a class +// that inherits from both D1 and D2 will change because the two __non_trivial_if base +// classes are not allowed to share the same address. +// +// By making those __non_trivial_if base classes unique, we work around this problem and +// it is safe to start deriving from __non_trivial_if in existing classes. +template +struct __non_trivial_if +{}; + +template +struct __non_trivial_if +{ + _LIBCUDACXX_INLINE_VISIBILITY constexpr __non_trivial_if() noexcept {} +}; + +// allocator +// +// Note: For ABI compatibility between C++20 and previous standards, we make +// allocator trivial in C++20. + +template +class _LIBCUDACXX_TEMPLATE_VIS allocator : private __non_trivial_if> +{ + static_assert(!_LIBCUDACXX_TRAIT(is_volatile, _Tp), "std::allocator does not support volatile types"); + +public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef _Tp value_type; + typedef true_type propagate_on_container_move_assignment; + typedef true_type is_always_equal; + + _CCCL_CONSTEXPR_CXX20 allocator() noexcept = default; + + template + _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 allocator(const allocator<_Up>&) noexcept + {} + + _CCCL_EXEC_CHECK_DISABLE + _LIBCUDACXX_NODISCARD_AFTER_CXX17 _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20_ALLOCATION _Tp* + allocate(size_t __n) + { + if (__n > allocator_traits::max_size(*this)) + { + __throw_bad_array_new_length(); + } +#if defined(_CCCL_HAS_CONSTEXPR_ALLOCATION) + if (__libcpp_is_constant_evaluated()) + { + return ::std::allocator<_Tp>{}.allocate(__n); + } +#endif // _CCCL_HAS_CONSTEXPR_ALLOCATION + { + return static_cast<_Tp*>(_CUDA_VSTD::__libcpp_allocate(__n * sizeof(_Tp), _LIBCUDACXX_ALIGNOF(_Tp))); + } + } + +#if _CCCL_STD_VER >= 2023 + _LIBCUDACXX_NODISCARD_ATTRIBUTE _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY constexpr allocation_result< + _Tp*> + allocate_at_least(size_t __n) + { + return {allocate(__n), __n}; + } +#endif // _CCCL_HAS_CONSTEXPR_ALLOCATION + + _CCCL_EXEC_CHECK_DISABLE + _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20_ALLOCATION void deallocate(_Tp* __p, size_t __n) noexcept + { +#if defined(_CCCL_HAS_CONSTEXPR_ALLOCATION) + if (__libcpp_is_constant_evaluated()) + { + return ::std::allocator<_Tp>{}.deallocate(__p, __n); + } + else +#endif // _CCCL_STD_VER >= 2020 + { + _CUDA_VSTD::__libcpp_deallocate((void*) __p, __n * sizeof(_Tp), _LIBCUDACXX_ALIGNOF(_Tp)); + } + } + + // C++20 Removed members +#if _CCCL_STD_VER <= 2017 + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef _Tp* pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef _Tp& reference; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const _Tp& const_reference; + + template + struct _LIBCUDACXX_DEPRECATED_IN_CXX17 rebind + { + typedef allocator<_Up> other; + }; + + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY pointer address(reference __x) const noexcept + { + return _CUDA_VSTD::addressof(__x); + } + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY const_pointer address(const_reference __x) const noexcept + { + return _CUDA_VSTD::addressof(__x); + } + + _LIBCUDACXX_NODISCARD_AFTER_CXX17 _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_DEPRECATED_IN_CXX17 _Tp* + allocate(size_t __n, const void*) + { + return allocate(__n); + } + + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY size_type max_size() const noexcept + { + return size_type(~0) / sizeof(_Tp); + } + + template + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY void construct(_Up* __p, _Args&&... __args) + { + ::new ((void*) __p) _Up(_CUDA_VSTD::forward<_Args>(__args)...); + } + + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY void destroy(pointer __p) + { + __p->~_Tp(); + } +#endif // _CCCL_STD_VER <= 2017 +}; + +template +class _LIBCUDACXX_TEMPLATE_VIS allocator + : private __non_trivial_if> +{ + static_assert(!_LIBCUDACXX_TRAIT(is_volatile, _Tp), "std::allocator does not support volatile types"); + +public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef const _Tp value_type; + typedef true_type propagate_on_container_move_assignment; + typedef true_type is_always_equal; + + _CCCL_CONSTEXPR_CXX20 allocator() noexcept = default; + + template + _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 allocator(const allocator<_Up>&) noexcept + {} + + _LIBCUDACXX_NODISCARD_AFTER_CXX17 _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 const _Tp* allocate(size_t __n) + { + if (__n > allocator_traits::max_size(*this)) + { + __throw_bad_array_new_length(); + } + if (__libcpp_is_constant_evaluated()) + { + return static_cast(::operator new(__n * sizeof(_Tp))); + } + else + { + return static_cast(_CUDA_VSTD::__libcpp_allocate(__n * sizeof(_Tp), _LIBCUDACXX_ALIGNOF(_Tp))); + } + } + +#if _CCCL_STD_VER >= 2023 + _LIBCUDACXX_NODISCARD_ATTRIBUTE _LIBCUDACXX_HIDE_FROM_ABI _LIBCUDACXX_INLINE_VISIBILITY constexpr allocation_result< + const _Tp*> + allocate_at_least(size_t __n) + { + return {allocate(__n), __n}; + } +#endif // _CCCL_STD_VER >= 2023 + + _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 void deallocate(const _Tp* __p, size_t __n) + { + if (__libcpp_is_constant_evaluated()) + { + ::operator delete(const_cast<_Tp*>(__p)); + } + else + { + _CUDA_VSTD::__libcpp_deallocate((void*) const_cast<_Tp*>(__p), __n * sizeof(_Tp), _LIBCUDACXX_ALIGNOF(_Tp)); + } + } + + // C++20 Removed members +#if _CCCL_STD_VER <= 2017 + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const _Tp* pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const _Tp& reference; + _LIBCUDACXX_DEPRECATED_IN_CXX17 typedef const _Tp& const_reference; + + template + struct _LIBCUDACXX_DEPRECATED_IN_CXX17 rebind + { + typedef allocator<_Up> other; + }; + + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY const_pointer address(const_reference __x) const noexcept + { + return _CUDA_VSTD::addressof(__x); + } + + _LIBCUDACXX_NODISCARD_AFTER_CXX17 _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_DEPRECATED_IN_CXX17 const _Tp* + allocate(size_t __n, const void*) + { + return allocate(__n); + } + + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY size_type max_size() const noexcept + { + return size_type(~0) / sizeof(_Tp); + } + + template + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY void construct(_Up* __p, _Args&&... __args) + { + ::new ((void*) __p) _Up(_CUDA_VSTD::forward<_Args>(__args)...); + } + + _LIBCUDACXX_DEPRECATED_IN_CXX17 _LIBCUDACXX_INLINE_VISIBILITY void destroy(pointer __p) + { + __p->~_Tp(); + } +#endif // _CCCL_STD_VER <= 2017 +}; + +template +inline _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 bool +operator==(const allocator<_Tp>&, const allocator<_Up>&) noexcept +{ + return true; +} + +template +inline _LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX20 bool +operator!=(const allocator<_Tp>&, const allocator<_Up>&) noexcept +{ + return false; +} + +_LIBCUDACXX_END_NAMESPACE_STD + +#endif // _LIBCUDACXX___MEMORY_ALLOCATOR_H diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_arg_t.h b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_arg_t.h new file mode 100644 index 0000000000..277ab32ea9 --- /dev/null +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_arg_t.h @@ -0,0 +1,80 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX___FUNCTIONAL_ALLOCATOR_ARG_T_H +#define _LIBCUDACXX___FUNCTIONAL_ALLOCATOR_ARG_T_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +_LIBCUDACXX_BEGIN_NAMESPACE_STD + +struct _LIBCUDACXX_TEMPLATE_VIS allocator_arg_t +{ + explicit allocator_arg_t() = default; +}; + +_LIBCUDACXX_INLINE_VAR constexpr allocator_arg_t allocator_arg = allocator_arg_t(); + +// allocator construction + +template +struct __uses_alloc_ctor_imp +{ + typedef _LIBCUDACXX_NODEBUG __remove_cvref_t<_Alloc> _RawAlloc; + static const bool __ua = uses_allocator<_Tp, _RawAlloc>::value; + static const bool __ic = is_constructible<_Tp, allocator_arg_t, _Alloc, _Args...>::value; + static const int value = __ua ? 2 - __ic : 0; +}; + +template +struct __uses_alloc_ctor : integral_constant::value> +{}; + +template +inline _LIBCUDACXX_INLINE_VISIBILITY void +__user_alloc_construct_impl(integral_constant, _Tp* __storage, const _Allocator&, _Args&&... __args) +{ + new (__storage) _Tp(_CUDA_VSTD::forward<_Args>(__args)...); +} + +// FIXME: This should have a version which takes a non-const alloc. +template +inline _LIBCUDACXX_INLINE_VISIBILITY void +__user_alloc_construct_impl(integral_constant, _Tp* __storage, const _Allocator& __a, _Args&&... __args) +{ + new (__storage) _Tp(allocator_arg, __a, _CUDA_VSTD::forward<_Args>(__args)...); +} + +// FIXME: This should have a version which takes a non-const alloc. +template +inline _LIBCUDACXX_INLINE_VISIBILITY void +__user_alloc_construct_impl(integral_constant, _Tp* __storage, const _Allocator& __a, _Args&&... __args) +{ + new (__storage) _Tp(_CUDA_VSTD::forward<_Args>(__args)..., __a); +} + +_LIBCUDACXX_END_NAMESPACE_STD + +#endif // _LIBCUDACXX___FUNCTIONAL_ALLOCATOR_ARG_T_H diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_destructor.h b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_destructor.h new file mode 100644 index 0000000000..dfd3bc8743 --- /dev/null +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_destructor.h @@ -0,0 +1,55 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX___MEMORY_ALLOCATOR_DESTRUCTOR_H +#define _LIBCUDACXX___MEMORY_ALLOCATOR_DESTRUCTOR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +_LIBCUDACXX_BEGIN_NAMESPACE_STD + +template +class __allocator_destructor +{ + typedef _LIBCUDACXX_NODEBUG_TYPE allocator_traits<_Alloc> __alloc_traits; + +public: + typedef _LIBCUDACXX_NODEBUG_TYPE typename __alloc_traits::pointer pointer; + typedef _LIBCUDACXX_NODEBUG_TYPE typename __alloc_traits::size_type size_type; + +private: + _Alloc& __alloc_; + size_type __s_; + +public: + _LIBCUDACXX_INLINE_VISIBILITY __allocator_destructor(_Alloc& __a, size_type __s) noexcept + : __alloc_(__a) + , __s_(__s) + {} + _LIBCUDACXX_INLINE_VISIBILITY void operator()(pointer __p) noexcept + { + __alloc_traits::deallocate(__alloc_, __p, __s_); + } +}; + +_LIBCUDACXX_END_NAMESPACE_STD + +#endif // _LIBCUDACXX___MEMORY_ALLOCATOR_DESTRUCTOR_H diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_traits.h b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_traits.h new file mode 100644 index 0000000000..df56f4d8ca --- /dev/null +++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__memory/allocator_traits.h @@ -0,0 +1,604 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX___MEMORY_ALLOCATOR_TRAITS_H +#define _LIBCUDACXX___MEMORY_ALLOCATOR_TRAITS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_CCCL_NV_DIAG_SUPPRESS(1215) + +_LIBCUDACXX_BEGIN_NAMESPACE_STD + +#if _CCCL_STD_VER <= 2014 +# define _LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(NAME, PROPERTY) \ + template \ + struct NAME : false_type \ + {}; \ + template \ + struct NAME<_Tp, __void_t> : true_type \ + {} + +#else // ^^^ _CCCL_STD_VER <= 2014 ^^^ / vvv _CCCL_STD_VER >= 2017 vvv +# define _LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(NAME, PROPERTY) \ + template \ + inline constexpr bool NAME##_v = false; \ + template \ + inline constexpr bool NAME##_v<_Tp, __void_t> = true; +#endif // _CCCL_STD_VER >= 2017 + +// __pointer +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_pointer, pointer); +template , + bool = _LIBCUDACXX_TRAIT(__has_pointer, _RawAlloc)> +struct __pointer +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _RawAlloc::pointer; +}; +template +struct __pointer<_Tp, _Alloc, _RawAlloc, false> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = _Tp*; +}; + +// __const_pointer +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_const_pointer, const_pointer); +template +struct __const_pointer +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::const_pointer; +}; +template +struct __const_pointer<_Tp, _Ptr, _Alloc, false> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename pointer_traits<_Ptr>::template rebind; +}; + +// __void_pointer +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_void_pointer, void_pointer); +template +struct __void_pointer +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::void_pointer; +}; +template +struct __void_pointer<_Ptr, _Alloc, false> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename pointer_traits<_Ptr>::template rebind; +}; + +// __const_void_pointer +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_const_void_pointer, const_void_pointer); +template +struct __const_void_pointer +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::const_void_pointer; +}; +template +struct __const_void_pointer<_Ptr, _Alloc, false> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename pointer_traits<_Ptr>::template rebind; +}; + +// __size_type +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_size_type, size_type); +template +struct __size_type : make_unsigned<_DiffType> +{}; +template +struct __size_type<_Alloc, _DiffType, true> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::size_type; +}; + +// __alloc_traits_difference_type +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_alloc_traits_difference_type, difference_type); +template +struct __alloc_traits_difference_type +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename pointer_traits<_Ptr>::difference_type; +}; +template +struct __alloc_traits_difference_type<_Alloc, _Ptr, true> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::difference_type; +}; + +// __propagate_on_container_copy_assignment +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_propagate_on_container_copy_assignment, + propagate_on_container_copy_assignment); +template +struct __propagate_on_container_copy_assignment : false_type +{}; +template +struct __propagate_on_container_copy_assignment<_Alloc, true> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::propagate_on_container_copy_assignment; +}; + +// __propagate_on_container_move_assignment +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_propagate_on_container_move_assignment, + propagate_on_container_move_assignment); +template +struct __propagate_on_container_move_assignment : false_type +{}; +template +struct __propagate_on_container_move_assignment<_Alloc, true> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::propagate_on_container_move_assignment; +}; + +// __propagate_on_container_swap +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_propagate_on_container_swap, propagate_on_container_swap); +template +struct __propagate_on_container_swap : false_type +{}; +template +struct __propagate_on_container_swap<_Alloc, true> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::propagate_on_container_swap; +}; + +// __is_always_equal +_LIBCUDACXX_ALLOCATOR_TRAITS_HAS_XXX(__has_is_always_equal, is_always_equal); +template +struct __is_always_equal : is_empty<_Alloc> +{}; +template +struct __is_always_equal<_Alloc, true> +{ + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Alloc::is_always_equal; +}; + +// __allocator_traits_rebind +_CCCL_SUPPRESS_DEPRECATED_PUSH +template +struct __has_rebind_other : false_type +{}; +template +struct __has_rebind_other<_Tp, _Up, __void_t::other>> : true_type +{}; + +template ::value> +struct __allocator_traits_rebind +{ + static_assert(__has_rebind_other<_Tp, _Up>::value, "This allocator has to implement rebind"); + using type _LIBCUDACXX_NODEBUG_TYPE = typename _Tp::template rebind<_Up>::other; +}; +template