Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

More optimizations for contiguous iterators #1772

Merged
merged 6 commits into from
Apr 5, 2021
Merged

More optimizations for contiguous iterators #1772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e3fc01c
Implement more optimizations
AdamBucior Mar 23, 2021
8fb4b7a
clang-format
AdamBucior Mar 23, 2021
a579dda
Code review
AdamBucior Mar 23, 2021
3827926
Code review
AdamBucior Mar 23, 2021
94213f5
Code review
AdamBucior Mar 26, 2021
3c635e4
Forgot to save
AdamBucior Mar 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions stl/inc/algorithm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1388,7 +1388,15 @@ namespace ranges {
// clang-format off
template <input_iterator _It, sentinel_for<_It> _Se, weakly_incrementable _Out>
requires indirectly_copyable<_It, _Out>
_NODISCARD constexpr copy_result<_It, _Out> _Copy_unchecked(_It _First, const _Se _Last, _Out _Result) {
_NODISCARD constexpr copy_result<_It, _Out> _Copy_unchecked(_It _First, _Se _Last, _Out _Result) {
if constexpr (_Ptr_copy_cat<_It, _Out>::_Trivially_copyable && sized_sentinel_for<_Se, _It>) {
if (!_STD is_constant_evaluated()) {
auto _Final = _RANGES next(_First, _STD move(_Last));
_Result = _Copy_memmove(_STD move(_First), _Final, _STD move(_Result));
return {_STD move(_Final), _STD move(_Result)};
}
}

for (; _First != _Last; ++_First, (void) ++_Result) {
*_Result = *_First;
}
Expand Down Expand Up @@ -1440,6 +1448,15 @@ namespace ranges {
requires indirectly_copyable<_It, _Out>
constexpr copy_n_result<_It, _Out> operator()(_It _First, iter_difference_t<_It> _Count, _Out _Result) const {
auto _UFirst = _Get_unwrapped_n(_STD move(_First), _Count);
if constexpr (_Ptr_copy_cat<decltype(_UFirst), _Out>::_Trivially_copyable) {
if (!_STD is_constant_evaluated()) {
auto _Final = _UFirst + _Count;
_Result = _Copy_memmove(_STD move(_UFirst), _Final, _STD move(_Result));
_Seek_wrapped(_First, _STD move(_Final));
return {_STD move(_First), _STD move(_Result)};
}
}

for (; _Count > 0; ++_UFirst, (void) ++_Result, --_Count) {
*_Result = *_UFirst;
}
Expand Down Expand Up @@ -1583,8 +1600,15 @@ namespace ranges {
// clang-format off
template <input_iterator _It, sentinel_for<_It> _Se, weakly_incrementable _Out>
requires indirectly_movable<_It, _Out>
constexpr move_result<_It, _Out> _Move_unchecked(_It _First, const _Se _Last, _Out _Result) {
constexpr move_result<_It, _Out> _Move_unchecked(_It _First, _Se _Last, _Out _Result) {
// clang-format on
if constexpr (_Ptr_move_cat<_It, _Out>::_Trivially_copyable) {
if (!_STD is_constant_evaluated()) {
auto _Final = _RANGES next(_First, _STD move(_Last));
_Result = _Copy_memmove(_STD move(_First), _Final, _STD move(_Result));
return {_STD move(_Final), _STD move(_Result)};
}
}

for (; _First != _Last; ++_First, (void) ++_Result) {
*_Result = _RANGES iter_move(_First);
Expand Down
18 changes: 12 additions & 6 deletions stl/inc/memory
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,17 @@ namespace ranges {
private:
template <class _It, class _Se, class _Out, class _OSe>
_NODISCARD static uninitialized_copy_result<_It, _Out> _Uninitialized_copy_unchecked(
_It _IFirst, const _Se _ILast, _Out _OFirst, const _OSe _OLast) {
_It _IFirst, _Se _ILast, _Out _OFirst, _OSe _OLast) {
_STL_INTERNAL_STATIC_ASSERT(input_iterator<_It>);
_STL_INTERNAL_STATIC_ASSERT(sentinel_for<_Se, _It>);
_STL_INTERNAL_STATIC_ASSERT(_No_throw_forward_iterator<_Out>);
_STL_INTERNAL_STATIC_ASSERT(_No_throw_sentinel_for<_OSe, _Out>);
_STL_INTERNAL_STATIC_ASSERT(constructible_from<iter_value_t<_Out>, iter_reference_t<_It>>);

if constexpr (is_same_v<_Se, _It> && is_same_v<_OSe, _Out> && _Ptr_copy_cat<_It, _Out>::_Really_trivial) {
return _Copy_memcpy_common(_IFirst, _ILast, _OFirst, _OLast);
if constexpr (_Ptr_copy_cat<_It, _Out>::_Really_trivial
&& sized_sentinel_for<_Se, _It> && sized_sentinel_for<_OSe, _Out>) {
return _Copy_memcpy_common(_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst,
_RANGES next(_OFirst, _STD move(_OLast)));
} else {
_Uninitialized_backout _Backout{_STD move(_OFirst)};

Expand Down Expand Up @@ -506,10 +508,14 @@ namespace ranges {
// clang-format off
template <_No_throw_input_iterator _It, _No_throw_sentinel_for<_It> _Se>
requires destructible<iter_value_t<_It>>
_NODISCARD _CONSTEXPR20_DYNALLOC _It _Destroy_unchecked(_It _First, const _Se _Last) noexcept {
_NODISCARD _CONSTEXPR20_DYNALLOC _It _Destroy_unchecked(_It _First, _Se _Last) noexcept {
// clang-format on
for (; _First != _Last; ++_First) {
_RANGES destroy_at(_STD addressof(*_First));
if constexpr (is_trivially_destructible_v<iter_value_t<_It>>) {
_RANGES advance(_First, _STD move(_Last));
} else {
for (; _First != _Last; ++_First) {
_RANGES destroy_at(_STD addressof(*_First));
}
}

return _First;
Expand Down
33 changes: 25 additions & 8 deletions stl/inc/xmemory
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1557,13 +1557,28 @@ namespace ranges {
template <class _InIt, class _OutIt>
in_out_result<_InIt, _OutIt> _Copy_memcpy_common(
_InIt _IFirst, _InIt _ILast, _OutIt _OFirst, _OutIt _OLast) noexcept {
const auto _IFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_IFirst));
const auto _ILast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_ILast));
const auto _OFirst_ch = const_cast<char*>(reinterpret_cast<volatile char*>(_OFirst));
const auto _OLast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_OLast));
const auto _IFirstPtr = _To_address(_IFirst);
const auto _ILastPtr = _To_address(_ILast);
const auto _OFirstPtr = _To_address(_OFirst);
const auto _OLastPtr = _To_address(_OLast);
const auto _IFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_IFirstPtr));
const auto _ILast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_ILastPtr));
const auto _OFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_OFirstPtr));
const auto _OLast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_OLastPtr));
const auto _Count = static_cast<size_t>((_STD min)(_ILast_ch - _IFirst_ch, _OLast_ch - _OFirst_ch));
_CSTD memcpy(_OFirst_ch, _IFirst_ch, _Count);
return {reinterpret_cast<_InIt>(_IFirst_ch + _Count), reinterpret_cast<_OutIt>(_OFirst_ch + _Count)};
if constexpr (is_pointer_v<_InIt>) {
_IFirst = reinterpret_cast<_InIt>(_IFirst_ch + _Count);
} else {
_IFirst += _Count / sizeof(iter_value_t<_InIt>);
}

if constexpr (is_pointer_v<_OutIt>) {
_OFirst = reinterpret_cast<_OutIt>(_OFirst_ch + _Count);
} else {
_OFirst += _Count / sizeof(iter_value_t<_OutIt>);
}
return {_STD move(_IFirst), _STD move(_OFirst)};
}

// ALIAS TEMPLATE uninitialized_move_result
Expand All @@ -1576,10 +1591,12 @@ namespace ranges {
_No_throw_sentinel_for<_Out> _OSe>
requires constructible_from<iter_value_t<_Out>, iter_rvalue_reference_t<_It>>
uninitialized_move_result<_It, _Out> _Uninitialized_move_unchecked(
_It _IFirst, const _Se _ILast, _Out _OFirst, const _OSe _OLast) {
_It _IFirst, _Se _ILast, _Out _OFirst, _OSe _OLast) {
// clang-format on
if constexpr (is_same_v<_Se, _It> && is_same_v<_OSe, _Out> && _Ptr_move_cat<_It, _Out>::_Really_trivial) {
return _Copy_memcpy_common(_IFirst, _ILast, _OFirst, _OLast);
if constexpr (_Ptr_move_cat<_It, _Out>::_Really_trivial
&& sized_sentinel_for<_Se, _It> && sized_sentinel_for<_OSe, _Out>) {
return _Copy_memcpy_common(
_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
} else {
_Uninitialized_backout _Backout{_STD move(_OFirst)};

Expand Down
128 changes: 72 additions & 56 deletions stl/inc/xutility
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4041,6 +4041,37 @@ struct unreachable_sentinel_t : _Unreachable_sentinel_detail::_Base {}; // TRANS
inline constexpr unreachable_sentinel_t unreachable_sentinel{};
#endif // __cpp_lib_concepts

// _Iterator_is_contiguous<_Iter> reports whether an iterator is known to be contiguous.
// (Without concepts, this detection is limited, which will limit when we can activate optimizations.)

#ifdef __cpp_lib_concepts
// When concepts are available, we can detect arbitrary contiguous iterators.
template <class _Iter>
inline constexpr bool _Iterator_is_contiguous = contiguous_iterator<_Iter>;

template <class _Iter>
_NODISCARD constexpr auto _To_address(const _Iter& _Val) noexcept {
_STL_INTERNAL_STATIC_ASSERT(contiguous_iterator<_Iter>);
return _STD to_address(_Val);
}
#else // ^^^ defined(__cpp_lib_concepts) ^^^ / vvv !defined(__cpp_lib_concepts) vvv
// When concepts aren't available, we can detect pointers. (Iterators should be unwrapped before using this.)
template <class _Iter>
_INLINE_VAR constexpr bool _Iterator_is_contiguous = is_pointer_v<_Iter>;

template <class _Iter>
_NODISCARD constexpr auto _To_address(const _Iter& _Val) noexcept {
_STL_INTERNAL_STATIC_ASSERT(is_pointer_v<_Iter>);
return _Val;
}
#endif // ^^^ !defined(__cpp_lib_concepts) ^^^

// _Iterators_are_contiguous<_Iter1, _Iter2> reports whether both iterators are known to be contiguous.

template <class _Iter1, class _Iter2>
_INLINE_VAR constexpr bool _Iterators_are_contiguous =
_Iterator_is_contiguous<_Iter1>&& _Iterator_is_contiguous<_Iter2>;

// FUNCTION TEMPLATE copy
template <class _Source, class _Dest>
struct _Ptr_cat_helper {
Expand Down Expand Up @@ -4084,36 +4115,44 @@ struct _False_copy_cat {
static constexpr bool _Trivially_copyable = false;
};

template <class _Source, class _Dest>
struct _Ptr_copy_cat : _False_copy_cat {}; // unwrap the pointer-like type and dispatch to _Ptr_cat_helper for copy
// NOTE: pointer is not a contiguous iterator if it points to volatile type
template <class _Source, class _Dest, bool _Non_contiguous = !_Iterators_are_contiguous<_Source, _Dest>>
struct _Ptr_move_cat : _False_copy_cat {};
AdamBucior marked this conversation as resolved.
Show resolved Hide resolved

template <class _Source, class _Dest>
struct _Ptr_copy_cat<_Source*, _Dest*>
: conditional_t<is_trivially_assignable_v<_Dest&, _Source&>,
_Ptr_cat_helper<remove_cv_t<_Source>, remove_cv_t<_Dest>>, _False_copy_cat> {};
struct _Ptr_move_cat<_Source, _Dest, false>
: conditional_t<is_trivially_assignable_v<_Iter_ref_t<_Dest>, remove_reference_t<_Iter_ref_t<_Source>>>,
_Ptr_cat_helper<_Iter_value_t<_Source>, _Iter_value_t<_Dest>>, _False_copy_cat> {};

template <class _Source, class _Dest>
struct _Ptr_copy_cat<move_iterator<_Source*>, _Dest*> : _Ptr_copy_cat<_Source*, _Dest*> {};
struct _Ptr_move_cat<move_iterator<_Source>, _Dest, true> : _Ptr_move_cat<_Source, _Dest> {};

template <class _Source, class _Dest>
struct _Ptr_move_cat : _False_copy_cat {}; // unwrap the pointer-like type and dispatch to _Ptr_cat_helper for move
template <class _Source, class _Dest, bool _Non_contiguous = !_Iterators_are_contiguous<_Source, _Dest>>
struct _Ptr_copy_cat : _False_copy_cat {};

template <class _Source, class _Dest>
struct _Ptr_move_cat<_Source*, _Dest*>
: conditional_t<is_trivially_assignable_v<_Dest&, _Source>,
_Ptr_cat_helper<remove_cv_t<_Source>, remove_cv_t<_Dest>>, _False_copy_cat> {};
struct _Ptr_copy_cat<_Source, _Dest, false>
: conditional_t<is_trivially_assignable_v<_Iter_ref_t<_Dest>, _Iter_ref_t<_Source>>,
_Ptr_cat_helper<_Iter_value_t<_Source>, _Iter_value_t<_Dest>>, _False_copy_cat> {};

template <class _Source, class _Dest>
struct _Ptr_move_cat<move_iterator<_Source*>, _Dest*> : _Ptr_move_cat<_Source*, _Dest*> {};

template <class _InIt, class _OutIt>
_OutIt _Copy_memmove(_InIt _First, _InIt _Last, _OutIt _Dest) {
const char* const _First_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_First));
const char* const _Last_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_Last));
char* const _Dest_ch = const_cast<char*>(reinterpret_cast<volatile char*>(_Dest));
struct _Ptr_copy_cat<move_iterator<_Source>, _Dest, true> : _Ptr_move_cat<_Source, _Dest> {};

template <class _CtgIt, class _OutCtgIt>
_OutCtgIt _Copy_memmove(_CtgIt _First, _CtgIt _Last, _OutCtgIt _Dest) {
auto _FirstPtr = _To_address(_First);
auto _LastPtr = _To_address(_Last);
auto _DestPtr = _To_address(_Dest);
const char* const _First_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_FirstPtr));
const char* const _Last_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_LastPtr));
char* const _Dest_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_DestPtr));
const auto _Count = static_cast<size_t>(_Last_ch - _First_ch);
_CSTD memmove(_Dest_ch, _First_ch, _Count);
return reinterpret_cast<_OutIt>(_Dest_ch + _Count);
if constexpr (is_pointer_v<_OutCtgIt>) {
return reinterpret_cast<_OutCtgIt>(_Dest_ch + _Count);
} else {
return _Dest + (_LastPtr - _FirstPtr);
}
}

template <class _InIt, class _OutIt>
Expand Down Expand Up @@ -4215,14 +4254,22 @@ _FwdIt2 copy_n(_ExPo&&, _FwdIt1 _First, _Diff _Count_raw, _FwdIt2 _Dest) noexcep
#endif // _HAS_CXX17

// FUNCTION TEMPLATE copy_backward
template <class _BidIt1, class _BidIt2>
_BidIt2 _Copy_backward_memmove(_BidIt1 _First, _BidIt1 _Last, _BidIt2 _Dest) {
template <class _CtgIt1, class _CtgIt2>
_CtgIt2 _Copy_backward_memmove(_CtgIt1 _First, _CtgIt1 _Last, _CtgIt2 _Dest) {
// implement copy_backward-like function as memmove
const char* const _First_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_First));
const char* const _Last_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_Last));
char* const _Dest_ch = const_cast<char*>(reinterpret_cast<volatile char*>(_Dest));
auto _FirstPtr = _To_address(_First);
auto _LastPtr = _To_address(_Last);
auto _DestPtr = _To_address(_Dest);
const char* const _First_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_FirstPtr));
const char* const _Last_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_LastPtr));
char* const _Dest_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_DestPtr));
const auto _Count = static_cast<size_t>(_Last_ch - _First_ch);
return static_cast<_BidIt2>(_CSTD memmove(_Dest_ch - _Count, _First_ch, _Count));
auto _Result = _CSTD memmove(_Dest_ch - _Count, _First_ch, _Count);
if constexpr (is_pointer_v<_CtgIt2>) {
return static_cast<_CtgIt2>(_Result);
} else {
return _Dest - (_LastPtr - _FirstPtr);
}
}

template <class _BidIt1, class _BidIt2>
Expand Down Expand Up @@ -4355,37 +4402,6 @@ _BidIt2 move_backward(_ExPo&&, _BidIt1 _First, _BidIt1 _Last, _BidIt2 _Dest) noe

// FUNCTION TEMPLATE fill

// _Iterator_is_contiguous<_Iter> reports whether an iterator is known to be contiguous.
// (Without concepts, this detection is limited, which will limit when we can activate optimizations.)

#ifdef __cpp_lib_concepts
// When concepts are available, we can detect arbitrary contiguous iterators.
template <class _Iter>
inline constexpr bool _Iterator_is_contiguous = contiguous_iterator<_Iter>;

template <class _Iter>
_NODISCARD constexpr auto _To_address(const _Iter& _Val) noexcept {
_STL_INTERNAL_STATIC_ASSERT(contiguous_iterator<_Iter>);
return _STD to_address(_Val);
}
#else // ^^^ defined(__cpp_lib_concepts) ^^^ / vvv !defined(__cpp_lib_concepts) vvv
// When concepts aren't available, we can detect pointers. (Iterators should be unwrapped before using this.)
template <class _Iter>
_INLINE_VAR constexpr bool _Iterator_is_contiguous = is_pointer_v<_Iter>;

template <class _Iter>
_NODISCARD constexpr auto _To_address(const _Iter& _Val) noexcept {
_STL_INTERNAL_STATIC_ASSERT(is_pointer_v<_Iter>);
return _Val;
}
#endif // ^^^ !defined(__cpp_lib_concepts) ^^^

// _Iterators_are_contiguous<_Iter1, _Iter2> reports whether both iterators are known to be contiguous.

template <class _Iter1, class _Iter2>
_INLINE_VAR constexpr bool _Iterators_are_contiguous =
_Iterator_is_contiguous<_Iter1>&& _Iterator_is_contiguous<_Iter2>;

template <class _Ty>
struct _Is_character : false_type {}; // by default, not a character type

Expand Down
16 changes: 3 additions & 13 deletions tests/std/tests/VSO_0180469_ptr_cat/test.compile.pass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ struct test_ptr_cat_helper {
STATIC_ASSERT(!MoveReallyTrivial || MoveTriviallyCopyable);
};

template <int Expected, class Source, class Dest, int Volatile = Expected>
template <int Expected, class Source, class Dest>
void test_ptr_cat() {
(void) test_ptr_cat_helper<Expected, Source*, Dest*>{};
// Also make sure that the source being const doesn't change the answer
Expand All @@ -59,16 +59,6 @@ void test_ptr_cat() {
(void) test_ptr_cat_helper<0, const volatile Source*, const Dest*>{};
(void) test_ptr_cat_helper<0, const Source*, const volatile Dest*>{};
(void) test_ptr_cat_helper<0, const volatile Source*, const volatile Dest*>{};
// volatile anywhere should go to the trivial implementation for builtin types, but
// the general implementation for PODs, since the compiler-generated copy assign has signature:
// Meow& operator=(const Meow&);
// which hates volatile on both the source and the target
(void) test_ptr_cat_helper<Volatile, volatile Source*, Dest*>{};
(void) test_ptr_cat_helper<Volatile, Source*, volatile Dest*>{};
(void) test_ptr_cat_helper<Volatile, volatile Source*, volatile Dest*>{};
(void) test_ptr_cat_helper<Volatile, const volatile Source*, Dest*>{};
(void) test_ptr_cat_helper<Volatile, const Source*, volatile Dest*>{};
(void) test_ptr_cat_helper<Volatile, const volatile Source*, volatile Dest*>{};

// Also make sure _Ptr_cat listens to the iterator type
(void) test_ptr_cat_helper<0, typename list<Source>::iterator, typename list<Dest>::iterator>{};
Expand Down Expand Up @@ -131,8 +121,8 @@ void ptr_cat_test_cases() {
// Identity cases:
test_ptr_cat<2, int, int>();
test_ptr_cat<2, bool, bool>();
test_ptr_cat<2, pod_struct, pod_struct, 0>();
test_ptr_cat<1, trivially_copyable_struct, trivially_copyable_struct, 0>();
test_ptr_cat<2, pod_struct, pod_struct>();
test_ptr_cat<1, trivially_copyable_struct, trivially_copyable_struct>();
test_ptr_cat<0, custom_copy_struct, custom_copy_struct>();
test_ptr_cat<2, int_enum, int_enum>();
test_ptr_cat<2, short_enum, short_enum>();
Expand Down