Skip to content

Commit

Permalink
Optimizations for unreachable sentinels (#1810)
Browse files Browse the repository at this point in the history 
Co-authored-by: Casey Carter <[email protected]>
Co-authored-by: Stephan T. Lavavej <[email protected]>
  • Loading branch information
@AdamBucior @CaseyCarter @StephanTLavavej
3 people authored Jun 12, 2021
1 parent 264765b commit e326ec1
Show file tree
Hide file tree
Showing 11 changed files with 247 additions and 52 deletions.
25 changes: 20 additions & 5 deletions stl/inc/algorithm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10260,12 +10260,27 @@ namespace ranges {

using _Memcmp_classification_pred =
typename decltype(_Lex_compare_memcmp_classify(_First1, _First2, _Pred))::_Pred;
if constexpr (!is_void_v<_Memcmp_classification_pred> && sized_sentinel_for<_Se1, _It1> //
&& sized_sentinel_for<_Se2, _It2> && same_as<_Pj1, identity> && same_as<_Pj2, identity>) {
constexpr bool _Is_sized1 = sized_sentinel_for<_Se1, _It1>;
constexpr bool _Is_sized2 = sized_sentinel_for<_Se2, _It2>;
if constexpr (!is_void_v<_Memcmp_classification_pred> && _Sized_or_unreachable_sentinel_for<_Se1, _It1> //
&& _Sized_or_unreachable_sentinel_for<_Se2, _It2> //
&& same_as<_Pj1, identity> && same_as<_Pj2, identity>) {
if (!_STD is_constant_evaluated()) {
const auto _Num1 = static_cast<size_t>(_Last1 - _First1);
const auto _Num2 = static_cast<size_t>(_Last2 - _First2);
const int _Ans = _Memcmp_count(_First1, _First2, (_STD min)(_Num1, _Num2));
size_t _Num1;
if constexpr (_Is_sized1) {
_Num1 = static_cast<size_t>(_Last1 - _First1);
} else {
_Num1 = SIZE_MAX;
}

size_t _Num2;
if constexpr (_Is_sized2) {
_Num2 = static_cast<size_t>(_Last2 - _First2);
} else {
_Num2 = SIZE_MAX;
}

const int _Ans = _Memcmp_count(_First1, _First2, (_STD min)(_Num1, _Num2));
return _Memcmp_classification_pred{}(_Ans, 0) || (_Ans == 0 && _Num1 < _Num2);
}
}
Expand Down
54 changes: 39 additions & 15 deletions stl/inc/memory
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,20 @@ namespace ranges {
_STL_INTERNAL_STATIC_ASSERT(_No_throw_sentinel_for<_OSe, _Out>);
_STL_INTERNAL_STATIC_ASSERT(constructible_from<iter_value_t<_Out>, iter_reference_t<_It>>);

if constexpr (_Ptr_copy_cat<_It, _Out>::_Really_trivial
&& sized_sentinel_for<_Se, _It> && sized_sentinel_for<_OSe, _Out>) {
return _Copy_memcpy_common(_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst,
_RANGES next(_OFirst, _STD move(_OLast)));
constexpr bool _Is_sized1 = sized_sentinel_for<_Se, _It>;
constexpr bool _Is_sized2 = sized_sentinel_for<_OSe, _Out>;
if constexpr (_Ptr_copy_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_Se, _It> //
&& _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (_Is_sized1 && _Is_sized2) {
return _Copy_memcpy_common(_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst,
_RANGES next(_OFirst, _STD move(_OLast)));
} else if constexpr (_Is_sized1) {
return _Copy_memcpy_distance(_IFirst, _OFirst, _IFirst, _RANGES next(_IFirst, _STD move(_ILast)));
} else if constexpr (_Is_sized2) {
return _Copy_memcpy_distance(_IFirst, _OFirst, _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
} else {
_STL_ASSERT(false, "Tried to uninitialized_copy two ranges with unreachable sentinels");
}
} else {
_Uninitialized_backout _Backout{_STD move(_OFirst)};

Expand Down Expand Up @@ -147,13 +157,20 @@ namespace ranges {
}

_Adl_verify_range(_First2, _Last2);
auto _IFirst = _Get_unwrapped_n(_STD move(_First1), _Count);
auto _OFirst = _Get_unwrapped(_STD move(_First2));
const auto _OLast = _Get_unwrapped(_STD move(_Last2));
if constexpr (_Ptr_copy_cat<_It, _Out>::_Really_trivial) {
auto _UResult = _Copy_memcpy_common(_IFirst, _IFirst + _Count, _OFirst, _OLast);
_IFirst = _UResult.in;
_OFirst = _UResult.out;
auto _IFirst = _Get_unwrapped_n(_STD move(_First1), _Count);
auto _OFirst = _Get_unwrapped(_STD move(_First2));
auto _OLast = _Get_unwrapped(_STD move(_Last2));
if constexpr (_Ptr_copy_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (sized_sentinel_for<_OSe, _Out>) {
auto _UResult = _Copy_memcpy_common(
_IFirst, _IFirst + _Count, _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
_IFirst = _STD move(_UResult.in);
_OFirst = _STD move(_UResult.out);
} else {
auto _UResult = _Copy_memcpy_count(_IFirst, _OFirst, static_cast<size_t>(_Count));
_IFirst = _STD move(_UResult.in);
_OFirst = _STD move(_UResult.out);
}
} else {
_Uninitialized_backout _Backout{_STD move(_OFirst)};

Expand Down Expand Up @@ -286,10 +303,17 @@ namespace ranges {
auto _IFirst = _Get_unwrapped_n(_STD move(_First1), _Count);
auto _OFirst = _Get_unwrapped(_STD move(_First2));
const auto _OLast = _Get_unwrapped(_STD move(_Last2));
if constexpr (_Ptr_move_cat<_It, _Out>::_Really_trivial) {
auto _UResult = _Copy_memcpy_common(_IFirst, _IFirst + _Count, _OFirst, _OLast);
_IFirst = _UResult.in;
_OFirst = _UResult.out;
if constexpr (_Ptr_move_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (sized_sentinel_for<_OSe, _Out>) {
auto _UResult = _Copy_memcpy_common(
_IFirst, _IFirst + _Count, _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
_IFirst = _STD move(_UResult.in);
_OFirst = _STD move(_UResult.out);
} else {
auto _UResult = _Copy_memcpy_count(_IFirst, _OFirst, static_cast<size_t>(_Count));
_IFirst = _STD move(_UResult.in);
_OFirst = _STD move(_UResult.out);
}
} else {
_Uninitialized_backout _Backout{_STD move(_OFirst)};

Expand Down
96 changes: 78 additions & 18 deletions stl/inc/xmemory
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1533,29 +1533,79 @@ namespace ranges {
&& _No_throw_forward_iterator<iterator_t<_Rng>>;
// clang-format on

template <class _InIt, class _OutIt>
in_out_result<_InIt, _OutIt> _Copy_memcpy_count(_InIt _IFirst, _OutIt _OFirst, const size_t _Count) noexcept {
const auto _IFirstPtr = _To_address(_IFirst);
const auto _OFirstPtr = _To_address(_OFirst);
const auto _IFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_IFirstPtr));
const auto _OFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_OFirstPtr));
const size_t _Count_bytes = _Count * sizeof(iter_value_t<_InIt>);
_CSTD memcpy(_OFirst_ch, _IFirst_ch, _Count_bytes);
if constexpr (is_pointer_v<_InIt>) {
_IFirst = reinterpret_cast<_InIt>(_IFirst_ch + _Count_bytes);
} else {
_IFirst += _Count;
}

if constexpr (is_pointer_v<_OutIt>) {
_OFirst = reinterpret_cast<_OutIt>(_OFirst_ch + _Count_bytes);
} else {
_OFirst += _Count;
}
return {_STD move(_IFirst), _STD move(_OFirst)};
}

template <class _InIt, class _OutIt, class _DistIt>
in_out_result<_InIt, _OutIt> _Copy_memcpy_distance(
_InIt _IFirst, _OutIt _OFirst, const _DistIt _DFirst, const _DistIt _DLast) noexcept {
// equivalent to _Copy_memcpy_count(_IFirst, _OFirst, _DLast - _DFirst) but computes distance more efficiently
const auto _IFirstPtr = _To_address(_IFirst);
const auto _OFirstPtr = _To_address(_OFirst);
const auto _DFirstPtr = _To_address(_DFirst);
const auto _DLastPtr = _To_address(_DLast);
const auto _IFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_IFirstPtr));
const auto _OFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_OFirstPtr));
const auto _DFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_DFirstPtr));
const auto _DLast_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_DLastPtr));
const auto _Count_bytes = static_cast<size_t>(_DLast_ch - _DFirst_ch);
_CSTD memcpy(_OFirst_ch, _IFirst_ch, _Count_bytes);
if constexpr (is_pointer_v<_InIt>) {
_IFirst = reinterpret_cast<_InIt>(_IFirst_ch + _Count_bytes);
} else {
_IFirst += _Count_bytes / sizeof(iter_value_t<_InIt>);
}

if constexpr (is_pointer_v<_OutIt>) {
_OFirst = reinterpret_cast<_OutIt>(_OFirst_ch + _Count_bytes);
} else {
_OFirst += _Count_bytes / sizeof(iter_value_t<_OutIt>);
}
return {_STD move(_IFirst), _STD move(_OFirst)};
}

template <class _InIt, class _OutIt>
in_out_result<_InIt, _OutIt> _Copy_memcpy_common(
_InIt _IFirst, _InIt _ILast, _OutIt _OFirst, _OutIt _OLast) noexcept {
const auto _IFirstPtr = _To_address(_IFirst);
const auto _ILastPtr = _To_address(_ILast);
const auto _OFirstPtr = _To_address(_OFirst);
const auto _OLastPtr = _To_address(_OLast);
const auto _IFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_IFirstPtr));
const auto _ILast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_ILastPtr));
const auto _OFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_OFirstPtr));
const auto _OLast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_OLastPtr));
const auto _Count = static_cast<size_t>((_STD min)(_ILast_ch - _IFirst_ch, _OLast_ch - _OFirst_ch));
_CSTD memcpy(_OFirst_ch, _IFirst_ch, _Count);
const auto _IFirstPtr = _To_address(_IFirst);
const auto _ILastPtr = _To_address(_ILast);
const auto _OFirstPtr = _To_address(_OFirst);
const auto _OLastPtr = _To_address(_OLast);
const auto _IFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_IFirstPtr));
const auto _ILast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_ILastPtr));
const auto _OFirst_ch = const_cast<char*>(reinterpret_cast<const volatile char*>(_OFirstPtr));
const auto _OLast_ch = const_cast<const char*>(reinterpret_cast<const volatile char*>(_OLastPtr));
const auto _Count_bytes = static_cast<size_t>((_STD min)(_ILast_ch - _IFirst_ch, _OLast_ch - _OFirst_ch));
_CSTD memcpy(_OFirst_ch, _IFirst_ch, _Count_bytes);
if constexpr (is_pointer_v<_InIt>) {
_IFirst = reinterpret_cast<_InIt>(_IFirst_ch + _Count);
_IFirst = reinterpret_cast<_InIt>(_IFirst_ch + _Count_bytes);
} else {
_IFirst += _Count / sizeof(iter_value_t<_InIt>);
_IFirst += _Count_bytes / sizeof(iter_value_t<_InIt>);
}

if constexpr (is_pointer_v<_OutIt>) {
_OFirst = reinterpret_cast<_OutIt>(_OFirst_ch + _Count);
_OFirst = reinterpret_cast<_OutIt>(_OFirst_ch + _Count_bytes);
} else {
_OFirst += _Count / sizeof(iter_value_t<_OutIt>);
_OFirst += _Count_bytes / sizeof(iter_value_t<_OutIt>);
}
return {_STD move(_IFirst), _STD move(_OFirst)};
}
Expand All @@ -1572,10 +1622,20 @@ namespace ranges {
uninitialized_move_result<_It, _Out> _Uninitialized_move_unchecked(
_It _IFirst, _Se _ILast, _Out _OFirst, _OSe _OLast) {
// clang-format on
if constexpr (_Ptr_move_cat<_It, _Out>::_Really_trivial
&& sized_sentinel_for<_Se, _It> && sized_sentinel_for<_OSe, _Out>) {
return _Copy_memcpy_common(
_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
constexpr bool _Is_sized1 = sized_sentinel_for<_Se, _It>;
constexpr bool _Is_sized2 = sized_sentinel_for<_OSe, _Out>;
if constexpr (_Ptr_move_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_Se, _It> //
&& _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (_Is_sized1 && _Is_sized2) {
return _Copy_memcpy_common(_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst,
_RANGES next(_OFirst, _STD move(_OLast)));
} else if constexpr (_Is_sized1) {
return _Copy_memcpy_distance(_IFirst, _OFirst, _IFirst, _RANGES next(_IFirst, _STD move(_ILast)));
} else if constexpr (_Is_sized2) {
return _Copy_memcpy_distance(_IFirst, _OFirst, _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
} else {
_STL_ASSERT(false, "Tried to uninitialized_move two ranges with unreachable sentinels");
}
} else {
_Uninitialized_backout _Backout{_STD move(_OFirst)};

Expand Down
22 changes: 19 additions & 3 deletions stl/inc/xutility
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5362,20 +5362,36 @@ _NODISCARD _FwdIt find(_ExPo&& _Exec, _FwdIt _First, const _FwdIt _Last, const _
namespace ranges {
// VARIABLE ranges::find
// clang-format off
template <class _Se, class _It>
concept _Sized_or_unreachable_sentinel_for = sized_sentinel_for<_Se, _It> || same_as<_Se, unreachable_sentinel_t>;

// concept-constrained for strict enforcement as it is used by several algorithms
template <input_iterator _It, sentinel_for<_It> _Se, class _Ty, class _Pj = identity>
requires indirect_binary_predicate<ranges::equal_to, projected<_It, _Pj>, const _Ty*>
_NODISCARD constexpr _It _Find_unchecked(_It _First, const _Se _Last, const _Ty& _Val, _Pj _Proj = {}) {
// clang-format on
if constexpr (_Memchr_in_find_is_safe<_It, _Ty> && sized_sentinel_for<_Se, _It> && same_as<_Pj, identity>) {
constexpr bool _Is_sized = sized_sentinel_for<_Se, _It>;
if constexpr (_Memchr_in_find_is_safe<_It,
_Ty> && _Sized_or_unreachable_sentinel_for<_Se, _It> && same_as<_Pj, identity>) {
if (!_STD is_constant_evaluated()) {
if (!_Within_limits(_First, _Val)) {
return _RANGES next(_STD move(_First), _Last);
if constexpr (_Is_sized) {
return _RANGES next(_STD move(_First), _Last);
} else {
_STL_ASSERT(false, "Tried to find a value in a range with unreachable sentinel"
" that cannot be represented by the range's value type");
}
}

size_t _Count;
if constexpr (_Is_sized) {
_Count = static_cast<size_t>(_Last - _First);
} else {
_Count = SIZE_MAX;
}
const auto _First_ptr = _STD to_address(_First);
const auto _Result = static_cast<remove_reference_t<_Iter_ref_t<_It>>*>(
_CSTD memchr(_First_ptr, static_cast<unsigned char>(_Val), static_cast<size_t>(_Last - _First)));
_CSTD memchr(_First_ptr, static_cast<unsigned char>(_Val), _Count));
if (_Result) {
if constexpr (is_pointer_v<_It>) {
return _Result;
Expand Down
11 changes: 9 additions & 2 deletions tests/std/tests/P0896R4_ranges_alg_equal/test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
#include <range_algorithm_support.hpp>

constexpr void smoke_test() {
using ranges::equal, ranges::equal_to;
using std::abort, std::array, std::pair, std::same_as;
using ranges::equal, ranges::equal_to, ranges::begin, ranges::end;
using std::abort, std::array, std::pair, std::same_as, std::unreachable_sentinel;

array<pair<int, int>, 3> const x = {{{0, 42}, {2, 42}, {4, 42}}};
array<pair<long, long>, 3> const y = {{{13, -1}, {13, 1}, {13, 3}}};
Expand Down Expand Up @@ -71,6 +71,13 @@ constexpr void smoke_test() {
arr2[1] = 7;
assert(!equal(arr1, arr2));
}
{
// Validate unreachable_sentinel cases
int arr1[3]{0, 2, 5};
int arr2[3]{0, 2, 5};
assert(!equal(begin(arr1), unreachable_sentinel, begin(arr2), end(arr2)));
assert(!equal(begin(arr1), end(arr1), begin(arr2), unreachable_sentinel));
}
}

int main() {
Expand Down
17 changes: 11 additions & 6 deletions tests/std/tests/P0896R4_ranges_alg_find/test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ struct instantiator {

template <ranges::input_range Read>
static constexpr void call() {
using ranges::find, ranges::iterator_t;
using ranges::find, ranges::iterator_t, ranges::begin, ranges::end;

for (const auto& [value, _] : haystack) {
{ // Validate range overload [found case]
Expand Down Expand Up @@ -48,15 +48,20 @@ struct instantiator {
STATIC_ASSERT(same_as<decltype(result), iterator_t<Read>>);
assert(result == wrapped_input.end());
}
{ // Validate memchr case [found case]
{ // Validate memchr case
char arr[5]{4, 8, 1, -15, 125};

// found case
auto result = find(arr, 1);
assert(*result == 1);
}
{ // Validate memchr case [not found case]
char arr[5]{4, 8, 1, -15, 125};
auto result = find(arr, 10);

// not found case
result = find(arr, 10);
assert(result == end(arr));

// unreachable_sentinel case
result = find(begin(arr), unreachable_sentinel, 1);
assert(*result == 1);
}
}
};
Expand Down
20 changes: 19 additions & 1 deletion tests/std/tests/P0896R4_ranges_alg_lexicographical_compare/test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ struct instantiator {

template <class In1, class In2>
static constexpr void call() {
using ranges::lexicographical_compare, ranges::less;
using ranges::lexicographical_compare, ranges::less, ranges::begin, ranges::end;

// Validate range overload
{
Expand Down Expand Up @@ -202,6 +202,24 @@ struct instantiator {
arr2[2] = 1;
assert(!lexicographical_compare(arr1, arr2));
}
{ // Validate memcmp + unreachable_sentinel cases
unsigned char arr1[3]{0, 1, 2};
unsigned char arr2[3]{0, 1, 3};

assert(lexicographical_compare(begin(arr1), end(arr1), begin(arr1), unreachable_sentinel));
assert(!lexicographical_compare(begin(arr1), unreachable_sentinel, begin(arr1), end(arr1)));

assert(lexicographical_compare(begin(arr1), unreachable_sentinel, begin(arr2), unreachable_sentinel));
assert(lexicographical_compare(begin(arr1), unreachable_sentinel, begin(arr2), end(arr2)));
assert(lexicographical_compare(begin(arr1), end(arr1), begin(arr2), unreachable_sentinel));
arr2[2] = 2;
assert(!lexicographical_compare(begin(arr1), unreachable_sentinel, begin(arr2), end(arr2)));
assert(lexicographical_compare(begin(arr1), end(arr1), begin(arr2), unreachable_sentinel));
arr2[2] = 1;
assert(!lexicographical_compare(begin(arr1), unreachable_sentinel, begin(arr2), unreachable_sentinel));
assert(!lexicographical_compare(begin(arr1), unreachable_sentinel, begin(arr2), end(arr2)));
assert(!lexicographical_compare(begin(arr1), end(arr1), begin(arr2), unreachable_sentinel));
}
}
};

Expand Down
2 changes: 1 addition & 1 deletion tests/std/tests/P0896R4_ranges_alg_uninitialized_copy_n/env.lst
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

RUNALL_INCLUDE ..\concepts_20_matrix.lst
RUNALL_INCLUDE ..\concepts_latest_matrix.lst
Loading

0 comments on commit e326ec1

Please sign in to comment.