diff --git a/libcxx/benchmarks/algorithms/pstl.stable_sort.bench.cpp b/libcxx/benchmarks/algorithms/pstl.stable_sort.bench.cpp index 9357b870bece6..72541f70640f5 100644 --- a/libcxx/benchmarks/algorithms/pstl.stable_sort.bench.cpp +++ b/libcxx/benchmarks/algorithms/pstl.stable_sort.bench.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include +#include #include "common.h" diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 33ee5b26bd621..bffc91a56c420 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -73,7 +73,6 @@ set(files __algorithm/pop_heap.h __algorithm/prev_permutation.h __algorithm/pstl.h - __algorithm/pstl_frontend_dispatch.h __algorithm/push_heap.h __algorithm/ranges_adjacent_find.h __algorithm/ranges_all_of.h @@ -570,11 +569,12 @@ set(files __numeric/transform_reduce.h __ostream/basic_ostream.h __ostream/print.h + __pstl/backend.h + __pstl/backend_fwd.h + __pstl/backends/default.h __pstl/backends/libdispatch.h __pstl/backends/serial.h __pstl/backends/std_thread.h - __pstl/configuration.h - __pstl/configuration_fwd.h __pstl/cpu_algos/any_of.h __pstl/cpu_algos/cpu_traits.h __pstl/cpu_algos/fill.h @@ -584,6 +584,8 @@ set(files __pstl/cpu_algos/stable_sort.h __pstl/cpu_algos/transform.h __pstl/cpu_algos/transform_reduce.h + __pstl/dispatch.h + __pstl/handle_exception.h __random/bernoulli_distribution.h __random/binomial_distribution.h __random/cauchy_distribution.h diff --git a/libcxx/include/__algorithm/pstl.h b/libcxx/include/__algorithm/pstl.h index 68b4e3e77ec6c..8f9b1b124c3b5 100644 --- a/libcxx/include/__algorithm/pstl.h +++ b/libcxx/include/__algorithm/pstl.h @@ -9,30 +9,18 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_H #define _LIBCPP___ALGORITHM_PSTL_H -#include <__algorithm/copy_n.h> -#include <__algorithm/count.h> -#include <__algorithm/equal.h> -#include <__algorithm/fill_n.h> -#include <__algorithm/for_each.h> -#include <__algorithm/for_each_n.h> -#include <__algorithm/pstl_frontend_dispatch.h> -#include <__atomic/atomic.h> #include <__config> -#include <__functional/identity.h> #include <__functional/operations.h> -#include <__iterator/concepts.h> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> -#include <__numeric/pstl.h> -#include <__pstl/configuration.h> +#include <__pstl/backend.h> +#include <__pstl/dispatch.h> +#include <__pstl/handle_exception.h> #include <__type_traits/enable_if.h> -#include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_execution_policy.h> -#include <__type_traits/is_trivially_copyable.h> #include <__type_traits/remove_cvref.h> -#include <__utility/empty.h> +#include <__utility/forward.h> #include <__utility/move.h> -#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -45,54 +33,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__remove_cvref_t<_ForwardIterator>> -__find_if(_ExecutionPolicy&&, _ForwardIterator&& __first, _ForwardIterator&& __last, _Predicate&& __pred) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_find_if<_RawPolicy>(_Backend{}, std::move(__first), std::move(__last), std::move(__pred)); -} - -template , - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI _ForwardIterator -find_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if requires ForwardIterators"); - auto __res = std::__find_if(__policy, std::move(__first), std::move(__last), std::move(__pred)); - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); -} - -template -void __pstl_any_of(); // declaration needed for the frontend dispatch below - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional __any_of( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Predicate&& __pred) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_any_of, _RawPolicy), - [&](_ForwardIterator __g_first, _ForwardIterator __g_last, _Predicate __g_pred) -> optional { - auto __res = std::__find_if(__policy, __g_first, __g_last, __g_pred); - if (!__res) - return nullopt; - return *__res != __g_last; - }, - std::move(__first), - std::move(__last), - std::move(__pred)); -} - template -void __pstl_all_of(); // declaration needed for the frontend dispatch below - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional -__all_of(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Pred&& __pred) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_all_of, _RawPolicy), - [&](_ForwardIterator __g_first, _ForwardIterator __g_last, _Pred __g_pred) -> optional { - auto __res = std::__any_of(__policy, __g_first, __g_last, [&](__iter_reference<_ForwardIterator> __value) { - return !__g_pred(__value); - }); - if (!__res) - return nullopt; - return !*__res; - }, - std::move(__first), - std::move(__last), - std::move(__pred)); + using _Implementation = __pstl::__dispatch<__pstl::__any_of, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred)); } template -void __pstl_none_of(); // declaration needed for the frontend dispatch below - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional -__none_of(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Pred&& __pred) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_none_of, _RawPolicy), - [&](_ForwardIterator __g_first, _ForwardIterator __g_last, _Pred __g_pred) -> optional { - auto __res = std::__any_of(__policy, __g_first, __g_last, __g_pred); - if (!__res) - return nullopt; - return !*__res; - }, - std::move(__first), - std::move(__last), - std::move(__pred)); + using _Implementation = __pstl::__dispatch<__pstl::__all_of, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred)); } template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__remove_cvref_t<_ForwardOutIterator>> -__transform(_ExecutionPolicy&&, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _ForwardOutIterator&& __result, - _UnaryOperation&& __op) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_transform<_RawPolicy>( - _Backend{}, std::move(__first), std::move(__last), std::move(__result), std::move(__op)); -} - -template , - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( - _ExecutionPolicy&& __policy, - _ForwardIterator __first, - _ForwardIterator __last, - _ForwardOutIterator __result, - _UnaryOperation __op) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform requires ForwardIterators"); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); - _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( - _ForwardOutIterator, decltype(__op(*__first)), "transform requires an OutputIterator"); - auto __res = std::__transform(__policy, std::move(__first), std::move(__last), std::move(__result), std::move(__op)); - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); -} - -template , - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI optional<__remove_cvref_t<_ForwardOutIterator>> -__transform(_ExecutionPolicy&&, - _ForwardIterator1&& __first1, - _ForwardIterator1&& __last1, - _ForwardIterator2&& __first2, - _ForwardOutIterator&& __result, - _BinaryOperation&& __op) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_transform<_RawPolicy>( - _Backend{}, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__result), std::move(__op)); -} - -template , - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( - _ExecutionPolicy&& __policy, - _ForwardIterator1 __first1, - _ForwardIterator1 __last1, - _ForwardIterator2 __first2, - _ForwardOutIterator __result, - _BinaryOperation __op) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform requires ForwardIterators"); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform requires ForwardIterators"); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); - _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( - _ForwardOutIterator, decltype(__op(*__first1, *__first2)), "transform requires an OutputIterator"); - auto __res = std::__transform( - __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__result), std::move(__op)); - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); -} - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> -__for_each(_ExecutionPolicy&&, _ForwardIterator&& __first, _ForwardIterator&& __last, _Function&& __func) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_for_each<_RawPolicy>(_Backend{}, std::move(__first), std::move(__last), std::move(__func)); -} - -template , - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI void -for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each requires ForwardIterators"); - if (!std::__for_each(__policy, std::move(__first), std::move(__last), std::move(__func))) - std::__throw_bad_alloc(); -} - -// TODO: Use the std::copy/move shenanigans to forward to std::memmove - -template -void __pstl_copy(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> -__copy(_ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _ForwardOutIterator&& __result) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_copy, _RawPolicy), - [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _ForwardOutIterator __g_result) { - return std::__transform(__policy, __g_first, __g_last, __g_result, __identity()); - }, - std::move(__first), - std::move(__last), - std::move(__result)); + using _Implementation = __pstl::__dispatch<__pstl::__none_of, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred)); } template -void __pstl_copy_n(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __copy_n( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _Size&& __n, _ForwardOutIterator&& __result) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_copy_n, _RawPolicy), - [&__policy]( - _ForwardIterator __g_first, _Size __g_n, _ForwardOutIterator __g_result) -> optional<_ForwardIterator> { - if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return std::__copy(__policy, std::move(__g_first), std::move(__g_first + __g_n), std::move(__g_result)); - } else { - (void)__policy; - return std::copy_n(__g_first, __g_n, __g_result); - } - }, - std::move(__first), - std::move(__n), - std::move(__result)); + using _Implementation = __pstl::__dispatch<__pstl::__copy, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__result)); } template -void __pstl_count_if(); // declaration needed for the frontend dispatch below - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>> __count_if( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Predicate&& __pred) noexcept { - using __diff_t = __iter_diff_t<_ForwardIterator>; - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_count_if, _RawPolicy), - [&](_ForwardIterator __g_first, _ForwardIterator __g_last, _Predicate __g_pred) -> optional<__diff_t> { - return std::__transform_reduce( - __policy, - std::move(__g_first), - std::move(__g_last), - __diff_t(), - std::plus{}, - [&](__iter_reference<_ForwardIterator> __element) -> bool { return __g_pred(__element); }); - }, - std::move(__first), - std::move(__last), - std::move(__pred)); + using _Implementation = __pstl::__dispatch<__pstl::__copy_n, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__n), std::move(__result)); } template count_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( _ForwardIterator, "count_if(first, last, pred) requires [first, last) to be ForwardIterators"); - auto __res = std::__count_if(__policy, std::move(__first), std::move(__last), std::move(__pred)); - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); -} - -template -void __pstl_count(); // declaration needed for the frontend dispatch below - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>> __count( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, const _Tp& __value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_count, _RawPolicy), - [&](_ForwardIterator __g_first, _ForwardIterator __g_last, const _Tp& __g_value) - -> optional<__iter_diff_t<_ForwardIterator>> { - return std::count_if(__policy, __g_first, __g_last, [&](__iter_reference<_ForwardIterator> __v) { - return __v == __g_value; - }); - }, - std::forward<_ForwardIterator>(__first), - std::forward<_ForwardIterator>(__last), - __value); + using _Implementation = __pstl::__dispatch<__pstl::__count_if, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred)); } template count(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( _ForwardIterator, "count(first, last, val) requires [first, last) to be ForwardIterators"); - auto __res = std::__count(__policy, std::move(__first), std::move(__last), __value); - if (!__res) - std::__throw_bad_alloc(); - return *__res; -} - -template -void __pstl_equal(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional -__equal(_ExecutionPolicy&& __policy, - _ForwardIterator1&& __first1, - _ForwardIterator1&& __last1, - _ForwardIterator2&& __first2, - _Pred&& __pred) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_equal, _RawPolicy), - [&__policy]( - _ForwardIterator1 __g_first1, _ForwardIterator1 __g_last1, _ForwardIterator2 __g_first2, _Pred __g_pred) { - return std::__transform_reduce( - __policy, - std::move(__g_first1), - std::move(__g_last1), - std::move(__g_first2), - true, - std::logical_and{}, - std::move(__g_pred)); - }, - std::move(__first1), - std::move(__last1), - std::move(__first2), - std::move(__pred)); + using _Implementation = __pstl::__dispatch<__pstl::__count, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __value); } template ; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__pred)); } template >, int> = 0> + class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, + enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI bool equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) { _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); - auto __res = std::__equal(__policy, std::move(__first1), std::move(__last1), std::move(__first2), std::equal_to{}); - if (!__res) - std::__throw_bad_alloc(); - return *__res; -} - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional -__equal(_ExecutionPolicy&& __policy, - _ForwardIterator1&& __first1, - _ForwardIterator1&& __last1, - _ForwardIterator2&& __first2, - _ForwardIterator2&& __last2, - _Pred&& __pred) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_equal, _RawPolicy), - [&__policy](_ForwardIterator1 __g_first1, - _ForwardIterator1 __g_last1, - _ForwardIterator2 __g_first2, - _ForwardIterator2 __g_last2, - _Pred __g_pred) -> optional { - if constexpr (__has_random_access_iterator_category<_ForwardIterator1>::value && - __has_random_access_iterator_category<_ForwardIterator2>::value) { - if (__g_last1 - __g_first1 != __g_last2 - __g_first2) - return false; - return std::__equal( - __policy, std::move(__g_first1), std::move(__g_last1), std::move(__g_first2), std::move(__g_pred)); - } else { - (void)__policy; // Avoid unused lambda capture warning - return std::equal( - std::move(__g_first1), - std::move(__g_last1), - std::move(__g_first2), - std::move(__g_last2), - std::move(__g_pred)); - } - }, + using _Implementation = __pstl::__dispatch<__pstl::__equal_3leg, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first1), std::move(__last1), std::move(__first2), - std::move(__last2), - std::move(__pred)); + equal_to{}); } template ; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + std::move(__pred)); } template >, int> = 0> + class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, + enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI bool equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __first1, @@ -606,33 +216,14 @@ equal(_ExecutionPolicy&& __policy, _ForwardIterator2 __last2) { _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); - auto __res = std::__equal( - __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::equal_to{}); - if (!__res) - std::__throw_bad_alloc(); - return *__res; -} - -template -void __pstl_fill(); // declaration needed for the frontend dispatch below - -template , - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI optional<__empty> __fill( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, const _Tp& __value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_fill, _RawPolicy), - [&](_ForwardIterator __g_first, _ForwardIterator __g_last, const _Tp& __g_value) { - return std::__for_each(__policy, __g_first, __g_last, [&](__iter_reference<_ForwardIterator> __element) { - __element = __g_value; - }); - }, - std::forward<_ForwardIterator>(__first), - std::forward<_ForwardIterator>(__last), - __value); + using _Implementation = __pstl::__dispatch<__pstl::__equal, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + equal_to{}); } template -void __pstl_fill_n(); // declaration needed for the frontend dispatch below - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> -__fill_n(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _SizeT&& __n, const _Tp& __value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_fill_n, _RawPolicy), - [&](_ForwardIterator __g_first, _SizeT __g_n, const _Tp& __g_value) { - if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) - std::fill(__policy, __g_first, __g_first + __g_n, __g_value); - else - std::fill_n(__g_first, __g_n, __g_value); - return optional<__empty>{__empty{}}; - }, - std::move(__first), - std::move(__n), - __value); + using _Implementation = __pstl::__dispatch<__pstl::__fill, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __value); } template , enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI void -fill_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _SizeT __n, const _Tp& __value) { +fill_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, const _Tp& __value) { _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "fill_n requires ForwardIterators"); - if (!std::__fill_n(__policy, std::move(__first), std::move(__n), __value)) - std::__throw_bad_alloc(); + using _Implementation = __pstl::__dispatch<__pstl::__fill_n, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__n), __value); } -template -void __pstl_find_if_not(); - template , enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__remove_cvref_t<_ForwardIterator>> __find_if_not( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Predicate&& __pred) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_find_if_not, _RawPolicy), - [&](_ForwardIterator&& __g_first, _ForwardIterator&& __g_last, _Predicate&& __g_pred) - -> optional<__remove_cvref_t<_ForwardIterator>> { - return std::__find_if( - __policy, __g_first, __g_last, [&](__iter_reference<__remove_cvref_t<_ForwardIterator>> __value) { - return !__g_pred(__value); - }); - }, - std::forward<_ForwardIterator>(__first), - std::forward<_ForwardIterator>(__last), - std::forward<_Predicate>(__pred)); +_LIBCPP_HIDE_FROM_ABI _ForwardIterator +find_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if requires ForwardIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__find_if, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred)); } template -void __pstl_find(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__remove_cvref_t<_ForwardIterator>> __find( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, const _Tp& __value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_find, _RawPolicy), - [&](_ForwardIterator __g_first, _ForwardIterator __g_last, const _Tp& __g_value) -> optional<_ForwardIterator> { - return std::find_if( - __policy, __g_first, __g_last, [&](__iter_reference<__remove_cvref_t<_ForwardIterator>> __element) { - return __element == __g_value; - }); - }, - std::forward<_ForwardIterator>(__first), - std::forward<_ForwardIterator>(__last), - __value); + using _Implementation = __pstl::__dispatch<__pstl::__find_if_not, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred)); } template ; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __value); } -template -void __pstl_for_each_n(); // declaration needed for the frontend dispatch below - template , enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> -__for_each_n(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _Size&& __size, _Function&& __func) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_for_each_n, _RawPolicy), - [&](_ForwardIterator __g_first, _Size __g_size, _Function __g_func) -> optional<__empty> { - if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - std::for_each(__policy, std::move(__g_first), __g_first + __g_size, std::move(__g_func)); - return __empty{}; - } else { - std::for_each_n(std::move(__g_first), __g_size, std::move(__g_func)); - return __empty{}; - } - }, - std::move(__first), - std::move(__size), - std::move(__func)); +_LIBCPP_HIDE_FROM_ABI void +for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each requires ForwardIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__for_each, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__func)); } template -void __pstl_generate(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __generate( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Generator&& __gen) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_generate, _RawPolicy), - [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _Generator __g_gen) { - return std::__for_each( - __policy, std::move(__g_first), std::move(__g_last), [&](__iter_reference<_ForwardIterator> __element) { - __element = __g_gen(); - }); - }, - std::move(__first), - std::move(__last), - std::move(__gen)); + using _Implementation = __pstl::__dispatch<__pstl::__for_each_n, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__size), std::move(__func)); } template -void __pstl_generate_n(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> -__generate_n(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _Size&& __n, _Generator&& __gen) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_generate_n, _RawPolicy), - [&__policy](_ForwardIterator __g_first, _Size __g_n, _Generator __g_gen) { - return std::__for_each_n( - __policy, std::move(__g_first), std::move(__g_n), [&](__iter_reference<_ForwardIterator> __element) { - __element = __g_gen(); - }); - }, - std::move(__first), - __n, - std::move(__gen)); + using _Implementation = __pstl::__dispatch<__pstl::__generate, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__gen)); } template -void __pstl_is_partitioned(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional __is_partitioned( - _ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Predicate&& __pred) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_is_partitioned, _RawPolicy), - [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _Predicate __g_pred) { - __g_first = std::find_if_not(__policy, __g_first, __g_last, __g_pred); - if (__g_first == __g_last) - return true; - ++__g_first; - return std::none_of(__policy, __g_first, __g_last, __g_pred); - }, - std::move(__first), - std::move(__last), - std::move(__pred)); + using _Implementation = __pstl::__dispatch<__pstl::__generate_n, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__n), std::move(__gen)); } template ; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred)); } template , enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> -__merge(_ExecutionPolicy&&, - _ForwardIterator1&& __first1, - _ForwardIterator1&& __last1, - _ForwardIterator2&& __first2, - _ForwardIterator2&& __last2, - _ForwardOutIterator&& __result, - _Comp&& __comp) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_merge<_RawPolicy>( - _Backend{}, - std::forward<_ForwardIterator1>(__first1), - std::forward<_ForwardIterator1>(__last1), - std::forward<_ForwardIterator2>(__first2), - std::forward<_ForwardIterator2>(__last2), - std::forward<_ForwardOutIterator>(__result), - std::forward<_Comp>(__comp)); -} - -template , - class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, - enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator merge(_ExecutionPolicy&& __policy, _ForwardIterator1 __first1, @@ -951,50 +373,48 @@ merge(_ExecutionPolicy&& __policy, _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardOutIterator __result, - _Comp __comp = {}) { + _Comp __comp) { _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "merge requires ForwardIterators"); _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "merge requires ForwardIterators"); _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first1), "merge requires an OutputIterator"); _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first2), "merge requires an OutputIterator"); - auto __res = std::__merge( - __policy, + using _Implementation = __pstl::__dispatch<__pstl::__merge, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::move(__result), std::move(__comp)); - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); } -// TODO: Use the std::copy/move shenanigans to forward to std::memmove -// Investigate whether we want to still forward to std::transform(policy) -// in that case for the execution::par part, or whether we actually want -// to run everything serially in that case. - -template -void __pstl_move(); - template , enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> -__move(_ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _ForwardOutIterator&& __result) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_move, _RawPolicy), - [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _ForwardOutIterator __g_result) { - return std::__transform(__policy, __g_first, __g_last, __g_result, [](auto&& __v) { return std::move(__v); }); - }, - std::move(__first), - std::move(__last), - std::move(__result)); +_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator +merge(_ExecutionPolicy&& __policy, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "merge requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "merge requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first1), "merge requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first2), "merge requires an OutputIterator"); + using _Implementation = __pstl::__dispatch<__pstl::__merge, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + std::move(__result), + less{}); } template -void __pstl_replace_if(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> -__replace_if(_ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _Pred&& __pred, - const _Tp& __new_value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_replace_if, _RawPolicy), - [&__policy]( - _ForwardIterator&& __g_first, _ForwardIterator&& __g_last, _Pred&& __g_pred, const _Tp& __g_new_value) { - std::for_each(__policy, __g_first, __g_last, [&](__iter_reference<_ForwardIterator> __element) { - if (__g_pred(__element)) - __element = __g_new_value; - }); - return optional<__empty>{__empty{}}; - }, - std::move(__first), - std::move(__last), - std::move(__pred), - __new_value); + using _Implementation = __pstl::__dispatch<__pstl::__move, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__result)); } template -void __pstl_replace(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> -__replace(_ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - const _Tp& __old_value, - const _Tp& __new_value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_replace, _RawPolicy), - [&__policy]( - _ForwardIterator __g_first, _ForwardIterator __g_last, const _Tp& __g_old_value, const _Tp& __g_new_value) { - return std::__replace_if( - __policy, - std::move(__g_first), - std::move(__g_last), - [&](__iter_reference<_ForwardIterator> __element) { return __element == __g_old_value; }, - __g_new_value); - }, - std::forward<_ForwardIterator>(__first), - std::forward<_ForwardIterator>(__last), - __old_value, - __new_value); + using _Implementation = __pstl::__dispatch<__pstl::__replace_if, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__pred), __new_value); } template -void __pstl_replace_copy_if(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __replace_copy_if( - _ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _ForwardOutIterator&& __result, - _Pred&& __pred, - const _Tp& __new_value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_replace_copy_if, _RawPolicy), - [&__policy](_ForwardIterator __g_first, - _ForwardIterator __g_last, - _ForwardOutIterator __g_result, - _Pred __g_pred, - const _Tp& __g_new_value) -> optional<__empty> { - if (!std::__transform( - __policy, __g_first, __g_last, __g_result, [&](__iter_reference<_ForwardIterator> __element) { - return __g_pred(__element) ? __g_new_value : __element; - })) - return nullopt; - return __empty{}; - }, - std::move(__first), - std::move(__last), - std::move(__result), - std::move(__pred), - __new_value); + using _Implementation = __pstl::__dispatch<__pstl::__replace, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), __old_value, __new_value); } template -void __pstl_replace_copy(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __replace_copy( - _ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _ForwardOutIterator&& __result, - const _Tp& __old_value, - const _Tp& __new_value) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_replace_copy, _RawPolicy), - [&__policy](_ForwardIterator __g_first, - _ForwardIterator __g_last, - _ForwardOutIterator __g_result, - const _Tp& __g_old_value, - const _Tp& __g_new_value) { - return std::__replace_copy_if( - __policy, - std::move(__g_first), - std::move(__g_last), - std::move(__g_result), - [&](__iter_reference<_ForwardIterator> __element) { return __element == __g_old_value; }, - __g_new_value); - }, + using _Implementation = __pstl::__dispatch<__pstl::__replace_copy_if, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__result), - __old_value, + std::move(__pred), __new_value); } @@ -1228,41 +515,14 @@ _LIBCPP_HIDE_FROM_ABI void replace_copy( _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( _ForwardOutIterator, decltype(*__first), "replace_copy requires an OutputIterator"); _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, const _Tp&, "replace_copy requires an OutputIterator"); - if (!std::__replace_copy( - __policy, std::move(__first), std::move(__last), std::move(__result), __old_value, __new_value)) - std::__throw_bad_alloc(); -} - -template -void __pstl_rotate_copy(); - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> -__rotate_copy(_ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __middle, - _ForwardIterator&& __last, - _ForwardOutIterator&& __result) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_rotate_copy, _RawPolicy), - [&__policy](_ForwardIterator __g_first, - _ForwardIterator __g_middle, - _ForwardIterator __g_last, - _ForwardOutIterator __g_result) -> optional<_ForwardOutIterator> { - auto __result_mid = - std::__copy(__policy, _ForwardIterator(__g_middle), std::move(__g_last), std::move(__g_result)); - if (!__result_mid) - return nullopt; - return std::__copy(__policy, std::move(__g_first), std::move(__g_middle), *std::move(__result_mid)); - }, + using _Implementation = __pstl::__dispatch<__pstl::__replace_copy, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), - std::move(__middle), std::move(__last), - std::move(__result)); + std::move(__result), + __old_value, + __new_value); } template ; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first), + std::move(__middle), + std::move(__last), + std::move(__result)); } template , + class _Comp, class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __stable_sort( - _ExecutionPolicy&&, _RandomAccessIterator&& __first, _RandomAccessIterator&& __last, _Comp&& __comp = {}) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_stable_sort<_RawPolicy>(_Backend{}, std::move(__first), std::move(__last), std::move(__comp)); +_LIBCPP_HIDE_FROM_ABI void +sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__sort, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__comp)); } template , class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI void stable_sort( - _ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp = {}) { - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "stable_sort requires RandomAccessIterators"); - if (!std::__stable_sort(__policy, std::move(__first), std::move(__last), std::move(__comp))) - std::__throw_bad_alloc(); +_LIBCPP_HIDE_FROM_ABI void +sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__sort, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), less{}); } -template -void __pstl_sort(); - template , enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> -__sort(_ExecutionPolicy&& __policy, - _RandomAccessIterator&& __first, - _RandomAccessIterator&& __last, - _Comp&& __comp) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_sort, _RawPolicy), - [&__policy](_RandomAccessIterator __g_first, _RandomAccessIterator __g_last, _Comp __g_comp) { - std::stable_sort(__policy, std::move(__g_first), std::move(__g_last), std::move(__g_comp)); - return optional<__empty>{__empty{}}; - }, - std::forward<_RandomAccessIterator>(__first), - std::forward<_RandomAccessIterator>(__last), - std::forward<_Comp>(__comp)); +_LIBCPP_HIDE_FROM_ABI void +stable_sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "stable_sort requires RandomAccessIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__stable_sort, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__comp)); } template , enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI void -sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); - if (!std::__sort(__policy, std::move(__first), std::move(__last), std::move(__comp))) - std::__throw_bad_alloc(); +stable_sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "stable_sort requires RandomAccessIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__stable_sort, __pstl::__current_configuration, _RawPolicy>; + __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), less{}); } template , enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI void -sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last) { - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); - if (!std::__sort(__policy, std::move(__first), std::move(__last), less{})) - std::__throw_bad_alloc(); +_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( + _ExecutionPolicy&& __policy, + _ForwardIterator __first, + _ForwardIterator __last, + _ForwardOutIterator __result, + _UnaryOperation __op) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(__op(*__first)), "transform requires an OutputIterator"); + using _Implementation = __pstl::__dispatch<__pstl::__transform, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first), + std::move(__last), + std::move(__result), + std::move(__op)); +} + +template , + enable_if_t, int> = 0> +_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( + _ExecutionPolicy&& __policy, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardOutIterator __result, + _BinaryOperation __op) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(__op(*__first1, *__first2)), "transform requires an OutputIterator"); + using _Implementation = __pstl::__dispatch<__pstl::__transform_binary, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__result), + std::move(__op)); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/pstl_frontend_dispatch.h b/libcxx/include/__algorithm/pstl_frontend_dispatch.h deleted file mode 100644 index 6fa1107491154..0000000000000 --- a/libcxx/include/__algorithm/pstl_frontend_dispatch.h +++ /dev/null @@ -1,44 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___ALGORITHM_PSTL_FRONTEND_DISPATCH -#define _LIBCPP___ALGORITHM_PSTL_FRONTEND_DISPATCH - -#include <__config> -#include <__type_traits/is_callable.h> -#include <__utility/forward.h> - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -#endif - -#if _LIBCPP_STD_VER >= 17 - -_LIBCPP_BEGIN_NAMESPACE_STD - -# define _LIBCPP_PSTL_CUSTOMIZATION_POINT(name, policy) \ - [](auto&&... __args) -> decltype(std::name( \ - typename __select_backend::type{}, std::forward(__args)...)) { \ - return std::name(typename __select_backend::type{}, std::forward(__args)...); \ - } - -template -_LIBCPP_HIDE_FROM_ABI decltype(auto) -__pstl_frontend_dispatch(_SpecializedImpl __specialized_impl, _GenericImpl __generic_impl, _Args&&... __args) { - if constexpr (__is_callable<_SpecializedImpl, _Args...>::value) { - return __specialized_impl(std::forward<_Args>(__args)...); - } else { - return __generic_impl(std::forward<_Args>(__args)...); - } -} - -_LIBCPP_END_NAMESPACE_STD - -#endif // _LIBCPP_STD_VER >= 17 - -#endif // _LIBCPP___ALGORITHM_PSTL_FRONTEND_DISPATCH diff --git a/libcxx/include/__numeric/pstl.h b/libcxx/include/__numeric/pstl.h index 05559b4d3f3c8..3238b59794cbf 100644 --- a/libcxx/include/__numeric/pstl.h +++ b/libcxx/include/__numeric/pstl.h @@ -9,17 +9,19 @@ #ifndef _LIBCPP___NUMERIC_PSTL_H #define _LIBCPP___NUMERIC_PSTL_H -#include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/identity.h> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> -#include <__numeric/transform_reduce.h> -#include <__pstl/configuration.h> +#include <__pstl/backend.h> +#include <__pstl/dispatch.h> +#include <__pstl/handle_exception.h> +#include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> +#include <__type_traits/remove_cvref.h> +#include <__utility/forward.h> #include <__utility/move.h> -#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -33,30 +35,50 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template , enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI optional<_Tp> __transform_reduce( - _ExecutionPolicy&&, - _ForwardIterator1&& __first1, - _ForwardIterator1&& __last1, - _ForwardIterator2&& __first2, - _Tp&& __init, - _BinaryOperation1&& __reduce, - _BinaryOperation2&& __transform) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_transform_reduce<_RawPolicy>( - _Backend{}, - std::move(__first1), - std::move(__last1), - std::move(__first2), +_LIBCPP_HIDE_FROM_ABI _Tp reduce( + _ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp __init, _BinaryOperation __op) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__reduce, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first), + std::move(__last), std::move(__init), - std::move(__reduce), - std::move(__transform)); + std::move(__op)); +} + +template , + enable_if_t, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp __init) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__reduce, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__init), plus{}); +} + +template , + enable_if_t, int> = 0> +_LIBCPP_HIDE_FROM_ABI __iter_value_type<_ForwardIterator> +reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); + using _Implementation = __pstl::__dispatch<__pstl::__reduce, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first), + std::move(__last), + __iter_value_type<_ForwardIterator>(), + plus{}); } template ; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first1), std::move(__last1), std::move(__first2), std::move(__init), std::move(__reduce), std::move(__transform)); - - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); } // This overload doesn't get a customization point because it's trivial to detect (through e.g. @@ -97,7 +117,8 @@ template >, int> = 0> + class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, + enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _ExecutionPolicy&& __policy, _ForwardIterator1 __first1, @@ -106,31 +127,16 @@ _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _Tp __init) { _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_reduce requires ForwardIterators"); _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_reduce requires ForwardIterators"); - return std::transform_reduce(__policy, __first1, __last1, __first2, __init, plus{}, multiplies{}); -} - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__remove_cvref_t<_Tp>> __transform_reduce( - _ExecutionPolicy&&, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _Tp&& __init, - _BinaryOperation&& __reduce, - _UnaryOperation&& __transform) noexcept { - using _Backend = typename __select_backend<_RawPolicy>::type; - return std::__pstl_transform_reduce<_RawPolicy>( - _Backend{}, - std::move(__first), - std::move(__last), + using _Implementation = + __pstl::__dispatch<__pstl::__transform_reduce_binary, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), + std::move(__first1), + std::move(__last1), + std::move(__first2), std::move(__init), - std::move(__reduce), - std::move(__transform)); + plus{}, + multiplies{}); } template -void __pstl_reduce(); - -template , - class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_Tp> -__reduce(_ExecutionPolicy&& __policy, - _ForwardIterator&& __first, - _ForwardIterator&& __last, - _Tp&& __init, - _BinaryOperation&& __op = {}) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_reduce, _RawPolicy), - [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _Tp __g_init, _BinaryOperation __g_op) { - return std::__transform_reduce( - __policy, std::move(__g_first), std::move(__g_last), std::move(__g_init), std::move(__g_op), __identity{}); - }, + using _Implementation = __pstl::__dispatch<__pstl::__transform_reduce, __pstl::__current_configuration, _RawPolicy>; + return __pstl::__handle_exception<_Implementation>( + std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), std::move(__init), - std::move(__op)); -} - -template , - class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI _Tp -reduce(_ExecutionPolicy&& __policy, - _ForwardIterator __first, - _ForwardIterator __last, - _Tp __init, - _BinaryOperation __op = {}) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); - auto __res = std::__reduce(__policy, std::move(__first), std::move(__last), std::move(__init), std::move(__op)); - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); -} - -template , - enable_if_t, int> = 0> -[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_value_type<_ForwardIterator>> -__reduce(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last) noexcept { - return std::__pstl_frontend_dispatch( - _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_reduce, _RawPolicy), - [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last) { - return std::__reduce( - __policy, std::move(__g_first), std::move(__g_last), __iter_value_type<_ForwardIterator>()); - }, - std::move(__first), - std::move(__last)); -} - -template , - enable_if_t, int> = 0> -_LIBCPP_HIDE_FROM_ABI __iter_value_type<_ForwardIterator> -reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); - auto __res = std::__reduce(__policy, std::move(__first), std::move(__last)); - if (!__res) - std::__throw_bad_alloc(); - return *std::move(__res); + std::move(__reduce), + std::move(__transform)); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/configuration.h b/libcxx/include/__pstl/backend.h similarity index 70% rename from libcxx/include/__pstl/configuration.h rename to libcxx/include/__pstl/backend.h index d32bd21df1f9e..86d9f28c77fa8 100644 --- a/libcxx/include/__pstl/configuration.h +++ b/libcxx/include/__pstl/backend.h @@ -6,22 +6,30 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___PSTL_CONFIGURATION_H -#define _LIBCPP___PSTL_CONFIGURATION_H +#ifndef _LIBCPP___PSTL_BACKEND_H +#define _LIBCPP___PSTL_BACKEND_H #include <__config> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + #if defined(_LIBCPP_PSTL_BACKEND_SERIAL) +# include <__pstl/backends/default.h> # include <__pstl/backends/serial.h> #elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) +# include <__pstl/backends/default.h> # include <__pstl/backends/std_thread.h> #elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) +# include <__pstl/backends/default.h> # include <__pstl/backends/libdispatch.h> #endif -#endif // _LIBCPP___PSTL_CONFIGURATION_H +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___PSTL_BACKEND_H diff --git a/libcxx/include/__pstl/backend_fwd.h b/libcxx/include/__pstl/backend_fwd.h new file mode 100644 index 0000000000000..32c5da576fb3c --- /dev/null +++ b/libcxx/include/__pstl/backend_fwd.h @@ -0,0 +1,301 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___PSTL_BACKEND_FWD_H +#define _LIBCPP___PSTL_BACKEND_FWD_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +// +// This header declares available PSTL backends and the functions that must be implemented in order for the +// PSTL algorithms to be provided. +// +// Backends often do not implement the full set of functions themselves -- a configuration of the PSTL is +// usually a set of backends "stacked" together which each implement some algorithms under some execution +// policies. It is only necessary for the "stack" of backends to implement all algorithms under all execution +// policies, but a single backend is not required to implement everything on its own. +// +// The signatures used by each backend function are documented below. +// +// Exception handling +// ================== +// +// PSTL backends are expected to report errors (i.e. failure to allocate) by returning a disengaged `optional` from +// their implementation. Exceptions shouldn't be used to report an internal failure-to-allocate, since all exceptions +// are turned into a program termination at the front-end level. When a backend returns a disengaged `optional` to the +// frontend, the frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to +// the user. +// + +_LIBCPP_BEGIN_NAMESPACE_STD +namespace __pstl { + +template +struct __backend_configuration; + +struct __default_backend_tag; +struct __libdispatch_backend_tag; +struct __serial_backend_tag; +struct __std_thread_backend_tag; + +#if defined(_LIBCPP_PSTL_BACKEND_SERIAL) +using __current_configuration = __backend_configuration<__serial_backend_tag, __default_backend_tag>; +#elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) +using __current_configuration = __backend_configuration<__std_thread_backend_tag, __default_backend_tag>; +#elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) +using __current_configuration = __backend_configuration<__libdispatch_backend_tag, __default_backend_tag>; +#else + +// ...New vendors can add parallel backends here... + +# error "Invalid PSTL backend configuration" +#endif + +template +struct __find_if; +// template +// optional<_ForwardIterator> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept; + +template +struct __find_if_not; +// template +// optional<_ForwardIterator> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept; + +template +struct __find; +// template +// optional<_ForwardIterator> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) const noexcept; + +template +struct __any_of; +// template +// optional +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept; + +template +struct __all_of; +// template +// optional +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept; + +template +struct __none_of; +// template +// optional +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept; + +template +struct __is_partitioned; +// template +// optional +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept; + +template +struct __for_each; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __func) const noexcept; + +template +struct __for_each_n; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _Size __size, _Function __func) const noexcept; + +template +struct __fill; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept; + +template +struct __fill_n; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _Size __n, _Tp const& __value) const noexcept; + +template +struct __replace; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _Tp const& __old, _Tp const& __new) const noexcept; + +template +struct __replace_if; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _Predicate __pred, _Tp const& __new_value) const noexcept; + +template +struct __generate; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __gen) const noexcept; + +template +struct __generate_n; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _Size __n, _Generator __gen) const noexcept; + +template +struct __merge; +// template +// optional<_ForwardOutIterator> +// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, +// _ForwardIterator2 __first2, _ForwardIterator2 __last2, +// _ForwardOutIterator __result, _Comp __comp) const noexcept; + +template +struct __stable_sort; +// template +// optional<__empty> +// operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) const noexcept; + +template +struct __sort; +// template +// optional<__empty> +// operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) const noexcept; + +template +struct __transform; +// template +// optional<_ForwardOutIterator> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _ForwardOutIterator __result, +// _UnaryOperation __op) const noexcept; + +template +struct __transform_binary; +// template +// optional<_ForwardOutIterator> +// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, +// _ForwardIterator2 __first2, +// _ForwardOutIterator __result, +// _BinaryOperation __op) const noexcept; + +template +struct __replace_copy_if; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _ForwardOutIterator __out_it, +// _Predicate __pred, +// _Tp const& __new_value) const noexcept; + +template +struct __replace_copy; +// template +// optional<__empty> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _ForwardOutIterator __out_it, +// _Tp const& __old_value, +// _Tp const& __new_value) const noexcept; + +template +struct __move; +// template +// optional<_ForwardOutIterator> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _ForwardOutIterator __out_it) const noexcept; + +template +struct __copy; +// template +// optional<_ForwardOutIterator> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _ForwardOutIterator __out_it) const noexcept; + +template +struct __copy_n; +// template +// optional<_ForwardOutIterator> +// operator()(_Policy&&, _ForwardIterator __first, _Size __n, _ForwardOutIterator __out_it) const noexcept; + +template +struct __rotate_copy; +// template +// optional<_ForwardOutIterator> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, +// _ForwardOutIterator __out_it) const noexcept; + +template +struct __transform_reduce; +// template +// optional<_Tp> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _Tp __init, +// _BinaryOperation __reduce, +// _UnaryOperation __transform) const noexcept; + +template +struct __transform_reduce_binary; +// template +// optional<_Tp> operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, +// _ForwardIterator2 __first2, +// _Tp __init, +// _BinaryOperation1 __reduce, +// _BinaryOperation2 __transform) const noexcept; + +template +struct __count_if; +// template +// optional<__iter_diff_t<_ForwardIterator>> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept; + +template +struct __count; +// template +// optional<__iter_diff_t<_ForwardIterator>> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept; + +template +struct __equal_3leg; +// template +// optional +// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, +// _ForwardIterator2 __first2, +// _Predicate __pred) const noexcept; + +template +struct __equal; +// template +// optional +// operator()(_Policy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, +// _ForwardIterator2 __first2, _ForwardIterator2 __last2, +// _Predicate __pred) const noexcept; + +template +struct __reduce; +// template +// optional<_Tp> +// operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, +// _Tp __init, _BinaryOperation __op) const noexcept; + +} // namespace __pstl +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___PSTL_BACKEND_FWD_H diff --git a/libcxx/include/__pstl/backends/default.h b/libcxx/include/__pstl/backends/default.h new file mode 100644 index 0000000000000..8372caf0b0a38 --- /dev/null +++ b/libcxx/include/__pstl/backends/default.h @@ -0,0 +1,507 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___PSTL_BACKENDS_DEFAULT_H +#define _LIBCPP___PSTL_BACKENDS_DEFAULT_H + +#include <__algorithm/copy_n.h> +#include <__algorithm/equal.h> +#include <__algorithm/fill_n.h> +#include <__algorithm/for_each_n.h> +#include <__config> +#include <__functional/identity.h> +#include <__functional/not_fn.h> +#include <__functional/operations.h> +#include <__iterator/concepts.h> +#include <__iterator/iterator_traits.h> +#include <__pstl/backend_fwd.h> +#include <__pstl/dispatch.h> +#include <__utility/empty.h> +#include <__utility/forward.h> +#include <__utility/move.h> +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD +namespace __pstl { + +// +// This file provides an incomplete PSTL backend that implements all of the PSTL algorithms +// based on a smaller set of basis operations. +// +// It is intended as a building block for other PSTL backends that implement some operations more +// efficiently but may not want to define the full set of PSTL algorithms. +// +// This backend implements all the PSTL algorithms based on the following basis operations: +// +// find_if family +// -------------- +// - find +// - find_if_not +// - any_of +// - all_of +// - none_of +// - is_partitioned +// +// for_each family +// --------------- +// - for_each_n +// - fill +// - fill_n +// - replace +// - replace_if +// - generate +// - generate_n +// +// merge family +// ------------ +// No other algorithms based on merge +// +// stable_sort family +// ------------------ +// - sort +// +// transform_reduce and transform_reduce_binary family +// --------------------------------------------------- +// - count_if +// - count +// - equal(3 legs) +// - equal +// - reduce +// +// transform and transform_binary family +// ------------------------------------- +// - replace_copy_if +// - replace_copy +// - move +// - copy +// - copy_n +// - rotate_copy +// + +////////////////////////////////////////////////////////////// +// find_if family +////////////////////////////////////////////////////////////// +template +struct __find<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) const noexcept { + using _FindIf = __dispatch<__find_if, __current_configuration, _ExecutionPolicy>; + return _FindIf()( + __policy, std::move(__first), std::move(__last), [&](__iter_reference<_ForwardIterator> __element) { + return __element == __value; + }); + } +}; + +template +struct __find_if_not<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept { + using _FindIf = __dispatch<__find_if, __current_configuration, _ExecutionPolicy>; + return _FindIf()(__policy, __first, __last, std::not_fn(std::forward<_Pred>(__pred))); + } +}; + +template +struct __any_of<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept { + using _FindIf = __dispatch<__find_if, __current_configuration, _ExecutionPolicy>; + auto __res = _FindIf()(__policy, __first, __last, std::forward<_Pred>(__pred)); + if (!__res) + return nullopt; + return *__res != __last; + } +}; + +template +struct __all_of<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept { + using _AnyOf = __dispatch<__any_of, __current_configuration, _ExecutionPolicy>; + auto __res = _AnyOf()(__policy, __first, __last, [&](__iter_reference<_ForwardIterator> __value) { + return !__pred(__value); + }); + if (!__res) + return nullopt; + return !*__res; + } +}; + +template +struct __none_of<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept { + using _AnyOf = __dispatch<__any_of, __current_configuration, _ExecutionPolicy>; + auto __res = _AnyOf()(__policy, __first, __last, std::forward<_Pred>(__pred)); + if (!__res) + return nullopt; + return !*__res; + } +}; + +template +struct __is_partitioned<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept { + using _FindIfNot = __dispatch<__find_if_not, __current_configuration, _ExecutionPolicy>; + auto __maybe_first = _FindIfNot()(__policy, std::move(__first), std::move(__last), __pred); + if (__maybe_first == nullopt) + return nullopt; + + __first = *__maybe_first; + if (__first == __last) + return true; + ++__first; + using _NoneOf = __dispatch<__none_of, __current_configuration, _ExecutionPolicy>; + return _NoneOf()(__policy, std::move(__first), std::move(__last), __pred); + } +}; + +////////////////////////////////////////////////////////////// +// for_each family +////////////////////////////////////////////////////////////// +template +struct __for_each_n<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) const noexcept { + if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>; + _ForwardIterator __last = __first + __size; + return _ForEach()(__policy, std::move(__first), std::move(__last), std::move(__func)); + } else { + // Otherwise, use the serial algorithm to avoid doing two passes over the input + std::for_each_n(std::move(__first), __size, std::move(__func)); + return __empty{}; + } + } +}; + +template +struct __fill<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept { + using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) { __element = __value; }); + } +}; + +template +struct __fill_n<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _Size __n, _Tp const& __value) const noexcept { + if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + using _Fill = __dispatch<__fill, __current_configuration, _ExecutionPolicy>; + _ForwardIterator __last = __first + __n; + return _Fill()(__policy, std::move(__first), std::move(__last), __value); + } else { + // Otherwise, use the serial algorithm to avoid doing two passes over the input + std::fill_n(std::move(__first), __n, __value); + return optional<__empty>{__empty{}}; + } + } +}; + +template +struct __replace<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __old, _Tp const& __new) + const noexcept { + using _ReplaceIf = __dispatch<__replace_if, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + return _ReplaceIf()( + __policy, std::move(__first), std::move(__last), [&](_Ref __element) { return __element == __old; }, __new); + } +}; + +template +struct __replace_if<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> operator()( + _Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred, _Tp const& __new_value) + const noexcept { + using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) { + if (__pred(__element)) + __element = __new_value; + }); + } +}; + +template +struct __generate<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Generator&& __gen) const noexcept { + using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) { __element = __gen(); }); + } +}; + +template +struct __generate_n<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _Size __n, _Generator&& __gen) const noexcept { + using _ForEachN = __dispatch<__for_each_n, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + return _ForEachN()(__policy, std::move(__first), __n, [&](_Ref __element) { __element = __gen(); }); + } +}; + +////////////////////////////////////////////////////////////// +// stable_sort family +////////////////////////////////////////////////////////////// +template +struct __sort<__default_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<__empty> operator()( + _Policy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp&& __comp) const noexcept { + using _StableSort = __dispatch<__stable_sort, __current_configuration, _ExecutionPolicy>; + return _StableSort()(__policy, std::move(__first), std::move(__last), std::forward<_Comp>(__comp)); + } +}; + +////////////////////////////////////////////////////////////// +// transform_reduce family +////////////////////////////////////////////////////////////// +template +struct __count_if<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>> operator()( + _Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate&& __pred) const noexcept { + using _TransformReduce = __dispatch<__transform_reduce, __current_configuration, _ExecutionPolicy>; + using _DiffT = __iter_diff_t<_ForwardIterator>; + using _Ref = __iter_reference<_ForwardIterator>; + return _TransformReduce()( + __policy, std::move(__first), std::move(__last), _DiffT{}, std::plus{}, [&](_Ref __element) -> _DiffT { + return __pred(__element) ? _DiffT(1) : _DiffT(0); + }); + } +}; + +template +struct __count<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept { + using _CountIf = __dispatch<__count_if, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + return _CountIf()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) -> bool { + return __element == __value; + }); + } +}; + +template +struct __equal_3leg<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional + operator()(_Policy&& __policy, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _Predicate&& __pred) const noexcept { + using _TransformReduce = __dispatch<__transform_reduce_binary, __current_configuration, _ExecutionPolicy>; + return _TransformReduce()( + __policy, + std::move(__first1), + std::move(__last1), + std::move(__first2), + true, + std::logical_and{}, + std::forward<_Predicate>(__pred)); + } +}; + +template +struct __equal<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional + operator()(_Policy&& __policy, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _Predicate&& __pred) const noexcept { + if constexpr (__has_random_access_iterator_category<_ForwardIterator1>::value && + __has_random_access_iterator_category<_ForwardIterator2>::value) { + if (__last1 - __first1 != __last2 - __first2) + return false; + // Fall back to the 3 legged algorithm + using _Equal3Leg = __dispatch<__equal_3leg, __current_configuration, _ExecutionPolicy>; + return _Equal3Leg()( + __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::forward<_Predicate>(__pred)); + } else { + // If we don't have random access, fall back to the serial algorithm cause we can't do much + return std::equal( + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + std::forward<_Predicate>(__pred)); + } + } +}; + +template +struct __reduce<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_Tp> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp __init, _BinaryOperation&& __op) + const noexcept { + using _TransformReduce = __dispatch<__transform_reduce, __current_configuration, _ExecutionPolicy>; + return _TransformReduce()( + __policy, + std::move(__first), + std::move(__last), + std::move(__init), + std::forward<_BinaryOperation>(__op), + __identity{}); + } +}; + +////////////////////////////////////////////////////////////// +// transform family +////////////////////////////////////////////////////////////// +template +struct __replace_copy_if<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, + _ForwardIterator __first, + _ForwardIterator __last, + _ForwardOutIterator __out_it, + _Pred&& __pred, + _Tp const& __new_value) const noexcept { + using _Transform = __dispatch<__transform, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + auto __res = + _Transform()(__policy, std::move(__first), std::move(__last), std::move(__out_it), [&](_Ref __element) { + return __pred(__element) ? __new_value : __element; + }); + if (__res == nullopt) + return nullopt; + return __empty{}; + } +}; + +template +struct __replace_copy<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, + _ForwardIterator __first, + _ForwardIterator __last, + _ForwardOutIterator __out_it, + _Tp const& __old_value, + _Tp const& __new_value) const noexcept { + using _ReplaceCopyIf = __dispatch<__replace_copy_if, __current_configuration, _ExecutionPolicy>; + using _Ref = __iter_reference<_ForwardIterator>; + return _ReplaceCopyIf()( + __policy, + std::move(__first), + std::move(__last), + std::move(__out_it), + [&](_Ref __element) { return __element == __old_value; }, + __new_value); + } +}; + +// TODO: Use the std::copy/move shenanigans to forward to std::memmove +// Investigate whether we want to still forward to std::transform(policy) +// in that case for the execution::par part, or whether we actually want +// to run everything serially in that case. +template +struct __move<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __out_it) + const noexcept { + using _Transform = __dispatch<__transform, __current_configuration, _ExecutionPolicy>; + return _Transform()(__policy, std::move(__first), std::move(__last), std::move(__out_it), [&](auto&& __element) { + return std::move(__element); + }); + } +}; + +// TODO: Use the std::copy/move shenanigans to forward to std::memmove +template +struct __copy<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __out_it) + const noexcept { + using _Transform = __dispatch<__transform, __current_configuration, _ExecutionPolicy>; + return _Transform()(__policy, std::move(__first), std::move(__last), std::move(__out_it), __identity()); + } +}; + +template +struct __copy_n<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> + operator()(_Policy&& __policy, _ForwardIterator __first, _Size __n, _ForwardOutIterator __out_it) const noexcept { + if constexpr (__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + using _Copy = __dispatch<__copy, __current_configuration, _ExecutionPolicy>; + _ForwardIterator __last = __first + __n; + return _Copy()(__policy, std::move(__first), std::move(__last), std::move(__out_it)); + } else { + // Otherwise, use the serial algorithm to avoid doing two passes over the input + return std::copy_n(std::move(__first), __n, std::move(__out_it)); + } + } +}; + +template +struct __rotate_copy<__default_backend_tag, _ExecutionPolicy> { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> + operator()(_Policy&& __policy, + _ForwardIterator __first, + _ForwardIterator __middle, + _ForwardIterator __last, + _ForwardOutIterator __out_it) const noexcept { + using _Copy = __dispatch<__copy, __current_configuration, _ExecutionPolicy>; + auto __result_mid = _Copy()(__policy, __middle, std::move(__last), std::move(__out_it)); + if (__result_mid == nullopt) + return nullopt; + return _Copy()(__policy, std::move(__first), std::move(__middle), *std::move(__result_mid)); + } +}; + +} // namespace __pstl +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___PSTL_BACKENDS_DEFAULT_H diff --git a/libcxx/include/__pstl/backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h index af1da80dc133e..414524223cfc0 100644 --- a/libcxx/include/__pstl/backends/libdispatch.h +++ b/libcxx/include/__pstl/backends/libdispatch.h @@ -23,8 +23,16 @@ #include <__memory/construct_at.h> #include <__memory/unique_ptr.h> #include <__numeric/reduce.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> +#include <__pstl/cpu_algos/any_of.h> #include <__pstl/cpu_algos/cpu_traits.h> +#include <__pstl/cpu_algos/fill.h> +#include <__pstl/cpu_algos/find_if.h> +#include <__pstl/cpu_algos/for_each.h> +#include <__pstl/cpu_algos/merge.h> +#include <__pstl/cpu_algos/stable_sort.h> +#include <__pstl/cpu_algos/transform.h> +#include <__pstl/cpu_algos/transform_reduce.h> #include <__utility/empty.h> #include <__utility/exception_guard.h> #include <__utility/move.h> @@ -341,6 +349,48 @@ struct __cpu_traits<__libdispatch_backend_tag> { static constexpr size_t __lane_size = 64; }; +// Mandatory implementations of the computational basis +template +struct __find_if<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_find_if<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __for_each<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_for_each<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __merge<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_merge<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __stable_sort<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_stable_sort<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform_binary<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform_binary<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform_reduce<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform_reduce<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform_reduce_binary<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform_reduce_binary<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +// Not mandatory, but better optimized +template +struct __any_of<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_any_of<__libdispatch_backend_tag, _ExecutionPolicy> {}; + +template +struct __fill<__libdispatch_backend_tag, _ExecutionPolicy> + : __cpu_parallel_fill<__libdispatch_backend_tag, _ExecutionPolicy> {}; + } // namespace __pstl _LIBCPP_END_NAMESPACE_STD @@ -348,14 +398,4 @@ _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS -// Implement PSTL algorithms based on the __cpu_traits specialized above -#include <__pstl/cpu_algos/any_of.h> -#include <__pstl/cpu_algos/fill.h> -#include <__pstl/cpu_algos/find_if.h> -#include <__pstl/cpu_algos/for_each.h> -#include <__pstl/cpu_algos/merge.h> -#include <__pstl/cpu_algos/stable_sort.h> -#include <__pstl/cpu_algos/transform.h> -#include <__pstl/cpu_algos/transform_reduce.h> - #endif // _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H diff --git a/libcxx/include/__pstl/backends/serial.h b/libcxx/include/__pstl/backends/serial.h index 6e343313bea36..72b500bfc864f 100644 --- a/libcxx/include/__pstl/backends/serial.h +++ b/libcxx/include/__pstl/backends/serial.h @@ -10,12 +10,17 @@ #ifndef _LIBCPP___PSTL_BACKENDS_SERIAL_H #define _LIBCPP___PSTL_BACKENDS_SERIAL_H +#include <__algorithm/find_if.h> +#include <__algorithm/for_each.h> +#include <__algorithm/merge.h> +#include <__algorithm/stable_sort.h> +#include <__algorithm/transform.h> #include <__config> -#include <__pstl/configuration_fwd.h> -#include <__pstl/cpu_algos/cpu_traits.h> +#include <__numeric/transform_reduce.h> +#include <__pstl/backend_fwd.h> #include <__utility/empty.h> +#include <__utility/forward.h> #include <__utility/move.h> -#include #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -30,48 +35,144 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { -template <> -struct __cpu_traits<__serial_backend_tag> { - template - _LIBCPP_HIDE_FROM_ABI static optional<__empty> - __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) { - __f(__first, __last); +// +// This partial PSTL backend runs everything serially. +// +// TODO: Right now, the serial backend must be used with another backend +// like the "default backend" because it doesn't implement all the +// necessary PSTL operations. It would be better to dispatch all +// algorithms to their serial counterpart directly, since this can +// often be more efficient than the "default backend"'s implementation +// if we end up running serially anyways. +// + +template +struct __find_if<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator> + operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept { + return std::find_if(std::move(__first), std::move(__last), std::forward<_Pred>(__pred)); + } +}; + +template +struct __for_each<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&&, _ForwardIterator __first, _ForwardIterator __last, _Function&& __func) const noexcept { + std::for_each(std::move(__first), std::move(__last), std::forward<_Function>(__func)); return __empty{}; } +}; - template - _LIBCPP_HIDE_FROM_ABI static optional<_Tp> - __transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) { - return __reduce(std::move(__first), std::move(__last), std::move(__init)); +template +struct __merge<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> operator()( + _Policy&&, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _ForwardOutIterator __outit, + _Comp&& __comp) const noexcept { + return std::merge( + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__last2), + std::move(__outit), + std::forward<_Comp>(__comp)); } +}; - template - _LIBCPP_HIDE_FROM_ABI static optional<__empty> - __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) { - __leaf_sort(__first, __last, __comp); +template +struct __stable_sort<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp&& __comp) const noexcept { + std::stable_sort(std::move(__first), std::move(__last), std::forward<_Comp>(__comp)); return __empty{}; } +}; + +template +struct __transform<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> operator()( + _Policy&&, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __outit, _UnaryOperation&& __op) + const noexcept { + return std::transform( + std::move(__first), std::move(__last), std::move(__outit), std::forward<_UnaryOperation>(__op)); + } +}; - _LIBCPP_HIDE_FROM_ABI static void __cancel_execution() {} - - template - _LIBCPP_HIDE_FROM_ABI static optional<__empty> - __merge(_RandomAccessIterator1 __first1, - _RandomAccessIterator1 __last1, - _RandomAccessIterator2 __first2, - _RandomAccessIterator2 __last2, - _RandomAccessIterator3 __outit, - _Compare __comp, - _LeafMerge __leaf_merge) { - __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp); - return __empty{}; +template +struct __transform_binary<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> + operator()(_Policy&&, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardOutIterator __outit, + _BinaryOperation&& __op) const noexcept { + return std::transform( + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__outit), + std::forward<_BinaryOperation>(__op)); + } +}; + +template +struct __transform_reduce<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<_Tp> + operator()(_Policy&&, + _ForwardIterator __first, + _ForwardIterator __last, + _Tp __init, + _BinaryOperation&& __reduce, + _UnaryOperation&& __transform) const noexcept { + return std::transform_reduce( + std::move(__first), + std::move(__last), + std::move(__init), + std::forward<_BinaryOperation>(__reduce), + std::forward<_UnaryOperation>(__transform)); } +}; - static constexpr size_t __lane_size = 64; +template +struct __transform_reduce_binary<__serial_backend_tag, _ExecutionPolicy> { + template + _LIBCPP_HIDE_FROM_ABI optional<_Tp> operator()( + _Policy&&, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _Tp __init, + _BinaryOperation1&& __reduce, + _BinaryOperation2&& __transform) const noexcept { + return std::transform_reduce( + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__init), + std::forward<_BinaryOperation1>(__reduce), + std::forward<_BinaryOperation2>(__transform)); + } }; } // namespace __pstl @@ -81,14 +182,4 @@ _LIBCPP_POP_MACROS #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 -// Implement PSTL algorithms based on the __cpu_traits specialized above -#include <__pstl/cpu_algos/any_of.h> -#include <__pstl/cpu_algos/fill.h> -#include <__pstl/cpu_algos/find_if.h> -#include <__pstl/cpu_algos/for_each.h> -#include <__pstl/cpu_algos/merge.h> -#include <__pstl/cpu_algos/stable_sort.h> -#include <__pstl/cpu_algos/transform.h> -#include <__pstl/cpu_algos/transform_reduce.h> - #endif // _LIBCPP___PSTL_BACKENDS_SERIAL_H diff --git a/libcxx/include/__pstl/backends/std_thread.h b/libcxx/include/__pstl/backends/std_thread.h index e58f4859e6c9e..4e85fc0d66155 100644 --- a/libcxx/include/__pstl/backends/std_thread.h +++ b/libcxx/include/__pstl/backends/std_thread.h @@ -9,10 +9,17 @@ #ifndef _LIBCPP___PSTL_BACKENDS_STD_THREAD_H #define _LIBCPP___PSTL_BACKENDS_STD_THREAD_H -#include <__assert> #include <__config> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> +#include <__pstl/cpu_algos/any_of.h> #include <__pstl/cpu_algos/cpu_traits.h> +#include <__pstl/cpu_algos/fill.h> +#include <__pstl/cpu_algos/find_if.h> +#include <__pstl/cpu_algos/for_each.h> +#include <__pstl/cpu_algos/merge.h> +#include <__pstl/cpu_algos/stable_sort.h> +#include <__pstl/cpu_algos/transform.h> +#include <__pstl/cpu_algos/transform_reduce.h> #include <__utility/empty.h> #include <__utility/move.h> #include @@ -27,12 +34,16 @@ _LIBCPP_PUSH_MACROS #if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 -// This backend implementation is for testing purposes only and not meant for production use. This will be replaced -// by a proper implementation once the PSTL implementation is somewhat stable. - _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { +// +// This partial backend implementation is for testing purposes only and not meant for production use. This will be +// replaced by a proper implementation once the PSTL implementation is somewhat stable. +// +// This is intended to be used on top of the "default backend". +// + template <> struct __cpu_traits<__std_thread_backend_tag> { template @@ -77,6 +88,48 @@ struct __cpu_traits<__std_thread_backend_tag> { static constexpr size_t __lane_size = 64; }; +// Mandatory implementations of the computational basis +template +struct __find_if<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_find_if<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __for_each<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_for_each<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __merge<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_merge<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __stable_sort<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_stable_sort<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform_binary<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform_binary<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform_reduce<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform_reduce<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __transform_reduce_binary<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_transform_reduce_binary<__std_thread_backend_tag, _ExecutionPolicy> {}; + +// Not mandatory, but better optimized +template +struct __any_of<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_any_of<__std_thread_backend_tag, _ExecutionPolicy> {}; + +template +struct __fill<__std_thread_backend_tag, _ExecutionPolicy> + : __cpu_parallel_fill<__std_thread_backend_tag, _ExecutionPolicy> {}; + } // namespace __pstl _LIBCPP_END_NAMESPACE_STD @@ -84,14 +137,4 @@ _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS -// Implement PSTL algorithms based on the __cpu_traits specialized above -#include <__pstl/cpu_algos/any_of.h> -#include <__pstl/cpu_algos/fill.h> -#include <__pstl/cpu_algos/find_if.h> -#include <__pstl/cpu_algos/for_each.h> -#include <__pstl/cpu_algos/merge.h> -#include <__pstl/cpu_algos/stable_sort.h> -#include <__pstl/cpu_algos/transform.h> -#include <__pstl/cpu_algos/transform_reduce.h> - #endif // _LIBCPP___PSTL_BACKENDS_STD_THREAD_H diff --git a/libcxx/include/__pstl/configuration_fwd.h b/libcxx/include/__pstl/configuration_fwd.h deleted file mode 100644 index 995fcfce847cb..0000000000000 --- a/libcxx/include/__pstl/configuration_fwd.h +++ /dev/null @@ -1,245 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___PSTL_CONFIGURATION_FWD_H -#define _LIBCPP___PSTL_CONFIGURATION_FWD_H - -#include <__config> -#include - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -#endif - -#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 - -_LIBCPP_BEGIN_NAMESPACE_STD - -/* -TODO: Documentation of how backends work - -A PSTL parallel backend is a tag type to which the following functions are associated, at minimum: - - template - optional<__empty> __pstl_for_each(_Backend, _ExecutionPolicy&&, _Iterator __first, _Iterator __last, _Func __f); - - template - optional<_Iterator> __pstl_find_if(_Backend, _Iterator __first, _Iterator __last, _Predicate __pred); - - template - optional<__empty> - __pstl_stable_sort(_Backend, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp); - - template - optional<_ForwardOutIterator> __pstl_merge(_Backend, - _ForwardIterator1 __first1, - _ForwardIterator1 __last1, - _ForwardIterator2 __first2, - _ForwardIterator2 __last2, - _ForwardOutIterator __result, - _Comp __comp); - - template - optional<_OutIterator> - __pstl_transform(_Backend, _InIterator __first, _InIterator __last, _OutIterator __result, _UnaryOperation __op); - - template - optional<_OutIterator> __pstl_transform(_InIterator1 __first1, - _InIterator2 __first2, - _InIterator1 __last1, - _OutIterator __result, - _BinaryOperation __op); - - template - optional<_Tp> __pstl_transform_reduce(_Backend, - _Iterator1 __first1, - _Iterator1 __last1, - _Iterator2 __first2, - _Iterator2 __last2, - _Tp __init, - _BinaryOperation1 __reduce, - _BinaryOperation2 __transform); - - template - optional<_Tp> __pstl_transform_reduce(_Backend, - _Iterator __first, - _Iterator __last, - _Tp __init, - _BinaryOperation __reduce, - _UnaryOperation __transform); - -// TODO: Complete this list - -The following functions are optional but can be provided. If provided, they are used by the corresponding -algorithms, otherwise they are implemented in terms of other algorithms. If none of the optional algorithms are -implemented, all the algorithms will eventually forward to the basis algorithms listed above: - - template - optional<__empty> __pstl_for_each_n(_Backend, _Iterator __first, _Size __n, _Func __f); - - template - optional __pstl_any_of(_Backend, _Iterator __first, _iterator __last, _Predicate __pred); - - template - optional __pstl_all_of(_Backend, _Iterator __first, _iterator __last, _Predicate __pred); - - template - optional __pstl_none_of(_Backend, _Iterator __first, _iterator __last, _Predicate __pred); - - template - optional<_Iterator> __pstl_find(_Backend, _Iterator __first, _Iterator __last, const _Tp& __value); - - template - optional<_Iterator> __pstl_find_if_not(_Backend, _Iterator __first, _Iterator __last, _Predicate __pred); - - template - optional<__empty> __pstl_fill(_Backend, _Iterator __first, _Iterator __last, const _Tp& __value); - - template - optional<__empty> __pstl_fill_n(_Backend, _Iterator __first, _SizeT __n, const _Tp& __value); - - template - optional<__empty> __pstl_generate(_Backend, _Iterator __first, _Iterator __last, _Generator __gen); - - template - optional<__empty> __pstl_is_partitioned(_Backend, _Iterator __first, _Iterator __last, _Predicate __pred); - - template - optional<__empty> __pstl_generator_n(_Backend, _Iterator __first, _Size __n, _Generator __gen); - - template - optional<_OutIterator> __pstl_merge(_Backend, - _Iterator1 __first1, - _Iterator1 __last1, - _Iterator2 __first2, - _Iterator2 __last2, - _OutIterator __result, - _Comp __comp); - - template - optional<_OutIterator> __pstl_move(_Backend, _Iterator __first, _Iterator __last, _OutIterator __result); - - template - optional<_Tp> __pstl_reduce(_Backend, _Iterator __first, _Iterator __last, _Tp __init, _BinaryOperation __op); - - temlate - optional<__iter_value_type<_Iterator>> __pstl_reduce(_Backend, _Iterator __first, _Iterator __last); - - template - optional<__iter_diff_t<_Iterator>> __pstl_count(_Backend, _Iterator __first, _Iterator __last, const _Tp& __value); - - template - optional<__iter_diff_t<_Iterator>> __pstl_count_if(_Backend, _Iterator __first, _Iterator __last, _Predicate __pred); - - template - optional<__empty> - __pstl_replace(_Backend, _Iterator __first, _Iterator __last, const _Tp& __old_value, const _Tp& __new_value); - - template - optional<__empty> - __pstl_replace_if(_Backend, _Iterator __first, _Iterator __last, _Pred __pred, const _Tp& __new_value); - - template - optional<__empty> __pstl_replace_copy(_Backend, - _Iterator __first, - _Iterator __last, - _OutIterator __result, - const _Tp& __old_value, - const _Tp& __new_value); - - template - optional<__empty> __pstl_replace_copy_if(_Backend, - _Iterator __first, - _Iterator __last, - _OutIterator __result, - _Pred __pred, - const _Tp& __new_value); - - template - optional<_Iterator> __pstl_rotate_copy( - _Backend, _Iterator __first, _Iterator __middle, _Iterator __last, _OutIterator __result); - - template - optional<__empty> __pstl_sort(_Backend, _Iterator __first, _Iterator __last, _Comp __comp); - - template - optional __pstl_equal(_Backend, _Iterator1 first1, _Iterator1 last1, _Iterator2 first2, _Comp __comp); - -// TODO: Complete this list - -Exception handling -================== - -PSTL backends are expected to report errors (i.e. failure to allocate) by returning a disengaged `optional` from their -implementation. Exceptions shouldn't be used to report an internal failure-to-allocate, since all exceptions are turned -into a program termination at the front-end level. When a backend returns a disengaged `optional` to the frontend, the -frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to the user. -*/ - -namespace __pstl { -struct __libdispatch_backend_tag {}; -struct __serial_backend_tag {}; -struct __std_thread_backend_tag {}; -} // namespace __pstl - -# if defined(_LIBCPP_PSTL_BACKEND_SERIAL) -using __cpu_backend_tag = __pstl::__serial_backend_tag; -# elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) -using __cpu_backend_tag = __pstl::__std_thread_backend_tag; -# elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) -using __cpu_backend_tag = __pstl::__libdispatch_backend_tag; -# endif - -template -struct __select_backend; - -template <> -struct __select_backend { - using type = __cpu_backend_tag; -}; - -# if _LIBCPP_STD_VER >= 20 -template <> -struct __select_backend { - using type = __cpu_backend_tag; -}; -# endif - -# if defined(_LIBCPP_PSTL_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) || \ - defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) -template <> -struct __select_backend { - using type = __cpu_backend_tag; -}; - -template <> -struct __select_backend { - using type = __cpu_backend_tag; -}; - -# else - -// ...New vendors can add parallel backends here... - -# error "Invalid choice of a PSTL parallel backend" -# endif - -_LIBCPP_END_NAMESPACE_STD - -#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 - -#endif // _LIBCPP___PSTL_CONFIGURATION_FWD_H diff --git a/libcxx/include/__pstl/cpu_algos/any_of.h b/libcxx/include/__pstl/cpu_algos/any_of.h index 01b9d214310a3..3173eade7585b 100644 --- a/libcxx/include/__pstl/cpu_algos/any_of.h +++ b/libcxx/include/__pstl/cpu_algos/any_of.h @@ -10,13 +10,12 @@ #define _LIBCPP___PSTL_CPU_ALGOS_ANY_OF_H #include <__algorithm/any_of.h> -#include <__algorithm/find_if.h> +#include <__assert> #include <__atomic/atomic.h> #include <__atomic/memory_order.h> #include <__config> -#include <__functional/operations.h> #include <__iterator/concepts.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> @@ -70,25 +69,28 @@ _LIBCPP_HIDE_FROM_ABI bool __simd_or(_Index __first, _DifferenceType __n, _Pred return false; } -template -_LIBCPP_HIDE_FROM_ABI optional -__pstl_any_of(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return std::__parallel_or<__cpu_backend_tag>( - __first, __last, [&__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { - auto __res = std::__pstl_any_of<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, __brick_first, __brick_last, __pred); - _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); - return *std::move(__res); - }); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return std::__simd_or(__first, __last - __first, __pred); - } else { - return std::any_of(__first, __last, __pred); +template +struct __cpu_parallel_any_of { + template + _LIBCPP_HIDE_FROM_ABI optional + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + return std::__parallel_or<_Backend>( + __first, __last, [&__policy, &__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + using _AnyOfUnseq = __pstl::__any_of<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + auto __res = _AnyOfUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __pred); + _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); + return *std::move(__res); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + return std::__simd_or(__first, __last - __first, __pred); + } else { + return std::any_of(__first, __last, __pred); + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/cpu_algos/fill.h b/libcxx/include/__pstl/cpu_algos/fill.h index 66fb751eb7a2e..b99a9d3c660d8 100644 --- a/libcxx/include/__pstl/cpu_algos/fill.h +++ b/libcxx/include/__pstl/cpu_algos/fill.h @@ -10,9 +10,10 @@ #define _LIBCPP___PSTL_CPU_ALGOS_FILL_H #include <__algorithm/fill.h> +#include <__assert> #include <__config> #include <__iterator/concepts.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> @@ -35,26 +36,30 @@ _LIBCPP_HIDE_FROM_ABI _Index __simd_fill_n(_Index __first, _DifferenceType __n, return __first + __n; } -template -_LIBCPP_HIDE_FROM_ABI optional<__empty> -__pstl_fill(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return __pstl::__cpu_traits<__cpu_backend_tag>::__for_each( - __first, __last, [&__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { - [[maybe_unused]] auto __res = std::__pstl_fill<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, __brick_first, __brick_last, __value); - _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); - }); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - std::__simd_fill_n(__first, __last - __first, __value); - return __empty{}; - } else { - std::fill(__first, __last, __value); - return __empty{}; +template +struct __cpu_parallel_fill { + template + _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + return __pstl::__cpu_traits<_Backend>::__for_each( + __first, __last, [&__policy, &__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + using _FillUnseq = __pstl::__fill<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + [[maybe_unused]] auto __res = + _FillUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __value); + _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + std::__simd_fill_n(__first, __last - __first, __value); + return __empty{}; + } else { + std::fill(__first, __last, __value); + return __empty{}; + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/cpu_algos/find_if.h b/libcxx/include/__pstl/cpu_algos/find_if.h index c99ec01bff48f..3ddbee44890f6 100644 --- a/libcxx/include/__pstl/cpu_algos/find_if.h +++ b/libcxx/include/__pstl/cpu_algos/find_if.h @@ -10,12 +10,13 @@ #define _LIBCPP___PSTL_CPU_ALGOS_FIND_IF_H #include <__algorithm/find_if.h> +#include <__assert> #include <__atomic/atomic.h> #include <__config> #include <__functional/operations.h> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> @@ -98,33 +99,36 @@ __simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Co return __first + __end; } -template -_LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator> -__pstl_find_if(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return std::__parallel_find<__cpu_backend_tag>( - __first, - __last, - [&__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { - auto __res = std::__pstl_find_if<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, __brick_first, __brick_last, __pred); - _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); - return *std::move(__res); - }, - less<>{}, - true); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - using __diff_t = __iter_diff_t<_ForwardIterator>; - return std::__simd_first<__cpu_backend_tag>( - __first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) { - return __pred(__iter[__i]); - }); - } else { - return std::find_if(__first, __last, __pred); +template +struct __cpu_parallel_find_if { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + return std::__parallel_find<_Backend>( + __first, + __last, + [&__policy, &__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + using _FindIfUnseq = __pstl::__find_if<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + auto __res = _FindIfUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __pred); + _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); + return *std::move(__res); + }, + less<>{}, + true); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + using __diff_t = __iter_diff_t<_ForwardIterator>; + return std::__simd_first<_Backend>( + __first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) { + return __pred(__iter[__i]); + }); + } else { + return std::find_if(__first, __last, __pred); + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/cpu_algos/for_each.h b/libcxx/include/__pstl/cpu_algos/for_each.h index cd7ce022469bd..9c49fb8432829 100644 --- a/libcxx/include/__pstl/cpu_algos/for_each.h +++ b/libcxx/include/__pstl/cpu_algos/for_each.h @@ -10,9 +10,10 @@ #define _LIBCPP___PSTL_CPU_ALGOS_FOR_EACH_H #include <__algorithm/for_each.h> +#include <__assert> #include <__config> #include <__iterator/concepts.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> @@ -35,26 +36,30 @@ _LIBCPP_HIDE_FROM_ABI _Iterator __simd_walk(_Iterator __first, _DifferenceType _ return __first + __n; } -template -_LIBCPP_HIDE_FROM_ABI optional<__empty> -__pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return __pstl::__cpu_traits<__cpu_backend_tag>::__for_each( - __first, __last, [__func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { - [[maybe_unused]] auto __res = std::__pstl_for_each<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, __brick_first, __brick_last, __func); - _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); - }); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - std::__simd_walk(__first, __last - __first, __func); - return __empty{}; - } else { - std::for_each(__first, __last, __func); - return __empty{}; +template +struct __cpu_parallel_for_each { + template + _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + return __pstl::__cpu_traits<_Backend>::__for_each( + __first, __last, [&__policy, __func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + using _ForEachUnseq = __pstl::__for_each<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + [[maybe_unused]] auto __res = + _ForEachUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __func); + _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + std::__simd_walk(__first, __last - __first, __func); + return __empty{}; + } else { + std::for_each(__first, __last, __func); + return __empty{}; + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/cpu_algos/merge.h b/libcxx/include/__pstl/cpu_algos/merge.h index b857fc1fb7a56..4f4192cccb3e8 100644 --- a/libcxx/include/__pstl/cpu_algos/merge.h +++ b/libcxx/include/__pstl/cpu_algos/merge.h @@ -10,9 +10,10 @@ #define _LIBCPP___PSTL_CPU_ALGOS_MERGE_H #include <__algorithm/merge.h> +#include <__assert> #include <__config> #include <__iterator/concepts.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> @@ -29,53 +30,53 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_merge( - __cpu_backend_tag, - _ForwardIterator1 __first1, - _ForwardIterator1 __last1, - _ForwardIterator2 __first2, - _ForwardIterator2 __last2, - _ForwardOutIterator __result, - _Comp __comp) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && - __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && - __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { - auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__merge( - __first1, - __last1, - __first2, - __last2, - __result, - __comp, - [](_ForwardIterator1 __g_first1, - _ForwardIterator1 __g_last1, - _ForwardIterator2 __g_first2, - _ForwardIterator2 __g_last2, - _ForwardOutIterator __g_result, - _Comp __g_comp) { - [[maybe_unused]] auto __g_res = std::__pstl_merge<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, - std::move(__g_first1), - std::move(__g_last1), - std::move(__g_first2), - std::move(__g_last2), - std::move(__g_result), - std::move(__g_comp)); - _LIBCPP_ASSERT_INTERNAL(__g_res, "unsed/sed should never try to allocate!"); - }); - if (!__res) - return nullopt; - return __result + (__last1 - __first1) + (__last2 - __first2); - } else { - return std::merge(__first1, __last1, __first2, __last2, __result, __comp); +template +struct __cpu_parallel_merge { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> operator()( + _Policy&& __policy, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardIterator2 __last2, + _ForwardOutIterator __result, + _Comp __comp) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { + auto __res = __pstl::__cpu_traits<_Backend>::__merge( + __first1, + __last1, + __first2, + __last2, + __result, + __comp, + [&__policy](_ForwardIterator1 __g_first1, + _ForwardIterator1 __g_last1, + _ForwardIterator2 __g_first2, + _ForwardIterator2 __g_last2, + _ForwardOutIterator __g_result, + _Comp __g_comp) { + using _MergeUnseq = __pstl::__merge<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + [[maybe_unused]] auto __g_res = _MergeUnseq()( + std::__remove_parallel_policy(__policy), + std::move(__g_first1), + std::move(__g_last1), + std::move(__g_first2), + std::move(__g_last2), + std::move(__g_result), + std::move(__g_comp)); + _LIBCPP_ASSERT_INTERNAL(__g_res, "unsed/sed should never try to allocate!"); + }); + if (!__res) + return nullopt; + return __result + (__last1 - __first1) + (__last2 - __first2); + } else { + return std::merge(__first1, __last1, __first2, __last2, __result, __comp); + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/cpu_algos/stable_sort.h b/libcxx/include/__pstl/cpu_algos/stable_sort.h index 18effb2108a2f..8ea5e8a01d2ce 100644 --- a/libcxx/include/__pstl/cpu_algos/stable_sort.h +++ b/libcxx/include/__pstl/cpu_algos/stable_sort.h @@ -11,7 +11,7 @@ #include <__algorithm/stable_sort.h> #include <__config> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> @@ -25,19 +25,22 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_HIDE_FROM_ABI optional<__empty> -__pstl_stable_sort(__cpu_backend_tag, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy>) { - return __pstl::__cpu_traits<__cpu_backend_tag>::__stable_sort( - __first, __last, __comp, [](_RandomAccessIterator __g_first, _RandomAccessIterator __g_last, _Comp __g_comp) { - std::stable_sort(__g_first, __g_last, __g_comp); - }); - } else { - std::stable_sort(__first, __last, __comp); - return __empty{}; +template +struct __cpu_parallel_stable_sort { + template + _LIBCPP_HIDE_FROM_ABI optional<__empty> + operator()(_Policy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy>) { + return __pstl::__cpu_traits<_Backend>::__stable_sort( + __first, __last, __comp, [](_RandomAccessIterator __g_first, _RandomAccessIterator __g_last, _Comp __g_comp) { + std::stable_sort(__g_first, __g_last, __g_comp); + }); + } else { + std::stable_sort(__first, __last, __comp); + return __empty{}; + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/cpu_algos/transform.h b/libcxx/include/__pstl/cpu_algos/transform.h index 70853dc9af24e..a4541fb22e8f6 100644 --- a/libcxx/include/__pstl/cpu_algos/transform.h +++ b/libcxx/include/__pstl/cpu_algos/transform.h @@ -10,14 +10,14 @@ #define _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_H #include <__algorithm/transform.h> +#include <__assert> #include <__config> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> -#include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> -#include <__type_traits/remove_cvref.h> +#include <__utility/move.h> #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -40,38 +40,48 @@ __simd_walk(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Func return __first2 + __n; } -template -_LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform( - __cpu_backend_tag, - _ForwardIterator __first, - _ForwardIterator __last, - _ForwardOutIterator __result, - _UnaryOperation __op) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value && - __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { - __pstl::__cpu_traits<__cpu_backend_tag>::__for_each( - __first, __last, [__op, __first, __result](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { - auto __res = std::__pstl_transform<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, __brick_first, __brick_last, __result + (__brick_first - __first), __op); - _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); - return *std::move(__res); - }); - return __result + (__last - __first); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value && - __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { - return std::__simd_walk( - __first, - __last - __first, - __result, - [&](__iter_reference<_ForwardIterator> __in_value, __iter_reference<_ForwardOutIterator> __out_value) { - __out_value = __op(__in_value); - }); - } else { - return std::transform(__first, __last, __result, __op); +template +struct __cpu_parallel_transform { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> + operator()(_Policy&& __policy, + _ForwardIterator __first, + _ForwardIterator __last, + _ForwardOutIterator __result, + _UnaryOperation __op) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { + __pstl::__cpu_traits<_Backend>::__for_each( + __first, + __last, + [&__policy, __op, __first, __result](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + using _TransformUnseq = __pstl::__transform<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + auto __res = _TransformUnseq()( + std::__remove_parallel_policy(__policy), + __brick_first, + __brick_last, + __result + (__brick_first - __first), + __op); + _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); + return *std::move(__res); + }); + return __result + (__last - __first); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { + return std::__simd_walk( + __first, + __last - __first, + __result, + [&](__iter_reference<_ForwardIterator> __in_value, __iter_reference<_ForwardOutIterator> __out_value) { + __out_value = __op(__in_value); + }); + } else { + return std::transform(__first, __last, __result, __op); + } } -} +}; template _LIBCPP_HIDE_FROM_ABI _Iterator3 __simd_walk( @@ -81,54 +91,60 @@ _LIBCPP_HIDE_FROM_ABI _Iterator3 __simd_walk( __f(__first1[__i], __first2[__i], __first3[__i]); return __first3 + __n; } -template >, int> = 0> -_LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform( - __cpu_backend_tag, - _ForwardIterator1 __first1, - _ForwardIterator1 __last1, - _ForwardIterator2 __first2, - _ForwardOutIterator __result, - _BinaryOperation __op) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && - __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && - __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { - auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__for_each( - __first1, - __last1, - [__op, __first1, __first2, __result](_ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last) { - return std::__pstl_transform<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, - __brick_first, - __brick_last, - __first2 + (__brick_first - __first1), - __result + (__brick_first - __first1), - __op); - }); - if (!__res) - return nullopt; - return __result + (__last1 - __first1); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && - __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && - __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { - return std::__simd_walk( - __first1, - __last1 - __first1, - __first2, - __result, - [&](__iter_reference<_ForwardIterator1> __in1, - __iter_reference<_ForwardIterator2> __in2, - __iter_reference<_ForwardOutIterator> __out_value) { __out_value = __op(__in1, __in2); }); - } else { - return std::transform(__first1, __last1, __first2, __result, __op); + +template +struct __cpu_parallel_transform_binary { + template + _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> + operator()(_Policy&& __policy, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _ForwardOutIterator __result, + _BinaryOperation __op) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { + auto __res = __pstl::__cpu_traits<_Backend>::__for_each( + __first1, + __last1, + [&__policy, __op, __first1, __first2, __result]( + _ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last) { + using _TransformBinaryUnseq = + __pstl::__transform_binary<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + return _TransformBinaryUnseq()( + std::__remove_parallel_policy(__policy), + __brick_first, + __brick_last, + __first2 + (__brick_first - __first1), + __result + (__brick_first - __first1), + __op); + }); + if (!__res) + return nullopt; + return __result + (__last1 - __first1); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value && + __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) { + return std::__simd_walk( + __first1, + __last1 - __first1, + __first2, + __result, + [&](__iter_reference<_ForwardIterator1> __in1, + __iter_reference<_ForwardIterator2> __in2, + __iter_reference<_ForwardOutIterator> __out_value) { __out_value = __op(__in1, __in2); }); + } else { + return std::transform(__first1, __last1, __first2, __result, __op); + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/cpu_algos/transform_reduce.h b/libcxx/include/__pstl/cpu_algos/transform_reduce.h index a85ee9fb773af..914c46dcd6dcf 100644 --- a/libcxx/include/__pstl/cpu_algos/transform_reduce.h +++ b/libcxx/include/__pstl/cpu_algos/transform_reduce.h @@ -9,16 +9,18 @@ #ifndef _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H #define _LIBCPP___PSTL_CPU_ALGOS_TRANSFORM_REDUCE_H +#include <__assert> #include <__config> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/transform_reduce.h> -#include <__pstl/configuration_fwd.h> +#include <__pstl/backend_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/desugars_to.h> #include <__type_traits/is_arithmetic.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> +#include #include #include @@ -103,99 +105,109 @@ __simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _Un return __init; } -template -_LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce( - __cpu_backend_tag, - _ForwardIterator1 __first1, - _ForwardIterator1 __last1, - _ForwardIterator2 __first2, - _Tp __init, - _BinaryOperation1 __reduce, - _BinaryOperation2 __transform) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && - __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) { - return __pstl::__cpu_traits<__cpu_backend_tag>::__transform_reduce( - __first1, - std::move(__last1), - [__first1, __first2, __transform](_ForwardIterator1 __iter) { - return __transform(*__iter, *(__first2 + (__iter - __first1))); - }, - std::move(__init), - std::move(__reduce), - [__first1, __first2, __reduce, __transform]( - _ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last, _Tp __brick_init) { - return *std::__pstl_transform_reduce<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, - __brick_first, - std::move(__brick_last), - __first2 + (__brick_first - __first1), - std::move(__brick_init), - std::move(__reduce), - std::move(__transform)); - }); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && - __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) { - return std::__simd_transform_reduce<__cpu_backend_tag>( - __last1 - __first1, std::move(__init), std::move(__reduce), [&](__iter_diff_t<_ForwardIterator1> __i) { - return __transform(__first1[__i], __first2[__i]); - }); - } else { - return std::transform_reduce( - std::move(__first1), - std::move(__last1), - std::move(__first2), - std::move(__init), - std::move(__reduce), - std::move(__transform)); +template +struct __cpu_parallel_transform_reduce_binary { + template + _LIBCPP_HIDE_FROM_ABI optional<_Tp> operator()( + _Policy&& __policy, + _ForwardIterator1 __first1, + _ForwardIterator1 __last1, + _ForwardIterator2 __first2, + _Tp __init, + _BinaryOperation1 __reduce, + _BinaryOperation2 __transform) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) { + return __pstl::__cpu_traits<_Backend>::__transform_reduce( + __first1, + std::move(__last1), + [__first1, __first2, __transform](_ForwardIterator1 __iter) { + return __transform(*__iter, *(__first2 + (__iter - __first1))); + }, + std::move(__init), + std::move(__reduce), + [&__policy, __first1, __first2, __reduce, __transform]( + _ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last, _Tp __brick_init) { + using _TransformReduceBinaryUnseq = + __pstl::__transform_reduce_binary<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + return *_TransformReduceBinaryUnseq()( + std::__remove_parallel_policy(__policy), + __brick_first, + std::move(__brick_last), + __first2 + (__brick_first - __first1), + std::move(__brick_init), + std::move(__reduce), + std::move(__transform)); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value && + __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) { + return std::__simd_transform_reduce<_Backend>( + __last1 - __first1, std::move(__init), std::move(__reduce), [&](__iter_diff_t<_ForwardIterator1> __i) { + return __transform(__first1[__i], __first2[__i]); + }); + } else { + return std::transform_reduce( + std::move(__first1), + std::move(__last1), + std::move(__first2), + std::move(__init), + std::move(__reduce), + std::move(__transform)); + } } -} +}; -template -_LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce( - __cpu_backend_tag, - _ForwardIterator __first, - _ForwardIterator __last, - _Tp __init, - _BinaryOperation __reduce, - _UnaryOperation __transform) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return __pstl::__cpu_traits<__cpu_backend_tag>::__transform_reduce( - std::move(__first), - std::move(__last), - [__transform](_ForwardIterator __iter) { return __transform(*__iter); }, - std::move(__init), - __reduce, - [__transform, __reduce](auto __brick_first, auto __brick_last, _Tp __brick_init) { - auto __res = std::__pstl_transform_reduce<__remove_parallel_policy_t<_ExecutionPolicy>>( - __cpu_backend_tag{}, - std::move(__brick_first), - std::move(__brick_last), - std::move(__brick_init), - std::move(__reduce), - std::move(__transform)); - _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); - return *std::move(__res); - }); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { - return std::__simd_transform_reduce<__cpu_backend_tag>( - __last - __first, - std::move(__init), - std::move(__reduce), - [=, &__transform](__iter_diff_t<_ForwardIterator> __i) { return __transform(__first[__i]); }); - } else { - return std::transform_reduce( - std::move(__first), std::move(__last), std::move(__init), std::move(__reduce), std::move(__transform)); +template +struct __cpu_parallel_transform_reduce { + template + _LIBCPP_HIDE_FROM_ABI optional<_Tp> + operator()(_Policy&& __policy, + _ForwardIterator __first, + _ForwardIterator __last, + _Tp __init, + _BinaryOperation __reduce, + _UnaryOperation __transform) const noexcept { + if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + return __pstl::__cpu_traits<_Backend>::__transform_reduce( + std::move(__first), + std::move(__last), + [__transform](_ForwardIterator __iter) { return __transform(*__iter); }, + std::move(__init), + __reduce, + [&__policy, __transform, __reduce](auto __brick_first, auto __brick_last, _Tp __brick_init) { + using _TransformReduceUnseq = + __pstl::__transform_reduce<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>; + auto __res = _TransformReduceUnseq()( + std::__remove_parallel_policy(__policy), + std::move(__brick_first), + std::move(__brick_last), + std::move(__brick_init), + std::move(__reduce), + std::move(__transform)); + _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!"); + return *std::move(__res); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> && + __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) { + return std::__simd_transform_reduce<_Backend>( + __last - __first, + std::move(__init), + std::move(__reduce), + [=, &__transform](__iter_diff_t<_ForwardIterator> __i) { return __transform(__first[__i]); }); + } else { + return std::transform_reduce( + std::move(__first), std::move(__last), std::move(__init), std::move(__reduce), std::move(__transform)); + } } -} +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__pstl/dispatch.h b/libcxx/include/__pstl/dispatch.h new file mode 100644 index 0000000000000..5e903f7524fe9 --- /dev/null +++ b/libcxx/include/__pstl/dispatch.h @@ -0,0 +1,66 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___PSTL_DISPATCH_H +#define _LIBCPP___PSTL_DISPATCH_H + +#include <__config> +#include <__pstl/backend_fwd.h> +#include <__type_traits/conditional.h> +#include <__type_traits/enable_if.h> +#include <__type_traits/integral_constant.h> +#include <__type_traits/type_identity.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD +namespace __pstl { + +template