Skip to content

Commit 8e2d09c

Browse files
ldionnephilnik777
ldionne
authored and
philnik777
committed
[libc++][PSTL] Add more specialized backend customization points
This allows backends to customize arbitrary parallel algorithms, which was requested pretty often. Reviewed By: #libc, ldionne Spies: arichardson, miyuki, crtrott, dalg24, __simt__, philnik, libcxx-commits Differential Revision: https://reviews.llvm.org/D149686
1 parent 3e7eab0 commit 8e2d09c

File tree

20 files changed

+348
-37
lines changed

20 files changed

+348
-37
lines changed

‎libcxx/include/CMakeLists.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,15 @@ set(files
7070
__algorithm/pop_heap.h
7171
__algorithm/prev_permutation.h
7272
__algorithm/pstl_any_all_none_of.h
73+
__algorithm/pstl_backend.h
74+
__algorithm/pstl_backends/cpu_backend.h
75+
__algorithm/pstl_backends/cpu_backends/backend.h
76+
__algorithm/pstl_backends/cpu_backends/for_each.h
77+
__algorithm/pstl_backends/cpu_backends/serial.h
7378
__algorithm/pstl_fill.h
7479
__algorithm/pstl_find.h
7580
__algorithm/pstl_for_each.h
81+
__algorithm/pstl_frontend_dispatch.h
7682
__algorithm/push_heap.h
7783
__algorithm/ranges_adjacent_find.h
7884
__algorithm/ranges_all_of.h

‎libcxx/include/__algorithm/pstl_backend.h

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef _LIBCPP___ALGORITHM_PSTL_BACKEND_H
10+
#define _LIBCPP___ALGORITHM_PSTL_BACKEND_H
11+
12+
#include <__algorithm/pstl_backends/cpu_backend.h>
13+
#include <__config>
14+
#include <execution>
15+
16+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
17+
# pragma GCC system_header
18+
#endif
19+
20+
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
21+
22+
_LIBCPP_BEGIN_NAMESPACE_STD
23+
24+
/*
25+
TODO: Documentation of how backends work
26+
27+
A PSTL parallel backend is a tag type to which the following functions are associated, at minimum:
28+
29+
template <class _ExecutionPolicy, class _Iterator, class _Func>
30+
void __pstl_for_each(_Backend, _ExecutionPolicy&&, _Iterator __first, _Iterator __last, _Func __f);
31+
32+
// TODO: Complete this list
33+
34+
The following functions are optional but can be provided. If provided, they are used by the corresponding
35+
algorithms, otherwise they are implemented in terms of other algorithms. If none of the optional algorithms are
36+
implemented, all the algorithms will eventually forward to the basis algorithms listed above:
37+
38+
template <class _ExecutionPolicy, class _Iterator, class _Size, class _Func>
39+
void __pstl_for_each_n(_Backend, _ExecutionPolicy&&, _Iterator __first, _Size __n, _Func __f);
40+
41+
// TODO: Complete this list
42+
43+
*/
44+
45+
template <class _ExecutionPolicy>
46+
struct __select_backend;
47+
48+
template <>
49+
struct __select_backend<std::execution::sequenced_policy> {
50+
using type = __cpu_backend_tag;
51+
};
52+
53+
# if _LIBCPP_STD_VER >= 20
54+
template <>
55+
struct __select_backend<std::execution::unsequenced_policy> {
56+
using type = __cpu_backend_tag;
57+
};
58+
# endif
59+
60+
# if defined(_PSTL_CPU_BACKEND_SERIAL)
61+
template <>
62+
struct __select_backend<std::execution::parallel_policy> {
63+
using type = __cpu_backend_tag;
64+
};
65+
66+
template <>
67+
struct __select_backend<std::execution::parallel_unsequenced_policy> {
68+
using type = __cpu_backend_tag;
69+
};
70+
71+
# else
72+
73+
// ...New vendors can add parallel backends here...
74+
75+
# error "Invalid choice of a PSTL parallel backend"
76+
# endif
77+
78+
_LIBCPP_END_NAMESPACE_STD
79+
80+
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
81+
82+
#endif // _LIBCPP___ALGORITHM_PSTL_BACKEND_H

‎libcxx/include/__algorithm/pstl_backends/cpu_backend.h

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H
10+
#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H
11+
12+
#include <__config>
13+
14+
/*
15+
16+
// _Functor takes a subrange for [__first, __last) that should be executed in serial
17+
template <class _RandomAccessIterator, class _Functor>
18+
void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func);
19+
20+
TODO: Document the parallel backend
21+
*/
22+
23+
#include <__algorithm/pstl_backends/cpu_backends/for_each.h>
24+
25+
#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H

‎libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H
10+
#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H
11+
12+
#include <__config>
13+
14+
#if defined(_LIBCPP_HAS_NO_THREADS) || defined(_PSTL_CPU_BACKEND_SERIAL)
15+
# include <__algorithm/pstl_backends/cpu_backends/serial.h>
16+
#else
17+
# error "Invalid CPU backend choice"
18+
#endif
19+
20+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
21+
# pragma GCC system_header
22+
#endif
23+
24+
_LIBCPP_BEGIN_NAMESPACE_STD
25+
26+
struct __cpu_backend_tag {};
27+
28+
_LIBCPP_END_NAMESPACE_STD
29+
30+
#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H

‎libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H
10+
#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H
11+
12+
#include <__algorithm/for_each.h>
13+
#include <__algorithm/pstl_backends/cpu_backends/backend.h>
14+
#include <__config>
15+
#include <__iterator/iterator_traits.h>
16+
#include <__type_traits/is_execution_policy.h>
17+
#include <__utility/terminate_on_exception.h>
18+
19+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
20+
# pragma GCC system_header
21+
#endif
22+
23+
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
24+
25+
_LIBCPP_BEGIN_NAMESPACE_STD
26+
27+
template <class _Iterator, class _DifferenceType, class _Function>
28+
_LIBCPP_HIDE_FROM_ABI _Iterator __simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept {
29+
_PSTL_PRAGMA_SIMD
30+
for (_DifferenceType __i = 0; __i < __n; ++__i)
31+
__f(__first[__i]);
32+
33+
return __first + __n;
34+
}
35+
36+
template <class _ExecutionPolicy, class _ForwardIterator, class _Functor>
37+
_LIBCPP_HIDE_FROM_ABI void
38+
__pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) {
39+
if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
40+
__is_cpp17_random_access_iterator<_ForwardIterator>::value) {
41+
std::__terminate_on_exception([&] {
42+
std::__par_backend::__parallel_for(
43+
__first, __last, [__func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
44+
std::__pstl_for_each<__remove_parallel_policy_t<_ExecutionPolicy>>(
45+
__cpu_backend_tag{}, __brick_first, __brick_last, __func);
46+
});
47+
});
48+
} else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
49+
__is_cpp17_random_access_iterator<_ForwardIterator>::value) {
50+
std::__simd_walk_1(__first, __last - __first, __func);
51+
} else {
52+
std::for_each(__first, __last, __func);
53+
}
54+
}
55+
56+
_LIBCPP_END_NAMESPACE_STD
57+
58+
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
59+
60+
#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H

‎libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// -*- C++ -*-
2+
//===----------------------------------------------------------------------===//
3+
//
4+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5+
// See https://llvm.org/LICENSE.txt for license information.
6+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7+
//
8+
//===----------------------------------------------------------------------===//
9+
10+
#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H
11+
#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H
12+
13+
#include <__config>
14+
15+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
16+
# pragma GCC system_header
17+
#endif
18+
19+
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
20+
21+
_LIBCPP_BEGIN_NAMESPACE_STD
22+
23+
namespace __par_backend {
24+
inline namespace __serial_cpu_backend {
25+
26+
template <class _RandomAccessIterator, class _Fp>
27+
_LIBCPP_HIDE_FROM_ABI void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
28+
__f(__first, __last);
29+
}
30+
31+
// TODO: Complete this list
32+
33+
} // namespace __serial_cpu_backend
34+
} // namespace __par_backend
35+
36+
_LIBCPP_END_NAMESPACE_STD
37+
38+
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17
39+
40+
#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H

‎libcxx/include/__algorithm/pstl_for_each.h

Lines changed: 25 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111

1212
#include <__algorithm/for_each.h>
1313
#include <__algorithm/for_each_n.h>
14+
#include <__algorithm/pstl_backend.h>
15+
#include <__algorithm/pstl_frontend_dispatch.h>
1416
#include <__config>
1517
#include <__iterator/iterator_traits.h>
1618
#include <__pstl/internal/parallel_backend.h>
@@ -19,6 +21,7 @@
1921
#include <__type_traits/enable_if.h>
2022
#include <__type_traits/is_execution_policy.h>
2123
#include <__type_traits/remove_cvref.h>
24+
#include <__type_traits/void_t.h>
2225
#include <__utility/terminate_on_exception.h>
2326

2427
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -32,41 +35,37 @@ _LIBCPP_BEGIN_NAMESPACE_STD
3235
template <class _ExecutionPolicy,
3336
class _ForwardIterator,
3437
class _Function,
35-
enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
38+
class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
39+
enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
3640
_LIBCPP_HIDE_FROM_ABI void
37-
for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) {
38-
if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
39-
__is_cpp17_random_access_iterator<_ForwardIterator>::value) {
40-
std::__terminate_on_exception([&] {
41-
__pstl::__par_backend::__parallel_for(
42-
{},
43-
__policy,
44-
__first,
45-
__last,
46-
[&__policy, __func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
47-
std::for_each(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __func);
48-
});
49-
});
50-
} else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
51-
__is_cpp17_random_access_iterator<_ForwardIterator>::value) {
52-
__pstl::__unseq_backend::__simd_walk_1(__first, __last - __first, __func);
53-
} else {
54-
std::for_each(__first, __last, __func);
55-
}
41+
for_each(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __func) {
42+
using _Backend = typename __select_backend<_RawPolicy>::type;
43+
std::__pstl_for_each<_RawPolicy>(_Backend{}, std::move(__first), std::move(__last), std::move(__func));
5644
}
5745

46+
template <class>
47+
void __pstl_for_each_n(); // declaration needed for the frontend dispatch below
48+
5849
template <class _ExecutionPolicy,
5950
class _ForwardIterator,
6051
class _Size,
6152
class _Function,
62-
enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
53+
class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
54+
enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
6355
_LIBCPP_HIDE_FROM_ABI void
6456
for_each_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) {
65-
if constexpr (__is_cpp17_random_access_iterator<_ForwardIterator>::value) {
66-
std::for_each(__policy, __first, __first + __size, __func);
67-
} else {
68-
std::for_each_n(__first, __size, __func);
69-
}
57+
return std::__pstl_frontend_dispatch(
58+
_LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_for_each_n),
59+
[&](_ForwardIterator __g_first, _Size __g_size, _Function __g_func) {
60+
if constexpr (__is_cpp17_random_access_iterator<_ForwardIterator>::value) {
61+
std::for_each(__policy, std::move(__g_first), __g_first + __g_size, std::move(__g_func));
62+
} else {
63+
std::for_each_n(std::move(__g_first), __g_size, std::move(__g_func));
64+
}
65+
},
66+
__first,
67+
__size,
68+
std::move(__func));
7069
}
7170

7271
_LIBCPP_END_NAMESPACE_STD

‎libcxx/include/__algorithm/pstl_frontend_dispatch.h

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef _LIBCPP___ALGORITHM_PSTL_FRONTEND_DISPATCH
10+
#define _LIBCPP___ALGORITHM_PSTL_FRONTEND_DISPATCH
11+
12+
#include <__config>
13+
#include <__type_traits/is_callable.h>
14+
#include <__utility/forward.h>
15+
16+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
17+
# pragma GCC system_header
18+
#endif
19+
20+
#if _LIBCPP_STD_VER >= 17
21+
22+
_LIBCPP_BEGIN_NAMESPACE_STD
23+
24+
# define _LIBCPP_PSTL_CUSTOMIZATION_POINT(name) \
25+
[](auto&&... __args) -> decltype(std::name<_RawPolicy>(typename __select_backend<_RawPolicy>::type{}, \
26+
std::forward<decltype(__args)>(__args)...)) { \
27+
return std::name<_RawPolicy>( \
28+
typename __select_backend<_RawPolicy>::type{}, std::forward<decltype(__args)>(__args)...); \
29+
}
30+
31+
template <class _SpecializedImpl, class _GenericImpl, class... _Args>
32+
_LIBCPP_HIDE_FROM_ABI decltype(auto)
33+
__pstl_frontend_dispatch(_SpecializedImpl __specialized_impl, _GenericImpl __generic_impl, _Args&&... __args) {
34+
if constexpr (__is_callable<_SpecializedImpl, _Args...>::value) {
35+
return __specialized_impl(std::forward<_Args>(__args)...);
36+
} else {
37+
return __generic_impl(std::forward<_Args>(__args)...);
38+
}
39+
}
40+
41+
_LIBCPP_END_NAMESPACE_STD
42+
43+
#endif // _LIBCPP_STD_VER >= 17
44+
45+
#endif // _LIBCPP___ALGORITHM_PSTL_FRONTEND_DISPATCH

‎libcxx/include/__config

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,6 +1273,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_END_NAMESPACE_STD
12731273

12741274
// TODO: Make this a proper configuration option
12751275
#define _PSTL_PAR_BACKEND_SERIAL
1276+
#define _PSTL_CPU_BACKEND_SERIAL
12761277

12771278
#define _PSTL_PRAGMA(x) _Pragma(# x)
12781279

0 commit comments

Comments
 (0)