Warning, /include/c++/v1/barrier is written in an unsupported language. File is not indexed.
0001 // -*- C++ -*-
0002 //===----------------------------------------------------------------------===//
0003 //
0004 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
0005 // See https://llvm.org/LICENSE.txt for license information.
0006 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
0007 //
0008 //===----------------------------------------------------------------------===//
0009
0010 #ifndef _LIBCPP_BARRIER
0011 #define _LIBCPP_BARRIER
0012
0013 /*
0014 barrier synopsis
0015
0016 namespace std
0017 {
0018
0019 template<class CompletionFunction = see below>
0020 class barrier // since C++20
0021 {
0022 public:
0023 using arrival_token = see below;
0024
0025 static constexpr ptrdiff_t max() noexcept;
0026
0027 constexpr explicit barrier(ptrdiff_t phase_count,
0028 CompletionFunction f = CompletionFunction());
0029 ~barrier();
0030
0031 barrier(const barrier&) = delete;
0032 barrier& operator=(const barrier&) = delete;
0033
0034 [[nodiscard]] arrival_token arrive(ptrdiff_t update = 1);
0035 void wait(arrival_token&& arrival) const;
0036
0037 void arrive_and_wait();
0038 void arrive_and_drop();
0039
0040 private:
0041 CompletionFunction completion; // exposition only
0042 };
0043
0044 }
0045
0046 */
0047
0048 #if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
0049 # include <__cxx03/barrier>
0050 #else
0051 # include <__config>
0052
0053 # if _LIBCPP_HAS_THREADS
0054
0055 # include <__assert>
0056 # include <__atomic/atomic.h>
0057 # include <__atomic/memory_order.h>
0058 # include <__cstddef/ptrdiff_t.h>
0059 # include <__memory/unique_ptr.h>
0060 # include <__thread/poll_with_backoff.h>
0061 # include <__thread/timed_backoff_policy.h>
0062 # include <__utility/move.h>
0063 # include <cstdint>
0064 # include <limits>
0065 # include <version>
0066
0067 # if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
0068 # pragma GCC system_header
0069 # endif
0070
0071 _LIBCPP_PUSH_MACROS
0072 # include <__undef_macros>
0073
0074 # if _LIBCPP_STD_VER >= 20
0075
0076 _LIBCPP_BEGIN_NAMESPACE_STD
0077
0078 struct __empty_completion {
0079 inline _LIBCPP_HIDE_FROM_ABI void operator()() noexcept {}
0080 };
0081
0082 /*
0083
0084 The default implementation of __barrier_base is a classic tree barrier.
0085
0086 It looks different from literature pseudocode for two main reasons:
0087 1. Threads that call into std::barrier functions do not provide indices,
0088 so a numbering step is added before the actual barrier algorithm,
0089 appearing as an N+1 round to the N rounds of the tree barrier.
0090 2. A great deal of attention has been paid to avoid cache line thrashing
0091 by flattening the tree structure into cache-line sized arrays, that
0092 are indexed in an efficient way.
0093
0094 */
0095
0096 using __barrier_phase_t _LIBCPP_NODEBUG = uint8_t;
0097
0098 class __barrier_algorithm_base;
0099
0100 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __barrier_algorithm_base*
0101 __construct_barrier_algorithm_base(ptrdiff_t& __expected);
0102
0103 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI bool
0104 __arrive_barrier_algorithm_base(__barrier_algorithm_base* __barrier, __barrier_phase_t __old_phase) noexcept;
0105
0106 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
0107 __destroy_barrier_algorithm_base(__barrier_algorithm_base* __barrier) noexcept;
0108
0109 template <class _CompletionF>
0110 class __barrier_base {
0111 ptrdiff_t __expected_;
0112 unique_ptr<__barrier_algorithm_base, void (*)(__barrier_algorithm_base*)> __base_;
0113 atomic<ptrdiff_t> __expected_adjustment_;
0114 _CompletionF __completion_;
0115 atomic<__barrier_phase_t> __phase_;
0116
0117 public:
0118 using arrival_token = __barrier_phase_t;
0119
0120 static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return numeric_limits<ptrdiff_t>::max(); }
0121
0122 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI
0123 __barrier_base(ptrdiff_t __expected, _CompletionF __completion = _CompletionF())
0124 : __expected_(__expected),
0125 __base_(std::__construct_barrier_algorithm_base(this->__expected_), &__destroy_barrier_algorithm_base),
0126 __expected_adjustment_(0),
0127 __completion_(std::move(__completion)),
0128 __phase_(0) {}
0129 [[nodiscard]] _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update) {
0130 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
0131 __update <= __expected_, "update is greater than the expected count for the current barrier phase");
0132
0133 auto const __old_phase = __phase_.load(memory_order_relaxed);
0134 for (; __update; --__update)
0135 if (__arrive_barrier_algorithm_base(__base_.get(), __old_phase)) {
0136 __completion_();
0137 __expected_ += __expected_adjustment_.load(memory_order_relaxed);
0138 __expected_adjustment_.store(0, memory_order_relaxed);
0139 __phase_.store(__old_phase + 2, memory_order_release);
0140 __phase_.notify_all();
0141 }
0142 return __old_phase;
0143 }
0144 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __old_phase) const {
0145 auto const __test_fn = [this, __old_phase]() -> bool { return __phase_.load(memory_order_acquire) != __old_phase; };
0146 std::__libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy());
0147 }
0148 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() {
0149 __expected_adjustment_.fetch_sub(1, memory_order_relaxed);
0150 (void)arrive(1);
0151 }
0152 };
0153
0154 template <class _CompletionF = __empty_completion>
0155 class barrier {
0156 __barrier_base<_CompletionF> __b_;
0157
0158 public:
0159 using arrival_token = typename __barrier_base<_CompletionF>::arrival_token;
0160
0161 static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return __barrier_base<_CompletionF>::max(); }
0162
0163 _LIBCPP_AVAILABILITY_SYNC
0164 _LIBCPP_HIDE_FROM_ABI explicit barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF())
0165 : __b_(__count, std::move(__completion)) {
0166 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
0167 __count >= 0,
0168 "barrier::barrier(ptrdiff_t, CompletionFunction): barrier cannot be initialized with a negative value");
0169 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
0170 __count <= max(),
0171 "barrier::barrier(ptrdiff_t, CompletionFunction): barrier cannot be initialized with "
0172 "a value greater than max()");
0173 }
0174
0175 barrier(barrier const&) = delete;
0176 barrier& operator=(barrier const&) = delete;
0177
0178 [[nodiscard]] _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update = 1) {
0179 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update > 0, "barrier:arrive must be called with a value greater than 0");
0180 return __b_.arrive(__update);
0181 }
0182 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __phase) const {
0183 __b_.wait(std::move(__phase));
0184 }
0185 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait() { wait(arrive()); }
0186 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() { __b_.arrive_and_drop(); }
0187 };
0188
0189 _LIBCPP_END_NAMESPACE_STD
0190
0191 # endif // _LIBCPP_STD_VER >= 20
0192
0193 _LIBCPP_POP_MACROS
0194
0195 # endif // _LIBCPP_HAS_THREADS
0196
0197 # if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
0198 # include <atomic>
0199 # include <concepts>
0200 # include <iterator>
0201 # include <memory>
0202 # include <stdexcept>
0203 # include <variant>
0204 # endif
0205 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
0206
0207 #endif // _LIBCPP_BARRIER
