v1/__atomic/atomic_sync.h

0001 //===----------------------------------------------------------------------===//
0002 //
0003 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
0004 // See https://llvm.org/LICENSE.txt for license information.
0005 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
0006 //
0007 //===----------------------------------------------------------------------===//
0008
0009 #ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H
0010 #define _LIBCPP___ATOMIC_ATOMIC_SYNC_H
0011
0012 #include <__atomic/contention_t.h>
0013 #include <__atomic/memory_order.h>
0014 #include <__atomic/to_gcc_order.h>
0015 #include <__chrono/duration.h>
0016 #include <__config>
0017 #include <__memory/addressof.h>
0018 #include <__thread/poll_with_backoff.h>
0019 #include <__type_traits/conjunction.h>
0020 #include <__type_traits/decay.h>
0021 #include <__type_traits/invoke.h>
0022 #include <__type_traits/void_t.h>
0023 #include <__utility/declval.h>
0024 #include <cstring>
0025
0026 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
0027 #  pragma GCC system_header
0028 #endif
0029
0030 _LIBCPP_BEGIN_NAMESPACE_STD
0031
0032 // The customisation points to enable the following functions:
0033 // - __atomic_wait
0034 // - __atomic_wait_unless
0035 // - __atomic_notify_one
0036 // - __atomic_notify_all
0037 // Note that std::atomic<T>::wait was back-ported to C++03
0038 // The below implementations look ugly to support C++03
0039 template <class _Tp, class = void>
0040 struct __atomic_waitable_traits {
0041   template <class _AtomicWaitable>
0042   static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
0043
0044   template <class _AtomicWaitable>
0045   static void __atomic_contention_address(_AtomicWaitable&&) = delete;
0046 };
0047
0048 template <class _Tp, class = void>
0049 struct __atomic_waitable : false_type {};
0050
0051 template <class _Tp>
0052 struct __atomic_waitable< _Tp,
0053                           __void_t<decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(
0054                                        std::declval<const _Tp&>(), std::declval<memory_order>())),
0055                                    decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(
0056                                        std::declval<const _Tp&>()))> > : true_type {};
0057
0058 #if _LIBCPP_STD_VER >= 20
0059 #  if _LIBCPP_HAS_THREADS
0060
0061 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
0062 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
0063 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
0064 __libcpp_atomic_monitor(void const volatile*) _NOEXCEPT;
0065 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
0066 __libcpp_atomic_wait(void const volatile*, __cxx_contention_t) _NOEXCEPT;
0067
0068 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
0069 __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
0070 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
0071 __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
0072 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
0073 __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
0074 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
0075 __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT;
0076
0077 template <class _AtomicWaitable, class _Poll>
0078 struct __atomic_wait_backoff_impl {
0079   const _AtomicWaitable& __a_;
0080   _Poll __poll_;
0081   memory_order __order_;
0082
0083   using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
0084
0085   _LIBCPP_AVAILABILITY_SYNC
0086   _LIBCPP_HIDE_FROM_ABI bool
0087   __update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const {
0088     // In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl<int64_t>,
0089     // the platform wait is directly monitoring the atomic value itself.
0090     // `__poll_` takes the current value of the atomic as an in-out argument
0091     // to potentially modify it. After it returns, `__monitor` has a value
0092     // which can be safely waited on by `std::__libcpp_atomic_wait` without any
0093     // ABA style issues.
0094     __monitor_val = __waitable_traits::__atomic_load(__a_, __order_);
0095     return __poll_(__monitor_val);
0096   }
0097
0098   _LIBCPP_AVAILABILITY_SYNC
0099   _LIBCPP_HIDE_FROM_ABI bool
0100   __update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const {
0101     // In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t
0102     // from the global pool, the monitor comes from __libcpp_atomic_monitor
0103     __monitor_val      = std::__libcpp_atomic_monitor(__contention_address);
0104     auto __current_val = __waitable_traits::__atomic_load(__a_, __order_);
0105     return __poll_(__current_val);
0106   }
0107
0108   _LIBCPP_AVAILABILITY_SYNC
0109   _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
0110     if (__elapsed > chrono::microseconds(4)) {
0111       auto __contention_address = __waitable_traits::__atomic_contention_address(__a_);
0112       __cxx_contention_t __monitor_val;
0113       if (__update_monitor_val_and_poll(__contention_address, __monitor_val))
0114         return true;
0115       std::__libcpp_atomic_wait(__contention_address, __monitor_val);
0116     } else {
0117     } // poll
0118     return false;
0119   }
0120 };
0121
0122 // The semantics of this function are similar to `atomic`'s
0123 // `.wait(T old, std::memory_order order)`, but instead of having a hardcoded
0124 // predicate (is the loaded value unequal to `old`?), the predicate function is
0125 // specified as an argument. The loaded value is given as an in-out argument to
0126 // the predicate. If the predicate function returns `true`,
0127 // `__atomic_wait_unless` will return. If the predicate function returns
0128 // `false`, it must set the argument to its current understanding of the atomic
0129 // value. The predicate function must not return `false` spuriously.
0130 template <class _AtomicWaitable, class _Poll>
0131 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
0132 __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) {
0133   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
0134   __atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order};
0135   std::__libcpp_thread_poll_with_backoff(
0136       /* poll */
0137       [&]() {
0138         auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
0139         return __poll(__current_val);
0140       },
0141       /* backoff */ __backoff_fn);
0142 }
0143
0144 template <class _AtomicWaitable>
0145 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
0146   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
0147   std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
0148 }
0149
0150 template <class _AtomicWaitable>
0151 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
0152   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
0153   std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
0154 }
0155
0156 #  else // _LIBCPP_HAS_THREADS
0157
0158 template <class _AtomicWaitable, class _Poll>
0159 _LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) {
0160   std::__libcpp_thread_poll_with_backoff(
0161       /* poll */
0162       [&]() {
0163         auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
0164         return __poll(__current_val);
0165       },
0166       /* backoff */ __spinning_backoff_policy());
0167 }
0168
0169 template <class _AtomicWaitable>
0170 _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable&) {}
0171
0172 template <class _AtomicWaitable>
0173 _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable&) {}
0174
0175 #  endif // _LIBCPP_HAS_THREADS
0176
0177 template <typename _Tp>
0178 _LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {
0179   return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0;
0180 }
0181
0182 template <class _AtomicWaitable, class _Tp>
0183 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
0184 __atomic_wait(_AtomicWaitable& __a, _Tp __val, memory_order __order) {
0185   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
0186   std::__atomic_wait_unless(__a, __order, [&](_Tp const& __current) {
0187     return !std::__cxx_nonatomic_compare_equal(__current, __val);
0188   });
0189 }
0190
0191 #endif // C++20
0192
0193 _LIBCPP_END_NAMESPACE_STD
0194
0195 #endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H