atomic/detail/wait_ops_darwin_ulock.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2021 Andrey Semashev
0007  */
0008 /*!
0009  * \file   atomic/detail/wait_ops_darwin_ulock.hpp
0010  *
0011  * This header contains implementation of the waiting/notifying atomic operations based on Darwin systems using ulock syscalls.
0012  *
0013  * https://github.com/apple/darwin-xnu/blob/master/bsd/sys/ulock.h
0014  * https://github.com/apple/darwin-xnu/blob/master/bsd/kern/sys_ulock.c
0015  */
0016
0017 #ifndef BOOST_ATOMIC_DETAIL_WAIT_OPS_DARWIN_ULOCK_HPP_INCLUDED_
0018 #define BOOST_ATOMIC_DETAIL_WAIT_OPS_DARWIN_ULOCK_HPP_INCLUDED_
0019
0020 #include <stdint.h>
0021 #include <cerrno>
0022 #include <boost/memory_order.hpp>
0023 #include <boost/atomic/detail/config.hpp>
0024 #include <boost/atomic/detail/wait_capabilities.hpp>
0025 #include <boost/atomic/detail/wait_operations_fwd.hpp>
0026 #include <boost/atomic/detail/header.hpp>
0027
0028 #ifdef BOOST_HAS_PRAGMA_ONCE
0029 #pragma once
0030 #endif
0031
0032 namespace boost {
0033 namespace atomics {
0034 namespace detail {
0035
0036 extern "C" {
0037 // Timeout is in microseconds with zero meaning no timeout
0038 int __ulock_wait(uint32_t operation, void* addr, uint64_t value, uint32_t timeout);
0039 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_WAIT2)
0040 // Timeout is in nanoseconds with zero meaning no timeout
0041 int __ulock_wait2(uint32_t operation, void* addr, uint64_t value, uint64_t timeout, uint64_t value2);
0042 #endif // defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_WAIT2)
0043 int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value);
0044 } // extern "C"
0045
0046 enum ulock_op
0047 {
0048     ulock_op_compare_and_wait = 1,
0049 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0050     ulock_op_compare_and_wait_shared = 3,
0051 #endif // defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0052 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK64)
0053     ulock_op_compare_and_wait64 = 5,
0054 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0055     ulock_op_compare_and_wait64_shared = 6,
0056 #endif // defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0057 #endif // defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK64)
0058
0059     // Flags for __ulock_wake
0060     ulock_flag_wake_all = 0x00000100,
0061
0062     // Generic flags
0063     ulock_flag_no_errno = 0x01000000
0064 };
0065
0066 template< typename Base, uint32_t Opcode >
0067 struct wait_operations_darwin_ulock_common :
0068     public Base
0069 {
0070     typedef Base base_type;
0071     typedef typename base_type::storage_type storage_type;
0072
0073     static BOOST_CONSTEXPR_OR_CONST bool always_has_native_wait_notify = true;
0074
0075     static BOOST_FORCEINLINE bool has_native_wait_notify(storage_type const volatile&) BOOST_NOEXCEPT
0076     {
0077         return true;
0078     }
0079
0080     static BOOST_FORCEINLINE storage_type wait(storage_type const volatile& storage, storage_type old_val, memory_order order) BOOST_NOEXCEPT
0081     {
0082         storage_type new_val = base_type::load(storage, order);
0083         while (new_val == old_val)
0084         {
0085 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_WAIT2)
0086             __ulock_wait2(Opcode | ulock_flag_no_errno, const_cast< storage_type* >(&storage), old_val, 0u, 0u);
0087 #else
0088             __ulock_wait(Opcode | ulock_flag_no_errno, const_cast< storage_type* >(&storage), old_val, 0u);
0089 #endif
0090             new_val = base_type::load(storage, order);
0091         }
0092
0093         return new_val;
0094     }
0095
0096     static BOOST_FORCEINLINE void notify_one(storage_type volatile& storage) BOOST_NOEXCEPT
0097     {
0098         while (true)
0099         {
0100             const int res = __ulock_wake(Opcode | ulock_flag_no_errno, const_cast< storage_type* >(&storage), 0u);
0101             if (BOOST_LIKELY(res != -EINTR))
0102                 break;
0103         }
0104     }
0105
0106     static BOOST_FORCEINLINE void notify_all(storage_type volatile& storage) BOOST_NOEXCEPT
0107     {
0108         while (true)
0109         {
0110             const int res = __ulock_wake(Opcode | ulock_flag_wake_all | ulock_flag_no_errno, const_cast< storage_type* >(&storage), 0u);
0111             if (BOOST_LIKELY(res != -EINTR))
0112                 break;
0113         }
0114     }
0115 };
0116
0117 template< typename Base >
0118 struct wait_operations< Base, sizeof(uint32_t), true, false > :
0119     public wait_operations_darwin_ulock_common< Base, ulock_op_compare_and_wait >
0120 {
0121 };
0122
0123 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0124
0125 template< typename Base >
0126 struct wait_operations< Base, sizeof(uint32_t), true, true > :
0127     public wait_operations_darwin_ulock_common< Base, ulock_op_compare_and_wait_shared >
0128 {
0129 };
0130
0131 #endif // defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0132
0133 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK64)
0134
0135 template< typename Base >
0136 struct wait_operations< Base, sizeof(uint64_t), true, false > :
0137     public wait_operations_darwin_ulock_common< Base, ulock_op_compare_and_wait64 >
0138 {
0139 };
0140
0141 #if defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0142
0143 template< typename Base >
0144 struct wait_operations< Base, sizeof(uint64_t), true, true > :
0145     public wait_operations_darwin_ulock_common< Base, ulock_op_compare_and_wait64_shared >
0146 {
0147 };
0148
0149 #endif // defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK_SHARED)
0150 #endif // defined(BOOST_ATOMIC_DETAIL_HAS_DARWIN_ULOCK64)
0151
0152 } // namespace detail
0153 } // namespace atomics
0154 } // namespace boost
0155
0156 #include <boost/atomic/detail/footer.hpp>
0157
0158 #endif // BOOST_ATOMIC_DETAIL_WAIT_OPS_DARWIN_ULOCK_HPP_INCLUDED_