atomic/detail/wait_ops_futex.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2020-2025 Andrey Semashev
0007  */
0008 /*!
0009  * \file   atomic/detail/wait_ops_futex.hpp
0010  *
0011  * This header contains implementation of the waiting/notifying atomic operations based on futexes.
0012  */
0013
0014 #ifndef BOOST_ATOMIC_DETAIL_WAIT_OPS_FUTEX_HPP_INCLUDED_
0015 #define BOOST_ATOMIC_DETAIL_WAIT_OPS_FUTEX_HPP_INCLUDED_
0016
0017 #include <unistd.h> // _POSIX_MONOTONIC_CLOCK
0018 #include <time.h>
0019 #include <cstdint>
0020 #include <cerrno>
0021 #include <limits>
0022 #include <chrono>
0023 #include <type_traits>
0024 #include <boost/memory_order.hpp>
0025 #include <boost/atomic/posix_clock_traits_fwd.hpp>
0026 #include <boost/atomic/detail/config.hpp>
0027 #include <boost/atomic/detail/chrono.hpp>
0028 #include <boost/atomic/detail/futex.hpp>
0029 #include <boost/atomic/detail/has_posix_clock_traits.hpp>
0030 #include <boost/atomic/detail/wait_operations_fwd.hpp>
0031 #include <boost/atomic/detail/header.hpp>
0032
0033 #ifdef BOOST_HAS_PRAGMA_ONCE
0034 #pragma once
0035 #endif
0036
0037 namespace boost {
0038 namespace atomics {
0039 namespace detail {
0040
0041 #if defined(BOOST_ATOMIC_DETAIL_FUTEX_WAIT_BITSET)
0042
0043 struct futex_wait_fallback {};
0044
0045 //! The type trait selects futex timed wait implementation tag
0046 template< typename Clock, bool = has_posix_clock_traits< Clock >::value >
0047 struct select_futex_wait
0048 {
0049     using type = futex_wait_fallback;
0050 };
0051
0052 template< clockid_t ClockId >
0053 struct futex_wait_clock;
0054
0055 template< clockid_t ClockId >
0056 struct select_futex_wait_impl
0057 {
0058     using type = futex_wait_fallback;
0059 };
0060
0061 #if defined(_POSIX_MONOTONIC_CLOCK) && (_POSIX_MONOTONIC_CLOCK >= 0)
0062 template< >
0063 struct futex_wait_clock< CLOCK_MONOTONIC >
0064 {
0065     static constexpr int futex_flags = 0;
0066 };
0067
0068 template< >
0069 struct select_futex_wait_impl< CLOCK_MONOTONIC >
0070 {
0071     using type = futex_wait_clock< CLOCK_MONOTONIC >;
0072 };
0073 #endif // defined(_POSIX_MONOTONIC_CLOCK) && (_POSIX_MONOTONIC_CLOCK >= 0)
0074
0075 #if defined(BOOST_ATOMIC_DETAIL_FUTEX_CLOCK_REALTIME)
0076 template< >
0077 struct futex_wait_clock< CLOCK_REALTIME >
0078 {
0079     static constexpr int futex_flags = BOOST_ATOMIC_DETAIL_FUTEX_CLOCK_REALTIME;
0080 };
0081
0082 template< >
0083 struct select_futex_wait_impl< CLOCK_REALTIME >
0084 {
0085     using type = futex_wait_clock< CLOCK_REALTIME >;
0086 };
0087 #endif // defined(BOOST_ATOMIC_DETAIL_FUTEX_CLOCK_REALTIME)
0088
0089 template< typename Clock >
0090 struct select_futex_wait< Clock, true >
0091 {
0092     using type = typename select_futex_wait_impl< posix_clock_traits< Clock >::clock_id >::type;
0093 };
0094
0095 #endif // defined(BOOST_ATOMIC_DETAIL_FUTEX_WAIT_BITSET)
0096
0097 template< typename Base, bool Interprocess >
0098 struct wait_operations< Base, 4u, true, Interprocess > :
0099     public Base
0100 {
0101     using base_type = Base;
0102     using storage_type = typename base_type::storage_type;
0103
0104     static constexpr bool always_has_native_wait_notify = true;
0105
0106 private:
0107     static constexpr int futex_private_flag = Interprocess ? 0 : BOOST_ATOMIC_DETAIL_FUTEX_PRIVATE_FLAG;
0108
0109 public:
0110     static BOOST_FORCEINLINE bool has_native_wait_notify(storage_type const volatile&) noexcept
0111     {
0112         return true;
0113     }
0114
0115     static BOOST_FORCEINLINE storage_type wait(storage_type const volatile& storage, storage_type old_val, memory_order order) noexcept
0116     {
0117         storage_type new_val = base_type::load(storage, order);
0118         while (new_val == old_val)
0119         {
0120             atomics::detail::futex_wait(const_cast< storage_type* >(&storage), old_val, futex_private_flag);
0121             new_val = base_type::load(storage, order);
0122         }
0123
0124         return new_val;
0125     }
0126
0127 private:
0128     template< typename Clock >
0129     static BOOST_FORCEINLINE storage_type wait_until_fallback
0130     (
0131         storage_type const volatile& storage,
0132         storage_type old_val,
0133         typename Clock::time_point timeout,
0134         typename Clock::time_point now,
0135         memory_order order,
0136         bool& timed_out
0137     ) noexcept(noexcept(Clock::now()))
0138     {
0139         futex_timespec ts{};
0140         storage_type new_val = base_type::load(storage, order);
0141         while (new_val == old_val)
0142         {
0143             const std::int64_t nsec = atomics::detail::chrono::ceil< std::chrono::nanoseconds >(timeout - now).count();
0144             if (nsec <= 0)
0145             {
0146                 timed_out = true;
0147                 break;
0148             }
0149
0150             const std::int64_t sec = nsec / 1000000000;
0151             if (BOOST_LIKELY(sec <= (std::numeric_limits< decltype(ts.tv_sec) >::max)()))
0152             {
0153                 ts.tv_sec = static_cast< decltype(ts.tv_sec) >(sec);
0154                 ts.tv_nsec = static_cast< decltype(ts.tv_nsec) >(nsec % 1000000000);
0155             }
0156             else
0157             {
0158                 ts.tv_sec = (std::numeric_limits< decltype(ts.tv_sec) >::max)();
0159                 ts.tv_nsec = static_cast< decltype(ts.tv_nsec) >(999999999);
0160             }
0161
0162             atomics::detail::futex_wait_for(const_cast< storage_type* >(&storage), old_val, ts, futex_private_flag);
0163
0164             now = Clock::now();
0165             new_val = base_type::load(storage, order);
0166         }
0167
0168         return new_val;
0169     }
0170
0171 #if defined(BOOST_ATOMIC_DETAIL_FUTEX_WAIT_BITSET)
0172
0173     template< typename Clock, clockid_t ClockId >
0174     static BOOST_FORCEINLINE storage_type wait_until_abs_timeout
0175     (
0176         storage_type const volatile& storage,
0177         storage_type old_val,
0178         typename Clock::time_point timeout,
0179         memory_order order,
0180         bool& timed_out
0181     ) noexcept(noexcept(Clock::now()))
0182     {
0183         futex_timespec ts(posix_clock_traits< Clock >::to_timespec(timeout));
0184         storage_type new_val = base_type::load(storage, order);
0185         if (BOOST_LIKELY(ts.tv_sec >= 0))
0186         {
0187             while (new_val == old_val)
0188             {
0189                 int err = atomics::detail::futex_wait_until
0190                 (
0191                     const_cast< storage_type* >(&storage),
0192                     old_val,
0193                     ts,
0194                     futex_private_flag | futex_wait_clock< ClockId >::futex_flags
0195                 );
0196                 if (err < 0)
0197                 {
0198                     err = errno;
0199                     if (err == ETIMEDOUT)
0200                     {
0201                         new_val = base_type::load(storage, order);
0202                         timed_out = new_val == old_val;
0203                         break;
0204                     }
0205
0206                     if (BOOST_UNLIKELY(err == ENOSYS))
0207                         return wait_until_fallback< Clock >(storage, old_val, timeout, Clock::now(), order, timed_out);
0208                 }
0209
0210                 new_val = base_type::load(storage, order);
0211             }
0212         }
0213         else
0214         {
0215             timed_out = new_val == old_val;
0216         }
0217
0218         return new_val;
0219     }
0220
0221     template< typename Clock >
0222     static BOOST_FORCEINLINE storage_type wait_until_dispatch
0223     (
0224         storage_type const volatile& storage,
0225         storage_type old_val,
0226         typename Clock::time_point timeout,
0227         memory_order order,
0228         bool& timed_out,
0229         futex_wait_fallback
0230     ) noexcept(noexcept(Clock::now()))
0231     {
0232         return wait_until_fallback< Clock >(storage, old_val, timeout, Clock::now(), order, timed_out);
0233     }
0234
0235     template< typename Clock, clockid_t ClockId >
0236     static BOOST_FORCEINLINE storage_type wait_until_dispatch
0237     (
0238         storage_type const volatile& storage,
0239         storage_type old_val,
0240         typename Clock::time_point timeout,
0241         memory_order order,
0242         bool& timed_out,
0243         futex_wait_clock< ClockId >
0244     ) noexcept(noexcept(Clock::now()))
0245     {
0246         return wait_until_abs_timeout< Clock, ClockId >(storage, old_val, timeout, order, timed_out);
0247     }
0248
0249 public:
0250     template< typename Clock, typename Duration >
0251     static BOOST_FORCEINLINE storage_type wait_until
0252     (
0253         storage_type const volatile& storage,
0254         storage_type old_val,
0255         std::chrono::time_point< Clock, Duration > timeout,
0256         memory_order order,
0257         bool& timed_out
0258     ) noexcept(noexcept(wait_until_dispatch< Clock >(storage, old_val, timeout, order, timed_out, typename select_futex_wait< Clock >::type())))
0259     {
0260         return wait_until_dispatch< Clock >(storage, old_val, timeout, order, timed_out, typename select_futex_wait< Clock >::type());
0261     }
0262
0263 #else // defined(BOOST_ATOMIC_DETAIL_FUTEX_WAIT_BITSET)
0264
0265 public:
0266     template< typename Clock, typename Duration >
0267     static BOOST_FORCEINLINE storage_type wait_until
0268     (
0269         storage_type const volatile& storage,
0270         storage_type old_val,
0271         std::chrono::time_point< Clock, Duration > timeout,
0272         memory_order order,
0273         bool& timed_out
0274     ) noexcept(noexcept(wait_until_fallback< Clock >(storage, old_val, timeout, Clock::now(), order, timed_out)))
0275     {
0276         return wait_until_fallback< Clock >(storage, old_val, timeout, Clock::now(), order, timed_out);
0277     }
0278
0279 #endif // defined(BOOST_ATOMIC_DETAIL_FUTEX_WAIT_BITSET)
0280
0281     template< typename Rep, typename Period >
0282     static BOOST_FORCEINLINE storage_type wait_for
0283     (
0284         storage_type const volatile& storage,
0285         storage_type old_val,
0286         std::chrono::duration< Rep, Period > timeout,
0287         memory_order order,
0288         bool& timed_out
0289     ) noexcept
0290     {
0291 #if defined(BOOST_ATOMIC_DETAIL_FUTEX_WAIT_BITSET) && defined(_POSIX_MONOTONIC_CLOCK) && (_POSIX_MONOTONIC_CLOCK >= 0)
0292         if (BOOST_LIKELY(timeout.count() >= 0))
0293         {
0294             timespec now{};
0295             if (BOOST_LIKELY(clock_gettime(CLOCK_MONOTONIC, &now) == 0))
0296             {
0297                 const std::int64_t nsec = static_cast< std::int64_t >(now.tv_nsec) + atomics::detail::chrono::ceil< std::chrono::nanoseconds >(timeout).count();
0298                 const std::int64_t sec = static_cast< std::int64_t >(now.tv_sec) + nsec / 1000000000;
0299                 if (BOOST_LIKELY(sec <= (std::numeric_limits< decltype(futex_timespec::tv_sec) >::max)()))
0300                 {
0301                     futex_timespec ts{};
0302                     ts.tv_sec = static_cast< decltype(ts.tv_sec) >(sec);
0303                     ts.tv_nsec = static_cast< decltype(ts.tv_nsec) >(nsec % 1000000000);
0304
0305                     storage_type new_val = base_type::load(storage, order);
0306                     while (new_val == old_val)
0307                     {
0308                         int err = atomics::detail::futex_wait_until(const_cast< storage_type* >(&storage), old_val, ts, futex_private_flag);
0309                         if (err < 0)
0310                         {
0311                             err = errno;
0312                             if (err == ETIMEDOUT)
0313                             {
0314                                 new_val = base_type::load(storage, order);
0315                                 timed_out = new_val == old_val;
0316                                 break;
0317                             }
0318
0319                             if (BOOST_UNLIKELY(err == ENOSYS))
0320                                 goto use_wait_until_fallback;
0321                         }
0322
0323                         new_val = base_type::load(storage, order);
0324                     }
0325
0326                     return new_val;
0327                 }
0328             }
0329         }
0330     use_wait_until_fallback:
0331 #endif // defined(BOOST_ATOMIC_DETAIL_FUTEX_WAIT_BITSET) && defined(_POSIX_MONOTONIC_CLOCK) && (_POSIX_MONOTONIC_CLOCK >= 0)
0332
0333         const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();
0334         return wait_until_fallback< std::chrono::steady_clock >(storage, old_val, now + timeout, now, order, timed_out);
0335     }
0336
0337     static BOOST_FORCEINLINE void notify_one(storage_type volatile& storage) noexcept
0338     {
0339         atomics::detail::futex_signal(const_cast< storage_type* >(&storage), futex_private_flag);
0340     }
0341
0342     static BOOST_FORCEINLINE void notify_all(storage_type volatile& storage) noexcept
0343     {
0344         atomics::detail::futex_broadcast(const_cast< storage_type* >(&storage), futex_private_flag);
0345     }
0346 };
0347
0348 } // namespace detail
0349 } // namespace atomics
0350 } // namespace boost
0351
0352 #include <boost/atomic/detail/footer.hpp>
0353
0354 #endif // BOOST_ATOMIC_DETAIL_WAIT_OPS_FUTEX_HPP_INCLUDED_