Back to home page

EIC code displayed by LXR
 
 

    

File indexing completed on 2026-08-30 08:22:15 

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2020-2025 Andrey Semashev
0007  */
0008 /*!
0009  * \file   atomic/detail/wait_ops_freebsd_umtx.hpp
0010  *
0011  * This header contains implementation of the waiting/notifying atomic operations based on FreeBSD _umtx_op syscall.
0012  * https://man.freebsd.org/cgi/man.cgi?query=_umtx_op&apropos=0&sektion=2&manpath=FreeBSD+11.0-RELEASE&arch=default&format=html
0013  */
0014 
0015 #ifndef BOOST_ATOMIC_DETAIL_WAIT_OPS_FREEBSD_UMTX_HPP_INCLUDED_
0016 #define BOOST_ATOMIC_DETAIL_WAIT_OPS_FREEBSD_UMTX_HPP_INCLUDED_
0017 
0018 #include <sys/umtx.h>
0019 #include <time.h>
0020 #include <cstdint>
0021 #include <cerrno>
0022 #include <limits>
0023 #include <chrono>
0024 #include <type_traits>
0025 #include <boost/memory_order.hpp>
0026 #include <boost/atomic/posix_clock_traits_fwd.hpp>
0027 #include <boost/atomic/detail/config.hpp>
0028 #if defined(UMTX_ABSTIME)
0029 #include <boost/atomic/detail/intptr.hpp>
0030 #endif
0031 #include <boost/atomic/detail/chrono.hpp>
0032 #include <boost/atomic/detail/int_sizes.hpp>
0033 #include <boost/atomic/detail/has_posix_clock_traits.hpp>
0034 #include <boost/atomic/detail/wait_operations_fwd.hpp>
0035 #include <boost/atomic/detail/header.hpp>
0036 
0037 #ifdef BOOST_HAS_PRAGMA_ONCE
0038 #pragma once
0039 #endif
0040 
0041 namespace boost {
0042 namespace atomics {
0043 namespace detail {
0044 
0045 // Brief evolution of _umtx_op in FreeBSD:
0046 //
0047 // * FreeBSD 6.0.
0048 //   Initial version that supports UMTX_OP_WAIT and UMTX_OP_WAKE for long-sized futexes. Supports timed waits, where initially the timeout was absolute
0049 //   against CLOCK_REALTIME (https://github.com/freebsd/freebsd-src/commit/cc1000ac5b235516dd312340fca34e8847add3d0), but later was changed to relative
0050 //   timeouts that count against CLOCK_MONOTONIC (https://github.com/freebsd/freebsd-src/commit/b7be40d612b794ea9165b71a8afe07777dc9d31a). Presumably,
0051 //   the initial version with absolute timeouts was not released, so we can ignore it.
0052 // * FreeBSD 8.0.
0053 //   Added UMTX_OP_WAIT_UINT (https://github.com/freebsd/freebsd-src/commit/110de0cf17923839e434ead831b7a7c74a7ce102) for int-sized futexes, as well as
0054 //   UMTX_OP_WAIT_UINT_PRIVATE and UMTX_OP_WAKE_PRIVATE (https://github.com/freebsd/freebsd-src/commit/727158f6f64df04094d41ca5ee4b0641308c39d0) for
0055 //   process-local futexes.
0056 // * FreeBSD 10.0.
0057 //   Added UMTX_ABSTIME (https://github.com/freebsd/freebsd-src/commit/df1f1bae9eac5f3f838c8939e4de2c5458aba001). By default, relative timeouts are now
0058 //   counted against CLOCK_REALTIME (a breaking change), but the caller can now supply the timeout in the form of struct _umtx_time, which allows for
0059 //   specifying flags (where UMTX_ABSTIME means absolute timeout) and the clock id. Presumably, all clocks supported by clock_gettime are supported
0060 //   by this new API. Whether the caller is using this new API or the legacy API with a relative timeout in struct timespec is indicated by the uaddr
0061 //   argument of _umtx_op, which must be the size of the timeout struct casted to a pointer. Previous FreeBSD releases ignored this argument, so
0062 //   new binaries running on old FreeBSD versions will silently misbehave (and old binaries on new FreeBSD as well, due to the CLOCK_REALTIME change).
0063 
0064 #if defined(UMTX_OP_WAIT_UINT) || defined(UMTX_OP_WAIT)
0065 
0066 template< typename Base, typename UmtxOps >
0067 struct wait_operations_freebsd_umtx_common :
0068     public Base
0069 {
0070     using base_type = Base;
0071     using storage_type = typename base_type::storage_type;
0072 
0073     static constexpr bool always_has_native_wait_notify = true;
0074 
0075 private:
0076     using umtx_ops = UmtxOps;
0077 
0078 public:
0079     static BOOST_FORCEINLINE bool has_native_wait_notify(storage_type const volatile&) noexcept
0080     {
0081         return true;
0082     }
0083 
0084     static BOOST_FORCEINLINE storage_type wait(storage_type const volatile& storage, storage_type old_val, memory_order order) noexcept
0085     {
0086         storage_type new_val = base_type::load(storage, order);
0087         while (new_val == old_val)
0088         {
0089             _umtx_op(const_cast< storage_type* >(&storage), umtx_ops::wait_op, old_val, nullptr, nullptr);
0090             new_val = base_type::load(storage, order);
0091         }
0092 
0093         return new_val;
0094     }
0095 
0096 #if defined(UMTX_ABSTIME)
0097 
0098 private:
0099     template< typename Clock >
0100     static BOOST_FORCEINLINE storage_type wait_until_fallback
0101     (
0102         storage_type const volatile& storage,
0103         storage_type old_val,
0104         typename Clock::time_point timeout,
0105         typename Clock::time_point now,
0106         memory_order order,
0107         bool& timed_out
0108     ) noexcept(noexcept(Clock::now()))
0109     {
0110         _umtx_time umt{};
0111         umt._clockid = CLOCK_MONOTONIC;
0112         storage_type new_val = base_type::load(storage, order);
0113         while (new_val == old_val)
0114         {
0115             const std::int64_t nsec = atomics::detail::chrono::ceil< std::chrono::nanoseconds >(timeout - now).count();
0116             if (nsec <= 0)
0117             {
0118                 timed_out = true;
0119                 break;
0120             }
0121 
0122             const std::int64_t sec = nsec / 1000000000;
0123             if (BOOST_LIKELY(sec <= (std::numeric_limits< decltype(umt._timeout.tv_sec) >::max)()))
0124             {
0125                 umt._timeout.tv_sec = static_cast< decltype(umt._timeout.tv_sec) >(sec);
0126                 umt._timeout.tv_nsec = static_cast< decltype(umt._timeout.tv_nsec) >(nsec % 1000000000);
0127             }
0128             else
0129             {
0130                 umt._timeout.tv_sec = (std::numeric_limits< decltype(umt._timeout.tv_sec) >::max)();
0131                 umt._timeout.tv_nsec = static_cast< decltype(umt._timeout.tv_nsec) >(999999999);
0132             }
0133 
0134             _umtx_op
0135             (
0136                 const_cast< storage_type* >(&storage),
0137                 umtx_ops::wait_op,
0138                 old_val,
0139                 reinterpret_cast< void* >(static_cast< uintptr_t >(sizeof(umt))),
0140                 &umt
0141             );
0142 
0143             now = Clock::now();
0144             new_val = base_type::load(storage, order);
0145         }
0146 
0147         return new_val;
0148     }
0149 
0150     static BOOST_FORCEINLINE storage_type wait_until_abs_timeout
0151     (
0152         storage_type const volatile& storage,
0153         storage_type old_val,
0154         _umtx_time const& umt,
0155         memory_order order,
0156         bool& timed_out
0157     ) noexcept
0158     {
0159         storage_type new_val = base_type::load(storage, order);
0160         while (new_val == old_val)
0161         {
0162             int err = _umtx_op
0163             (
0164                 const_cast< storage_type* >(&storage),
0165                 umtx_ops::wait_op,
0166                 old_val,
0167                 reinterpret_cast< void* >(static_cast< uintptr_t >(sizeof(umt))),
0168                 const_cast< _umtx_time* >(&umt)
0169             );
0170             if (err < 0)
0171             {
0172                 err = errno;
0173                 if (err == ETIMEDOUT)
0174                 {
0175                     new_val = base_type::load(storage, order);
0176                     timed_out = new_val == old_val;
0177                     break;
0178                 }
0179             }
0180 
0181             new_val = base_type::load(storage, order);
0182         }
0183 
0184         return new_val;
0185     }
0186 
0187     template< typename Clock >
0188     static BOOST_FORCEINLINE storage_type wait_until_dispatch
0189     (
0190         storage_type const volatile& storage,
0191         storage_type old_val,
0192         typename Clock::time_point timeout,
0193         memory_order order,
0194         bool& timed_out,
0195         std::false_type
0196     ) noexcept(noexcept(Clock::now()))
0197     {
0198         return wait_until_fallback< Clock >(storage, old_val, timeout, Clock::now(), order, timed_out);
0199     }
0200 
0201     template< typename Clock >
0202     static BOOST_FORCEINLINE storage_type wait_until_dispatch
0203     (
0204         storage_type const volatile& storage,
0205         storage_type old_val,
0206         typename Clock::time_point timeout,
0207         memory_order order,
0208         bool& timed_out,
0209         std::true_type
0210     ) noexcept
0211     {
0212         _umtx_time umt{};
0213         umt._timeout = posix_clock_traits< Clock >::to_timespec(timeout);
0214         if (BOOST_LIKELY(umt._timeout.tv_sec >= 0))
0215         {
0216             umt._flags = UMTX_ABSTIME;
0217             umt._clockid = posix_clock_traits< Clock >::clock_id;
0218             return wait_until_abs_timeout(storage, old_val, umt, order, timed_out);
0219         }
0220         else
0221         {
0222             storage_type new_val = base_type::load(storage, order);
0223             timed_out = new_val == old_val;
0224             return new_val;
0225         }
0226     }
0227 
0228 public:
0229     template< typename Clock, typename Duration >
0230     static BOOST_FORCEINLINE storage_type wait_until
0231     (
0232         storage_type const volatile& storage,
0233         storage_type old_val,
0234         std::chrono::time_point< Clock, Duration > timeout,
0235         memory_order order,
0236         bool& timed_out
0237     ) noexcept(noexcept(wait_until_dispatch< Clock >(
0238         storage, old_val, timeout, order, timed_out, std::integral_constant< bool, has_posix_clock_traits< Clock >::value >())))
0239     {
0240         return wait_until_dispatch< Clock >(storage, old_val, timeout, order, timed_out, std::integral_constant< bool, has_posix_clock_traits< Clock >::value >());
0241     }
0242 
0243     template< typename Rep, typename Period >
0244     static BOOST_FORCEINLINE storage_type wait_for
0245     (
0246         storage_type const volatile& storage,
0247         storage_type old_val,
0248         std::chrono::duration< Rep, Period > timeout,
0249         memory_order order,
0250         bool& timed_out
0251     ) noexcept
0252     {
0253         if (BOOST_LIKELY(timeout.count() >= 0))
0254         {
0255             _umtx_time umt{};
0256             if (BOOST_LIKELY(clock_gettime(CLOCK_MONOTONIC, &umt._timeout) == 0))
0257             {
0258                 const std::int64_t nsec = static_cast< std::int64_t >(umt._timeout.tv_nsec) + atomics::detail::chrono::ceil< std::chrono::nanoseconds >(timeout).count();
0259                 const std::int64_t sec = static_cast< std::int64_t >(umt._timeout.tv_sec) + nsec / 1000000000;
0260                 if (BOOST_LIKELY(sec <= (std::numeric_limits< decltype(timespec::tv_sec) >::max)()))
0261                 {
0262                     umt._timeout.tv_sec = static_cast< decltype(umt._timeout.tv_sec) >(sec);
0263                     umt._timeout.tv_nsec = static_cast< decltype(umt._timeout.tv_nsec) >(nsec % 1000000000);
0264                     umt._flags = UMTX_ABSTIME;
0265                     umt._clockid = CLOCK_MONOTONIC;
0266                     return wait_until_abs_timeout(storage, old_val, umt, order, timed_out);
0267                 }
0268             }
0269         }
0270 
0271         const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();
0272         return wait_until_fallback< std::chrono::steady_clock >(storage, old_val, now + timeout, now, order, timed_out);
0273     }
0274 
0275 #else // defined(UMTX_ABSTIME)
0276 
0277 private:
0278     template< typename Clock >
0279     static BOOST_FORCEINLINE storage_type wait_until_impl
0280     (
0281         storage_type const volatile& storage,
0282         storage_type old_val,
0283         typename Clock::time_point timeout,
0284         typename Clock::time_point now,
0285         memory_order order,
0286         bool& timed_out
0287     ) noexcept(noexcept(Clock::now()))
0288     {
0289         timespec ts{};
0290         storage_type new_val = base_type::load(storage, order);
0291         while (new_val == old_val)
0292         {
0293             const std::int64_t nsec = atomics::detail::chrono::ceil< std::chrono::nanoseconds >(timeout - now).count();
0294             if (nsec <= 0)
0295             {
0296                 timed_out = true;
0297                 break;
0298             }
0299 
0300             const std::int64_t sec = nsec / 1000000000;
0301             if (BOOST_LIKELY(sec <= (std::numeric_limits< decltype(ts.tv_sec) >::max)()))
0302             {
0303                 ts.tv_sec = static_cast< decltype(ts.tv_sec) >(sec);
0304                 ts.tv_nsec = static_cast< decltype(ts.tv_nsec) >(nsec % 1000000000);
0305             }
0306             else
0307             {
0308                 ts.tv_sec = (std::numeric_limits< decltype(ts.tv_sec) >::max)();
0309                 ts.tv_nsec = static_cast< decltype(ts.tv_nsec) >(999999999);
0310             }
0311 
0312             _umtx_op(const_cast< storage_type* >(&storage), umtx_ops::wait_op, old_val, nullptr, &ts);
0313 
0314             now = Clock::now();
0315             new_val = base_type::load(storage, order);
0316         }
0317 
0318         return new_val;
0319     }
0320 
0321 public:
0322     template< typename Clock, typename Duration >
0323     static BOOST_FORCEINLINE storage_type wait_until
0324     (
0325         storage_type const volatile& storage,
0326         storage_type old_val,
0327         std::chrono::time_point< Clock, Duration > timeout,
0328         memory_order order,
0329         bool& timed_out
0330     ) noexcept(noexcept(Clock::now()))
0331     {
0332         return wait_until_impl< Clock >(storage, old_val, timeout, Clock::now(), order, timed_out);
0333     }
0334 
0335     template< typename Rep, typename Period >
0336     static BOOST_FORCEINLINE storage_type wait_for
0337     (
0338         storage_type const volatile& storage,
0339         storage_type old_val,
0340         std::chrono::duration< Rep, Period > timeout,
0341         memory_order order,
0342         bool& timed_out
0343     ) noexcept
0344     {
0345         const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();
0346         return wait_until_impl< std::chrono::steady_clock >(storage, old_val, now + timeout, now, order, timed_out);
0347     }
0348 
0349 #endif // defined(UMTX_ABSTIME)
0350 
0351     static BOOST_FORCEINLINE void notify_one(storage_type volatile& storage) noexcept
0352     {
0353         _umtx_op(const_cast< storage_type* >(&storage), umtx_ops::wake_op, 1u, nullptr, nullptr);
0354     }
0355 
0356     static BOOST_FORCEINLINE void notify_all(storage_type volatile& storage) noexcept
0357     {
0358         _umtx_op(const_cast< storage_type* >(&storage), umtx_ops::wake_op, (~static_cast< unsigned int >(0u)) >> 1u, nullptr, nullptr);
0359     }
0360 };
0361 
0362 #if defined(UMTX_OP_WAIT_UINT)
0363 
0364 template< bool Interprocess >
0365 struct uint_umtx_ops
0366 {
0367 #if defined(UMTX_OP_WAIT_UINT_PRIVATE) && defined(UMTX_OP_WAKE_PRIVATE)
0368     static constexpr int wait_op = Interprocess ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE;
0369     static constexpr int wake_op = Interprocess ? UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE;
0370 #else
0371     static constexpr int wait_op = UMTX_OP_WAIT_UINT;
0372     static constexpr int wake_op = UMTX_OP_WAKE;
0373 #endif
0374 };
0375 
0376 template< typename Base, bool Interprocess >
0377 struct wait_operations< Base, sizeof(unsigned int), true, Interprocess > :
0378     public wait_operations_freebsd_umtx_common< Base, uint_umtx_ops< Interprocess > >
0379 {
0380 };
0381 
0382 #endif // defined(UMTX_OP_WAIT_UINT)
0383 
0384 #if defined(UMTX_OP_WAIT) && (!defined(UMTX_OP_WAIT_UINT) || BOOST_ATOMIC_DETAIL_SIZEOF_INT < BOOST_ATOMIC_DETAIL_SIZEOF_LONG)
0385 
0386 struct ulong_umtx_ops
0387 {
0388     static constexpr int wait_op = UMTX_OP_WAIT;
0389     static constexpr int wake_op = UMTX_OP_WAKE;
0390 };
0391 
0392 template< typename Base, bool Interprocess >
0393 struct wait_operations< Base, sizeof(unsigned long), true, Interprocess > :
0394     public wait_operations_freebsd_umtx_common< Base, ulong_umtx_ops >
0395 {
0396 };
0397 
0398 #endif // defined(UMTX_OP_WAIT) && (!defined(UMTX_OP_WAIT_UINT) || BOOST_ATOMIC_DETAIL_SIZEOF_INT < BOOST_ATOMIC_DETAIL_SIZEOF_LONG)
0399 
0400 #endif // defined(UMTX_OP_WAIT_UINT) || defined(UMTX_OP_WAIT)
0401 
0402 } // namespace detail
0403 } // namespace atomics
0404 } // namespace boost
0405 
0406 #include <boost/atomic/detail/footer.hpp>
0407 
0408 #endif // BOOST_ATOMIC_DETAIL_WAIT_OPS_FREEBSD_UMTX_HPP_INCLUDED_
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.3.7 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!