atomic/detail/futex.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2020 Andrey Semashev
0007  */
0008 /*!
0009  * \file   atomic/detail/futex.hpp
0010  *
0011  * This header defines wrappers around futex syscall.
0012  *
0013  * http://man7.org/linux/man-pages/man2/futex.2.html
0014  * https://man.openbsd.org/futex
0015  */
0016
0017 #ifndef BOOST_ATOMIC_DETAIL_FUTEX_HPP_INCLUDED_
0018 #define BOOST_ATOMIC_DETAIL_FUTEX_HPP_INCLUDED_
0019
0020 #include <boost/atomic/detail/config.hpp>
0021
0022 #ifdef BOOST_HAS_PRAGMA_ONCE
0023 #pragma once
0024 #endif
0025
0026 #if defined(__linux__) || defined(__NETBSD__) || defined(__NetBSD__)
0027
0028 #include <sys/syscall.h>
0029
0030 #if defined(SYS_futex)
0031 #define BOOST_ATOMIC_DETAIL_SYS_FUTEX SYS_futex
0032 #elif defined(SYS_futex_time64)
0033 // On some 32-bit targets (e.g. riscv32) SYS_futex is not defined and instead SYS_futex_time64 is implemented,
0034 // which is equivalent to SYS_futex but uses 64-bit time_t.
0035 #define BOOST_ATOMIC_DETAIL_SYS_FUTEX SYS_futex_time64
0036 #elif defined(__NR_futex)
0037 // Some Android NDKs (Google NDK and older Crystax.NET NDK versions) don't define SYS_futex.
0038 #define BOOST_ATOMIC_DETAIL_SYS_FUTEX __NR_futex
0039 #elif defined(SYS___futex)
0040 // NetBSD defines SYS___futex, which has slightly different parameters. Basically, it has decoupled timeout and val2 parameters:
0041 // int __futex(int *addr1, int op, int val1, const struct timespec *timeout, int *addr2, int val2, int val3);
0042 // https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/sys/sys/syscall.h
0043 // http://bxr.su/NetBSD/sys/kern/sys_futex.c
0044 #define BOOST_ATOMIC_DETAIL_SYS_FUTEX SYS___futex
0045 #define BOOST_ATOMIC_DETAIL_NETBSD_FUTEX
0046 #endif
0047
0048 #elif defined(__OpenBSD__)
0049
0050 // OpenBSD provides futex(2) function wrapper since OpenBSD 6.2 (https://man.openbsd.org/OpenBSD-6.2/futex.2).
0051 // It has also removed syscall(2) interface:
0052 // https://github.com/openbsd/src/commit/cafeb892b121ee89c39c2b940e8ccd6950f50009
0053
0054 #include <sys/param.h>
0055
0056 #if OpenBSD >= 201711
0057 #define BOOST_ATOMIC_DETAIL_OPENBSD_FUTEX
0058 #endif
0059
0060 #endif
0061
0062 #if defined(BOOST_ATOMIC_DETAIL_SYS_FUTEX) || defined(BOOST_ATOMIC_DETAIL_OPENBSD_FUTEX)
0063
0064 #include <cstddef>
0065 #if defined(__linux__)
0066 #include <linux/futex.h>
0067 #else
0068 #include <sys/futex.h>
0069 #endif
0070 #include <boost/cstdint.hpp>
0071 #include <boost/atomic/detail/intptr.hpp>
0072 #include <boost/atomic/detail/header.hpp>
0073
0074 #define BOOST_ATOMIC_DETAIL_HAS_FUTEX
0075
0076 #if defined(FUTEX_PRIVATE_FLAG)
0077 #define BOOST_ATOMIC_DETAIL_FUTEX_PRIVATE_FLAG FUTEX_PRIVATE_FLAG
0078 #elif defined(__ANDROID__)
0079 // On Android, futex.h is lacking many definitions, but the actual Linux kernel supports the API in full.
0080 #define BOOST_ATOMIC_DETAIL_FUTEX_PRIVATE_FLAG 128
0081 #else
0082 #define BOOST_ATOMIC_DETAIL_FUTEX_PRIVATE_FLAG 0
0083 #endif
0084
0085 namespace boost {
0086 namespace atomics {
0087 namespace detail {
0088
0089 //! Invokes an operation on the futex
0090 BOOST_FORCEINLINE int futex_invoke(void* addr1, int op, unsigned int val1, const void* timeout = NULL, void* addr2 = NULL, unsigned int val3 = 0) BOOST_NOEXCEPT
0091 {
0092 #if defined(BOOST_ATOMIC_DETAIL_OPENBSD_FUTEX)
0093     return ::futex
0094     (
0095         static_cast< volatile uint32_t* >(addr1),
0096         op,
0097         static_cast< int >(val1),
0098         static_cast< const struct timespec* >(timeout),
0099         static_cast< volatile uint32_t* >(addr2)
0100     );
0101 #elif defined(BOOST_ATOMIC_DETAIL_NETBSD_FUTEX)
0102     // Pass 0 in val2.
0103     return ::syscall(BOOST_ATOMIC_DETAIL_SYS_FUTEX, addr1, op, val1, timeout, addr2, 0u, val3);
0104 #else
0105     return ::syscall(BOOST_ATOMIC_DETAIL_SYS_FUTEX, addr1, op, val1, timeout, addr2, val3);
0106 #endif
0107 }
0108
0109 //! Invokes an operation on the futex
0110 BOOST_FORCEINLINE int futex_invoke(void* addr1, int op, unsigned int val1, unsigned int val2, void* addr2 = NULL, unsigned int val3 = 0) BOOST_NOEXCEPT
0111 {
0112 #if defined(BOOST_ATOMIC_DETAIL_OPENBSD_FUTEX)
0113     return ::futex
0114     (
0115         static_cast< volatile uint32_t* >(addr1),
0116         op,
0117         static_cast< int >(val1),
0118         reinterpret_cast< const struct timespec* >(static_cast< atomics::detail::uintptr_t >(val2)),
0119         static_cast< volatile uint32_t* >(addr2)
0120     );
0121 #elif defined(BOOST_ATOMIC_DETAIL_NETBSD_FUTEX)
0122     // Pass NULL in timeout.
0123     return ::syscall(BOOST_ATOMIC_DETAIL_SYS_FUTEX, addr1, op, val1, static_cast< void* >(NULL), addr2, val2, val3);
0124 #else
0125     return ::syscall(BOOST_ATOMIC_DETAIL_SYS_FUTEX, addr1, op, val1, static_cast< atomics::detail::uintptr_t >(val2), addr2, val3);
0126 #endif
0127 }
0128
0129 //! Checks that the value \c pval is \c expected and blocks
0130 BOOST_FORCEINLINE int futex_wait(void* pval, unsigned int expected) BOOST_NOEXCEPT
0131 {
0132     return futex_invoke(pval, FUTEX_WAIT, expected);
0133 }
0134
0135 //! Checks that the value \c pval is \c expected and blocks
0136 BOOST_FORCEINLINE int futex_wait_private(void* pval, unsigned int expected) BOOST_NOEXCEPT
0137 {
0138     return futex_invoke(pval, FUTEX_WAIT | BOOST_ATOMIC_DETAIL_FUTEX_PRIVATE_FLAG, expected);
0139 }
0140
0141 //! Wakes the specified number of threads waiting on the futex
0142 BOOST_FORCEINLINE int futex_signal(void* pval, unsigned int count = 1u) BOOST_NOEXCEPT
0143 {
0144     return futex_invoke(pval, FUTEX_WAKE, count);
0145 }
0146
0147 //! Wakes the specified number of threads waiting on the futex
0148 BOOST_FORCEINLINE int futex_signal_private(void* pval, unsigned int count = 1u) BOOST_NOEXCEPT
0149 {
0150     return futex_invoke(pval, FUTEX_WAKE | BOOST_ATOMIC_DETAIL_FUTEX_PRIVATE_FLAG, count);
0151 }
0152
0153 //! Wakes all threads waiting on the futex
0154 BOOST_FORCEINLINE int futex_broadcast(void* pval) BOOST_NOEXCEPT
0155 {
0156     return futex_signal(pval, (~static_cast< unsigned int >(0u)) >> 1);
0157 }
0158
0159 //! Wakes all threads waiting on the futex
0160 BOOST_FORCEINLINE int futex_broadcast_private(void* pval) BOOST_NOEXCEPT
0161 {
0162     return futex_signal_private(pval, (~static_cast< unsigned int >(0u)) >> 1);
0163 }
0164
0165 //! Wakes the wake_count threads waiting on the futex pval1 and requeues up to requeue_count of the blocked threads onto another futex pval2
0166 BOOST_FORCEINLINE int futex_requeue(void* pval1, void* pval2, unsigned int wake_count = 1u, unsigned int requeue_count = (~static_cast< unsigned int >(0u)) >> 1) BOOST_NOEXCEPT
0167 {
0168     return futex_invoke(pval1, FUTEX_REQUEUE, wake_count, requeue_count, pval2);
0169 }
0170
0171 //! Wakes the wake_count threads waiting on the futex pval1 and requeues up to requeue_count of the blocked threads onto another futex pval2
0172 BOOST_FORCEINLINE int futex_requeue_private(void* pval1, void* pval2, unsigned int wake_count = 1u, unsigned int requeue_count = (~static_cast< unsigned int >(0u)) >> 1) BOOST_NOEXCEPT
0173 {
0174     return futex_invoke(pval1, FUTEX_REQUEUE | BOOST_ATOMIC_DETAIL_FUTEX_PRIVATE_FLAG, wake_count, requeue_count, pval2);
0175 }
0176
0177 } // namespace detail
0178 } // namespace atomics
0179 } // namespace boost
0180
0181 #include <boost/atomic/detail/footer.hpp>
0182
0183 #endif // defined(BOOST_ATOMIC_DETAIL_SYS_FUTEX) || defined(BOOST_ATOMIC_DETAIL_OPENBSD_FUTEX)
0184
0185 #endif // BOOST_ATOMIC_DETAIL_FUTEX_HPP_INCLUDED_