upb/port/atomic.h

0001 // Protocol Buffers - Google's data interchange format
0002 // Copyright 2023 Google LLC.  All rights reserved.
0003 //
0004 // Use of this source code is governed by a BSD-style
0005 // license that can be found in the LICENSE file or at
0006 // https://developers.google.com/open-source/licenses/bsd
0007
0008 #ifndef UPB_PORT_ATOMIC_H_
0009 #define UPB_PORT_ATOMIC_H_
0010
0011 #include "upb/port/def.inc"
0012
0013 #ifdef UPB_USE_C11_ATOMICS
0014
0015 // IWYU pragma: begin_exports
0016 #include <stdatomic.h>
0017 #include <stdbool.h>
0018 // IWYU pragma: end_exports
0019
0020 #define upb_Atomic_Init(addr, val) atomic_init(addr, val)
0021 #define upb_Atomic_Load(addr, order) atomic_load_explicit(addr, order)
0022 #define upb_Atomic_Store(addr, val, order) \
0023   atomic_store_explicit(addr, val, order)
0024 #define upb_Atomic_Exchange(addr, val, order) \
0025   atomic_exchange_explicit(addr, val, order)
0026 #define upb_Atomic_CompareExchangeStrong(addr, expected, desired,      \
0027                                          success_order, failure_order) \
0028   atomic_compare_exchange_strong_explicit(addr, expected, desired,     \
0029                                           success_order, failure_order)
0030 #define upb_Atomic_CompareExchangeWeak(addr, expected, desired, success_order, \
0031                                        failure_order)                          \
0032   atomic_compare_exchange_weak_explicit(addr, expected, desired,               \
0033                                         success_order, failure_order)
0034
0035 #elif defined(UPB_USE_MSC_ATOMICS)
0036 #include <intrin.h>
0037 #include <stdbool.h>
0038 #include <stdint.h>
0039
0040 #define upb_Atomic_Init(addr, val) (*(addr) = val)
0041
0042 #if defined(_WIN64)
0043 // MSVC, without C11 atomics, does not have any way in pure C to force
0044 // load-acquire store-release behavior, so we hack it with exchanges.
0045 #pragma intrinsic(_InterlockedExchange64)
0046 #define upb_Atomic_Store(addr, val, order) \
0047   (void)_InterlockedExchange64((uint64_t volatile *)addr, (uint64_t)val)
0048
0049 #pragma intrinsic(_InterlockedCompareExchange64)
0050 static uintptr_t upb_Atomic_LoadMsc(uint64_t volatile *addr) {
0051   // Compare exchange with an unlikely value reduces the risk of a spurious
0052   // (but harmless) store
0053   return _InterlockedCompareExchange64(addr, 0xDEADC0DEBAADF00D,
0054                                        0xDEADC0DEBAADF00D);
0055 }
0056 // If _Generic is available, use it to avoid emitting a "'uintptr_t' differs in
0057 // levels of indirection from 'void *'" or -Wint-conversion compiler warning.
0058 #if __STDC_VERSION__ >= 201112L
0059 #define upb_Atomic_Load(addr, order)                           \
0060   _Generic(addr,                                               \
0061       UPB_ATOMIC(uintptr_t) *: upb_Atomic_LoadMsc(             \
0062                                  (uint64_t volatile *)(addr)), \
0063       default: (void *)upb_Atomic_LoadMsc((uint64_t volatile *)(addr)))
0064
0065 #define upb_Atomic_Exchange(addr, val, order)                                 \
0066   _Generic(addr,                                                              \
0067       UPB_ATOMIC(uintptr_t) *: _InterlockedExchange64(                        \
0068                                  (uint64_t volatile *)(addr), (uint64_t)val), \
0069       default: (void *)_InterlockedExchange64((uint64_t volatile *)addr,      \
0070                                               (uint64_t)val))
0071 #else
0072 // Compare exchange with an unlikely value reduces the risk of a spurious
0073 // (but harmless) store
0074 #define upb_Atomic_Load(addr, order) \
0075   (void *)upb_Atomic_LoadMsc((uint64_t volatile *)(addr))
0076
0077 #define upb_Atomic_Exchange(addr, val, order) \
0078   (void *)_InterlockedExchange64((uint64_t volatile *)addr, (uint64_t)val)
0079 #endif
0080
0081 #pragma intrinsic(_InterlockedCompareExchange64)
0082 static bool upb_Atomic_CompareExchangeMscP(uint64_t volatile *addr,
0083                                            uint64_t *expected,
0084                                            uint64_t desired) {
0085   uint64_t expect_val = *expected;
0086   uint64_t actual_val =
0087       _InterlockedCompareExchange64(addr, desired, expect_val);
0088   if (expect_val != actual_val) {
0089     *expected = actual_val;
0090     return false;
0091   }
0092   return true;
0093 }
0094
0095 #define upb_Atomic_CompareExchangeStrong(addr, expected, desired,      \
0096                                          success_order, failure_order) \
0097   upb_Atomic_CompareExchangeMscP((uint64_t volatile *)addr,            \
0098                                  (uint64_t *)expected, (uint64_t)desired)
0099
0100 #define upb_Atomic_CompareExchangeWeak(addr, expected, desired, success_order, \
0101                                        failure_order)                          \
0102   upb_Atomic_CompareExchangeMscP((uint64_t volatile *)addr,                    \
0103                                  (uint64_t *)expected, (uint64_t)desired)
0104
0105 #else  // 32 bit pointers
0106 #pragma intrinsic(_InterlockedExchange)
0107 #define upb_Atomic_Store(addr, val, order) \
0108   (void)_InterlockedExchange((uint32_t volatile *)addr, (uint32_t)val)
0109
0110 #pragma intrinsic(_InterlockedCompareExchange)
0111 static uintptr_t upb_Atomic_LoadMsc(uint32_t volatile *addr) {
0112   // Compare exchange with an unlikely value reduces the risk of a spurious
0113   // (but harmless) store
0114   return _InterlockedCompareExchange(addr, 0xDEADC0DE, 0xDEADC0DE);
0115 }
0116 // If _Generic is available, use it to avoid emitting 'uintptr_t' differs in
0117 // levels of indirection from 'void *'
0118 #if __STDC_VERSION__ >= 201112L
0119 #define upb_Atomic_Load(addr, order)                           \
0120   _Generic(addr,                                               \
0121       UPB_ATOMIC(uintptr_t) *: upb_Atomic_LoadMsc(             \
0122                                  (uint32_t volatile *)(addr)), \
0123       default: (void *)upb_Atomic_LoadMsc((uint32_t volatile *)(addr)))
0124
0125 #define upb_Atomic_Exchange(addr, val, order)                                 \
0126   _Generic(addr,                                                              \
0127       UPB_ATOMIC(uintptr_t) *: _InterlockedExchange(                          \
0128                                  (uint32_t volatile *)(addr), (uint32_t)val), \
0129       default: (void *)_InterlockedExchange64((uint32_t volatile *)addr,      \
0130                                               (uint32_t)val))
0131 #else
0132 #define upb_Atomic_Load(addr, order) \
0133   (void *)upb_Atomic_LoadMsc((uint32_t volatile *)(addr))
0134
0135 #define upb_Atomic_Exchange(addr, val, order) \
0136   (void *)_InterlockedExchange((uint32_t volatile *)addr, (uint32_t)val)
0137 #endif
0138
0139 #pragma intrinsic(_InterlockedCompareExchange)
0140 static bool upb_Atomic_CompareExchangeMscP(uint32_t volatile *addr,
0141                                            uint32_t *expected,
0142                                            uint32_t desired) {
0143   uint32_t expect_val = *expected;
0144   uint32_t actual_val = _InterlockedCompareExchange(addr, desired, expect_val);
0145   if (expect_val != actual_val) {
0146     *expected = actual_val;
0147     return false;
0148   }
0149   return true;
0150 }
0151
0152 #define upb_Atomic_CompareExchangeStrong(addr, expected, desired,      \
0153                                          success_order, failure_order) \
0154   upb_Atomic_CompareExchangeMscP((uint32_t volatile *)addr,            \
0155                                  (uint32_t *)expected, (uint32_t)desired)
0156
0157 #define upb_Atomic_CompareExchangeWeak(addr, expected, desired, success_order, \
0158                                        failure_order)                          \
0159   upb_Atomic_CompareExchangeMscP((uint32_t volatile *)addr,                    \
0160                                  (uint32_t *)expected, (uint32_t)desired)
0161 #endif
0162
0163 #else  // No atomics
0164
0165 #if !defined(UPB_SUPPRESS_MISSING_ATOMICS)
0166 // NOLINTNEXTLINE
0167 #error Your compiler does not support atomic instructions, which UPB uses. If you do not use UPB on multiple threads, you can suppress this error by defining UPB_SUPPRESS_MISSING_ATOMICS.
0168 #endif
0169
0170 #include <string.h>
0171
0172 #define upb_Atomic_Init(addr, val) (*addr = val)
0173 #define upb_Atomic_Load(addr, order) (*addr)
0174 #define upb_Atomic_Store(addr, val, order) (*(addr) = val)
0175
0176 UPB_INLINE void* _upb_NonAtomic_Exchange(void* addr, void* value) {
0177   void* old;
0178   memcpy(&old, addr, sizeof(value));
0179   memcpy(addr, &value, sizeof(value));
0180   return old;
0181 }
0182
0183 #define upb_Atomic_Exchange(addr, val, order) _upb_NonAtomic_Exchange(addr, val)
0184
0185 // `addr` and `expected` are logically double pointers.
0186 UPB_INLINE bool _upb_NonAtomic_CompareExchangeStrongP(void* addr,
0187                                                       void* expected,
0188                                                       void* desired) {
0189   if (memcmp(addr, expected, sizeof(desired)) == 0) {
0190     memcpy(addr, &desired, sizeof(desired));
0191     return true;
0192   } else {
0193     memcpy(expected, addr, sizeof(desired));
0194     return false;
0195   }
0196 }
0197
0198 #define upb_Atomic_CompareExchangeStrong(addr, expected, desired,      \
0199                                          success_order, failure_order) \
0200   _upb_NonAtomic_CompareExchangeStrongP((void*)addr, (void*)expected,  \
0201                                         (void*)desired)
0202 #define upb_Atomic_CompareExchangeWeak(addr, expected, desired, success_order, \
0203                                        failure_order)                          \
0204   upb_Atomic_CompareExchangeStrong(addr, expected, desired, 0, 0)
0205
0206 #endif
0207
0208 #include "upb/port/undef.inc"
0209
0210 #endif  // UPB_PORT_ATOMIC_H_