python3.13/cpython/pyatomic.h

0001 // This header provides cross-platform low-level atomic operations
0002 // similar to C11 atomics.
0003 //
0004 // Operations are sequentially consistent unless they have a suffix indicating
0005 // otherwise. If in doubt, prefer the sequentially consistent operations.
0006 //
0007 // The "_relaxed" suffix for load and store operations indicates the "relaxed"
0008 // memory order. They don't provide synchronization, but (roughly speaking)
0009 // guarantee somewhat sane behavior for races instead of undefined behavior.
0010 // In practice, they correspond to "normal" hardware load and store
0011 // instructions, so they are almost as inexpensive as plain loads and stores
0012 // in C.
0013 //
0014 // Note that atomic read-modify-write operations like _Py_atomic_add_* return
0015 // the previous value of the atomic variable, not the new value.
0016 //
0017 // See https://en.cppreference.com/w/c/atomic for more information on C11
0018 // atomics.
0019 // See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf
0020 // "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage
0021 // or relaxed atomics.
0022 //
0023 // Functions with pseudo Python code:
0024 //
0025 //   def _Py_atomic_load(obj):
0026 //        return obj  # sequential consistency
0027 //
0028 //   def _Py_atomic_load_relaxed(obj):
0029 //       return obj  # relaxed consistency
0030 //
0031 //   def _Py_atomic_store(obj, value):
0032 //       obj = value  # sequential consistency
0033 //
0034 //   def _Py_atomic_store_relaxed(obj, value):
0035 //       obj = value  # relaxed consistency
0036 //
0037 //   def _Py_atomic_exchange(obj, value):
0038 //       # sequential consistency
0039 //       old_obj = obj
0040 //       obj = value
0041 //       return old_obj
0042 //
0043 //   def _Py_atomic_compare_exchange(obj, expected, desired):
0044 //       # sequential consistency
0045 //       if obj == expected:
0046 //           obj = desired
0047 //           return True
0048 //       else:
0049 //           expected = obj
0050 //           return False
0051 //
0052 //   def _Py_atomic_add(obj, value):
0053 //       # sequential consistency
0054 //       old_obj = obj
0055 //       obj += value
0056 //       return old_obj
0057 //
0058 //   def _Py_atomic_and(obj, value):
0059 //       # sequential consistency
0060 //       old_obj = obj
0061 //       obj &= value
0062 //       return old_obj
0063 //
0064 //   def _Py_atomic_or(obj, value):
0065 //       # sequential consistency
0066 //       old_obj = obj
0067 //       obj |= value
0068 //       return old_obj
0069 //
0070 // Other functions:
0071 //
0072 //   def _Py_atomic_load_ptr_acquire(obj):
0073 //       return obj  # acquire
0074 //
0075 //   def _Py_atomic_store_ptr_release(obj):
0076 //       return obj  # release
0077 //
0078 //   def _Py_atomic_fence_seq_cst():
0079 //       # sequential consistency
0080 //       ...
0081 //
0082 //   def _Py_atomic_fence_release():
0083 //       # release
0084 //       ...
0085
0086 #ifndef Py_CPYTHON_ATOMIC_H
0087 #  error "this header file must not be included directly"
0088 #endif
0089
0090 // --- _Py_atomic_add --------------------------------------------------------
0091 // Atomically adds `value` to `obj` and returns the previous value
0092
0093 static inline int
0094 _Py_atomic_add_int(int *obj, int value);
0095
0096 static inline int8_t
0097 _Py_atomic_add_int8(int8_t *obj, int8_t value);
0098
0099 static inline int16_t
0100 _Py_atomic_add_int16(int16_t *obj, int16_t value);
0101
0102 static inline int32_t
0103 _Py_atomic_add_int32(int32_t *obj, int32_t value);
0104
0105 static inline int64_t
0106 _Py_atomic_add_int64(int64_t *obj, int64_t value);
0107
0108 static inline intptr_t
0109 _Py_atomic_add_intptr(intptr_t *obj, intptr_t value);
0110
0111 static inline unsigned int
0112 _Py_atomic_add_uint(unsigned int *obj, unsigned int value);
0113
0114 static inline uint8_t
0115 _Py_atomic_add_uint8(uint8_t *obj, uint8_t value);
0116
0117 static inline uint16_t
0118 _Py_atomic_add_uint16(uint16_t *obj, uint16_t value);
0119
0120 static inline uint32_t
0121 _Py_atomic_add_uint32(uint32_t *obj, uint32_t value);
0122
0123 static inline uint64_t
0124 _Py_atomic_add_uint64(uint64_t *obj, uint64_t value);
0125
0126 static inline uintptr_t
0127 _Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value);
0128
0129 static inline Py_ssize_t
0130 _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value);
0131
0132
0133 // --- _Py_atomic_compare_exchange -------------------------------------------
0134 // Performs an atomic compare-and-exchange.
0135 //
0136 // - If `*obj` and `*expected` are equal, store `desired` into `*obj`
0137 //   and return 1 (success).
0138 // - Otherwise, store the `*obj` current value into `*expected`
0139 //   and return 0 (failure).
0140 //
0141 // These correspond to the C11 atomic_compare_exchange_strong() function.
0142
0143 static inline int
0144 _Py_atomic_compare_exchange_int(int *obj, int *expected, int desired);
0145
0146 static inline int
0147 _Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired);
0148
0149 static inline int
0150 _Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired);
0151
0152 static inline int
0153 _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired);
0154
0155 static inline int
0156 _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired);
0157
0158 static inline int
0159 _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired);
0160
0161 static inline int
0162 _Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired);
0163
0164 static inline int
0165 _Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired);
0166
0167 static inline int
0168 _Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired);
0169
0170 static inline int
0171 _Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired);
0172
0173 static inline int
0174 _Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired);
0175
0176 static inline int
0177 _Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired);
0178
0179 static inline int
0180 _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired);
0181
0182 // NOTE: `obj` and `expected` are logically `void**` types, but we use `void*`
0183 // so that we can pass types like `PyObject**` without a cast.
0184 static inline int
0185 _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value);
0186
0187
0188 // --- _Py_atomic_exchange ---------------------------------------------------
0189 // Atomically replaces `*obj` with `value` and returns the previous value of `*obj`.
0190
0191 static inline int
0192 _Py_atomic_exchange_int(int *obj, int value);
0193
0194 static inline int8_t
0195 _Py_atomic_exchange_int8(int8_t *obj, int8_t value);
0196
0197 static inline int16_t
0198 _Py_atomic_exchange_int16(int16_t *obj, int16_t value);
0199
0200 static inline int32_t
0201 _Py_atomic_exchange_int32(int32_t *obj, int32_t value);
0202
0203 static inline int64_t
0204 _Py_atomic_exchange_int64(int64_t *obj, int64_t value);
0205
0206 static inline intptr_t
0207 _Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value);
0208
0209 static inline unsigned int
0210 _Py_atomic_exchange_uint(unsigned int *obj, unsigned int value);
0211
0212 static inline uint8_t
0213 _Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value);
0214
0215 static inline uint16_t
0216 _Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value);
0217
0218 static inline uint32_t
0219 _Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value);
0220
0221 static inline uint64_t
0222 _Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value);
0223
0224 static inline uintptr_t
0225 _Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value);
0226
0227 static inline Py_ssize_t
0228 _Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value);
0229
0230 static inline void *
0231 _Py_atomic_exchange_ptr(void *obj, void *value);
0232
0233
0234 // --- _Py_atomic_and --------------------------------------------------------
0235 // Performs `*obj &= value` atomically and returns the previous value of `*obj`.
0236
0237 static inline uint8_t
0238 _Py_atomic_and_uint8(uint8_t *obj, uint8_t value);
0239
0240 static inline uint16_t
0241 _Py_atomic_and_uint16(uint16_t *obj, uint16_t value);
0242
0243 static inline uint32_t
0244 _Py_atomic_and_uint32(uint32_t *obj, uint32_t value);
0245
0246 static inline uint64_t
0247 _Py_atomic_and_uint64(uint64_t *obj, uint64_t value);
0248
0249 static inline uintptr_t
0250 _Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value);
0251
0252
0253 // --- _Py_atomic_or ---------------------------------------------------------
0254 // Performs `*obj |= value` atomically and returns the previous value of `*obj`.
0255
0256 static inline uint8_t
0257 _Py_atomic_or_uint8(uint8_t *obj, uint8_t value);
0258
0259 static inline uint16_t
0260 _Py_atomic_or_uint16(uint16_t *obj, uint16_t value);
0261
0262 static inline uint32_t
0263 _Py_atomic_or_uint32(uint32_t *obj, uint32_t value);
0264
0265 static inline uint64_t
0266 _Py_atomic_or_uint64(uint64_t *obj, uint64_t value);
0267
0268 static inline uintptr_t
0269 _Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value);
0270
0271
0272 // --- _Py_atomic_load -------------------------------------------------------
0273 // Atomically loads `*obj` (sequential consistency)
0274
0275 static inline int
0276 _Py_atomic_load_int(const int *obj);
0277
0278 static inline int8_t
0279 _Py_atomic_load_int8(const int8_t *obj);
0280
0281 static inline int16_t
0282 _Py_atomic_load_int16(const int16_t *obj);
0283
0284 static inline int32_t
0285 _Py_atomic_load_int32(const int32_t *obj);
0286
0287 static inline int64_t
0288 _Py_atomic_load_int64(const int64_t *obj);
0289
0290 static inline intptr_t
0291 _Py_atomic_load_intptr(const intptr_t *obj);
0292
0293 static inline uint8_t
0294 _Py_atomic_load_uint8(const uint8_t *obj);
0295
0296 static inline uint16_t
0297 _Py_atomic_load_uint16(const uint16_t *obj);
0298
0299 static inline uint32_t
0300 _Py_atomic_load_uint32(const uint32_t *obj);
0301
0302 static inline uint64_t
0303 _Py_atomic_load_uint64(const uint64_t *obj);
0304
0305 static inline uintptr_t
0306 _Py_atomic_load_uintptr(const uintptr_t *obj);
0307
0308 static inline unsigned int
0309 _Py_atomic_load_uint(const unsigned int *obj);
0310
0311 static inline Py_ssize_t
0312 _Py_atomic_load_ssize(const Py_ssize_t *obj);
0313
0314 static inline void *
0315 _Py_atomic_load_ptr(const void *obj);
0316
0317
0318 // --- _Py_atomic_load_relaxed -----------------------------------------------
0319 // Loads `*obj` (relaxed consistency, i.e., no ordering)
0320
0321 static inline int
0322 _Py_atomic_load_int_relaxed(const int *obj);
0323
0324 static inline int8_t
0325 _Py_atomic_load_int8_relaxed(const int8_t *obj);
0326
0327 static inline int16_t
0328 _Py_atomic_load_int16_relaxed(const int16_t *obj);
0329
0330 static inline int32_t
0331 _Py_atomic_load_int32_relaxed(const int32_t *obj);
0332
0333 static inline int64_t
0334 _Py_atomic_load_int64_relaxed(const int64_t *obj);
0335
0336 static inline intptr_t
0337 _Py_atomic_load_intptr_relaxed(const intptr_t *obj);
0338
0339 static inline uint8_t
0340 _Py_atomic_load_uint8_relaxed(const uint8_t *obj);
0341
0342 static inline uint16_t
0343 _Py_atomic_load_uint16_relaxed(const uint16_t *obj);
0344
0345 static inline uint32_t
0346 _Py_atomic_load_uint32_relaxed(const uint32_t *obj);
0347
0348 static inline uint64_t
0349 _Py_atomic_load_uint64_relaxed(const uint64_t *obj);
0350
0351 static inline uintptr_t
0352 _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj);
0353
0354 static inline unsigned int
0355 _Py_atomic_load_uint_relaxed(const unsigned int *obj);
0356
0357 static inline Py_ssize_t
0358 _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj);
0359
0360 static inline void *
0361 _Py_atomic_load_ptr_relaxed(const void *obj);
0362
0363 static inline unsigned long long
0364 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj);
0365
0366 // --- _Py_atomic_store ------------------------------------------------------
0367 // Atomically performs `*obj = value` (sequential consistency)
0368
0369 static inline void
0370 _Py_atomic_store_int(int *obj, int value);
0371
0372 static inline void
0373 _Py_atomic_store_int8(int8_t *obj, int8_t value);
0374
0375 static inline void
0376 _Py_atomic_store_int16(int16_t *obj, int16_t value);
0377
0378 static inline void
0379 _Py_atomic_store_int32(int32_t *obj, int32_t value);
0380
0381 static inline void
0382 _Py_atomic_store_int64(int64_t *obj, int64_t value);
0383
0384 static inline void
0385 _Py_atomic_store_intptr(intptr_t *obj, intptr_t value);
0386
0387 static inline void
0388 _Py_atomic_store_uint8(uint8_t *obj, uint8_t value);
0389
0390 static inline void
0391 _Py_atomic_store_uint16(uint16_t *obj, uint16_t value);
0392
0393 static inline void
0394 _Py_atomic_store_uint32(uint32_t *obj, uint32_t value);
0395
0396 static inline void
0397 _Py_atomic_store_uint64(uint64_t *obj, uint64_t value);
0398
0399 static inline void
0400 _Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value);
0401
0402 static inline void
0403 _Py_atomic_store_uint(unsigned int *obj, unsigned int value);
0404
0405 static inline void
0406 _Py_atomic_store_ptr(void *obj, void *value);
0407
0408 static inline void
0409 _Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value);
0410
0411
0412 // --- _Py_atomic_store_relaxed ----------------------------------------------
0413 // Stores `*obj = value` (relaxed consistency, i.e., no ordering)
0414
0415 static inline void
0416 _Py_atomic_store_int_relaxed(int *obj, int value);
0417
0418 static inline void
0419 _Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value);
0420
0421 static inline void
0422 _Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value);
0423
0424 static inline void
0425 _Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value);
0426
0427 static inline void
0428 _Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value);
0429
0430 static inline void
0431 _Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value);
0432
0433 static inline void
0434 _Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value);
0435
0436 static inline void
0437 _Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value);
0438
0439 static inline void
0440 _Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value);
0441
0442 static inline void
0443 _Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value);
0444
0445 static inline void
0446 _Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value);
0447
0448 static inline void
0449 _Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value);
0450
0451 static inline void
0452 _Py_atomic_store_ptr_relaxed(void *obj, void *value);
0453
0454 static inline void
0455 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);
0456
0457 static inline void
0458 _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
0459                                 unsigned long long value);
0460
0461
0462 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
0463
0464 // Loads `*obj` (acquire operation)
0465 static inline void *
0466 _Py_atomic_load_ptr_acquire(const void *obj);
0467
0468 static inline uintptr_t
0469 _Py_atomic_load_uintptr_acquire(const uintptr_t *obj);
0470
0471 // Stores `*obj = value` (release operation)
0472 static inline void
0473 _Py_atomic_store_ptr_release(void *obj, void *value);
0474
0475 static inline void
0476 _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value);
0477
0478 static inline void
0479 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value);
0480
0481 static inline void
0482 _Py_atomic_store_int_release(int *obj, int value);
0483
0484 static inline int
0485 _Py_atomic_load_int_acquire(const int *obj);
0486
0487 static inline void
0488 _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value);
0489
0490 static inline void
0491 _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value);
0492
0493 static inline uint64_t
0494 _Py_atomic_load_uint64_acquire(const uint64_t *obj);
0495
0496 static inline uint32_t
0497 _Py_atomic_load_uint32_acquire(const uint32_t *obj);
0498
0499 static inline Py_ssize_t
0500 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj);
0501
0502
0503
0504
0505 // --- _Py_atomic_fence ------------------------------------------------------
0506
0507 // Sequential consistency fence. C11 fences have complex semantics. When
0508 // possible, use the atomic operations on variables defined above, which
0509 // generally do not require explicit use of a fence.
0510 // See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence
0511 static inline void _Py_atomic_fence_seq_cst(void);
0512
0513 // Acquire fence
0514 static inline void _Py_atomic_fence_acquire(void);
0515
0516 // Release fence
0517 static inline void _Py_atomic_fence_release(void);
0518
0519
0520 #ifndef _Py_USE_GCC_BUILTIN_ATOMICS
0521 #  if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))
0522 #    define _Py_USE_GCC_BUILTIN_ATOMICS 1
0523 #  elif defined(__clang__)
0524 #    if __has_builtin(__atomic_load)
0525 #      define _Py_USE_GCC_BUILTIN_ATOMICS 1
0526 #    endif
0527 #  endif
0528 #endif
0529
0530 #if _Py_USE_GCC_BUILTIN_ATOMICS
0531 #  define Py_ATOMIC_GCC_H
0532 #  include "pyatomic_gcc.h"
0533 #  undef Py_ATOMIC_GCC_H
0534 #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
0535 #  define Py_ATOMIC_STD_H
0536 #  include "pyatomic_std.h"
0537 #  undef Py_ATOMIC_STD_H
0538 #elif defined(_MSC_VER)
0539 #  define Py_ATOMIC_MSC_H
0540 #  include "pyatomic_msc.h"
0541 #  undef Py_ATOMIC_MSC_H
0542 #else
0543 #  error "no available pyatomic implementation for this platform/compiler"
0544 #endif
0545
0546
0547 // --- aliases ---------------------------------------------------------------
0548
0549 #if SIZEOF_LONG == 8
0550 # define _Py_atomic_load_ulong(p) \
0551     _Py_atomic_load_uint64((uint64_t *)p)
0552 # define _Py_atomic_load_ulong_relaxed(p) \
0553     _Py_atomic_load_uint64_relaxed((uint64_t *)p)
0554 # define _Py_atomic_store_ulong(p, v) \
0555     _Py_atomic_store_uint64((uint64_t *)p, v)
0556 # define _Py_atomic_store_ulong_relaxed(p, v) \
0557     _Py_atomic_store_uint64_relaxed((uint64_t *)p, v)
0558 #elif SIZEOF_LONG == 4
0559 # define _Py_atomic_load_ulong(p) \
0560     _Py_atomic_load_uint32((uint32_t *)p)
0561 # define _Py_atomic_load_ulong_relaxed(p) \
0562     _Py_atomic_load_uint32_relaxed((uint32_t *)p)
0563 # define _Py_atomic_store_ulong(p, v) \
0564     _Py_atomic_store_uint32((uint32_t *)p, v)
0565 # define _Py_atomic_store_ulong_relaxed(p, v) \
0566     _Py_atomic_store_uint32_relaxed((uint32_t *)p, v)
0567 #else
0568 # error "long must be 4 or 8 bytes in size"
0569 #endif  // SIZEOF_LONG