python3.13/cpython/pyatomic_std.h

0001 // This is the implementation of Python atomic operations using C++11 or C11
0002 // atomics. Note that the pyatomic_gcc.h implementation is preferred for GCC
0003 // compatible compilers, even if they support C++11 atomics.
0004
0005 #ifndef Py_ATOMIC_STD_H
0006 #  error "this header file must not be included directly"
0007 #endif
0008
0009 #ifdef __cplusplus
0010 extern "C++" {
0011 #  include <atomic>
0012 }
0013 #  define _Py_USING_STD using namespace std
0014 #  define _Atomic(tp) atomic<tp>
0015 #else
0016 #  define  _Py_USING_STD
0017 #  include <stdatomic.h>
0018 #endif
0019
0020
0021 // --- _Py_atomic_add --------------------------------------------------------
0022
0023 static inline int
0024 _Py_atomic_add_int(int *obj, int value)
0025 {
0026     _Py_USING_STD;
0027     return atomic_fetch_add((_Atomic(int)*)obj, value);
0028 }
0029
0030 static inline int8_t
0031 _Py_atomic_add_int8(int8_t *obj, int8_t value)
0032 {
0033     _Py_USING_STD;
0034     return atomic_fetch_add((_Atomic(int8_t)*)obj, value);
0035 }
0036
0037 static inline int16_t
0038 _Py_atomic_add_int16(int16_t *obj, int16_t value)
0039 {
0040     _Py_USING_STD;
0041     return atomic_fetch_add((_Atomic(int16_t)*)obj, value);
0042 }
0043
0044 static inline int32_t
0045 _Py_atomic_add_int32(int32_t *obj, int32_t value)
0046 {
0047     _Py_USING_STD;
0048     return atomic_fetch_add((_Atomic(int32_t)*)obj, value);
0049 }
0050
0051 static inline int64_t
0052 _Py_atomic_add_int64(int64_t *obj, int64_t value)
0053 {
0054     _Py_USING_STD;
0055     return atomic_fetch_add((_Atomic(int64_t)*)obj, value);
0056 }
0057
0058 static inline intptr_t
0059 _Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
0060 {
0061     _Py_USING_STD;
0062     return atomic_fetch_add((_Atomic(intptr_t)*)obj, value);
0063 }
0064
0065 static inline unsigned int
0066 _Py_atomic_add_uint(unsigned int *obj, unsigned int value)
0067 {
0068     _Py_USING_STD;
0069     return atomic_fetch_add((_Atomic(unsigned int)*)obj, value);
0070 }
0071
0072 static inline uint8_t
0073 _Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
0074 {
0075     _Py_USING_STD;
0076     return atomic_fetch_add((_Atomic(uint8_t)*)obj, value);
0077 }
0078
0079 static inline uint16_t
0080 _Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
0081 {
0082     _Py_USING_STD;
0083     return atomic_fetch_add((_Atomic(uint16_t)*)obj, value);
0084 }
0085
0086 static inline uint32_t
0087 _Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
0088 {
0089     _Py_USING_STD;
0090     return atomic_fetch_add((_Atomic(uint32_t)*)obj, value);
0091 }
0092
0093 static inline uint64_t
0094 _Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
0095 {
0096     _Py_USING_STD;
0097     return atomic_fetch_add((_Atomic(uint64_t)*)obj, value);
0098 }
0099
0100 static inline uintptr_t
0101 _Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
0102 {
0103     _Py_USING_STD;
0104     return atomic_fetch_add((_Atomic(uintptr_t)*)obj, value);
0105 }
0106
0107 static inline Py_ssize_t
0108 _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
0109 {
0110     _Py_USING_STD;
0111     return atomic_fetch_add((_Atomic(Py_ssize_t)*)obj, value);
0112 }
0113
0114
0115 // --- _Py_atomic_compare_exchange -------------------------------------------
0116
0117 static inline int
0118 _Py_atomic_compare_exchange_int(int *obj, int *expected, int desired)
0119 {
0120     _Py_USING_STD;
0121     return atomic_compare_exchange_strong((_Atomic(int)*)obj,
0122                                           expected, desired);
0123 }
0124
0125 static inline int
0126 _Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired)
0127 {
0128     _Py_USING_STD;
0129     return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj,
0130                                           expected, desired);
0131 }
0132
0133 static inline int
0134 _Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired)
0135 {
0136     _Py_USING_STD;
0137     return atomic_compare_exchange_strong((_Atomic(int16_t)*)obj,
0138                                           expected, desired);
0139 }
0140
0141 static inline int
0142 _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired)
0143 {
0144     _Py_USING_STD;
0145     return atomic_compare_exchange_strong((_Atomic(int32_t)*)obj,
0146                                           expected, desired);
0147 }
0148
0149 static inline int
0150 _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired)
0151 {
0152     _Py_USING_STD;
0153     return atomic_compare_exchange_strong((_Atomic(int64_t)*)obj,
0154                                           expected, desired);
0155 }
0156
0157 static inline int
0158 _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired)
0159 {
0160     _Py_USING_STD;
0161     return atomic_compare_exchange_strong((_Atomic(intptr_t)*)obj,
0162                                           expected, desired);
0163 }
0164
0165 static inline int
0166 _Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired)
0167 {
0168     _Py_USING_STD;
0169     return atomic_compare_exchange_strong((_Atomic(unsigned int)*)obj,
0170                                           expected, desired);
0171 }
0172
0173 static inline int
0174 _Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired)
0175 {
0176     _Py_USING_STD;
0177     return atomic_compare_exchange_strong((_Atomic(uint8_t)*)obj,
0178                                           expected, desired);
0179 }
0180
0181 static inline int
0182 _Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired)
0183 {
0184     _Py_USING_STD;
0185     return atomic_compare_exchange_strong((_Atomic(uint16_t)*)obj,
0186                                           expected, desired);
0187 }
0188
0189 static inline int
0190 _Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired)
0191 {
0192     _Py_USING_STD;
0193     return atomic_compare_exchange_strong((_Atomic(uint32_t)*)obj,
0194                                           expected, desired);
0195 }
0196
0197 static inline int
0198 _Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired)
0199 {
0200     _Py_USING_STD;
0201     return atomic_compare_exchange_strong((_Atomic(uint64_t)*)obj,
0202                                           expected, desired);
0203 }
0204
0205 static inline int
0206 _Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired)
0207 {
0208     _Py_USING_STD;
0209     return atomic_compare_exchange_strong((_Atomic(uintptr_t)*)obj,
0210                                           expected, desired);
0211 }
0212
0213 static inline int
0214 _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired)
0215 {
0216     _Py_USING_STD;
0217     return atomic_compare_exchange_strong((_Atomic(Py_ssize_t)*)obj,
0218                                           expected, desired);
0219 }
0220
0221 static inline int
0222 _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired)
0223 {
0224     _Py_USING_STD;
0225     return atomic_compare_exchange_strong((_Atomic(void *)*)obj,
0226                                           (void **)expected, desired);
0227 }
0228
0229
0230 // --- _Py_atomic_exchange ---------------------------------------------------
0231
0232 static inline int
0233 _Py_atomic_exchange_int(int *obj, int value)
0234 {
0235     _Py_USING_STD;
0236     return atomic_exchange((_Atomic(int)*)obj, value);
0237 }
0238
0239 static inline int8_t
0240 _Py_atomic_exchange_int8(int8_t *obj, int8_t value)
0241 {
0242     _Py_USING_STD;
0243     return atomic_exchange((_Atomic(int8_t)*)obj, value);
0244 }
0245
0246 static inline int16_t
0247 _Py_atomic_exchange_int16(int16_t *obj, int16_t value)
0248 {
0249     _Py_USING_STD;
0250     return atomic_exchange((_Atomic(int16_t)*)obj, value);
0251 }
0252
0253 static inline int32_t
0254 _Py_atomic_exchange_int32(int32_t *obj, int32_t value)
0255 {
0256     _Py_USING_STD;
0257     return atomic_exchange((_Atomic(int32_t)*)obj, value);
0258 }
0259
0260 static inline int64_t
0261 _Py_atomic_exchange_int64(int64_t *obj, int64_t value)
0262 {
0263     _Py_USING_STD;
0264     return atomic_exchange((_Atomic(int64_t)*)obj, value);
0265 }
0266
0267 static inline intptr_t
0268 _Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
0269 {
0270     _Py_USING_STD;
0271     return atomic_exchange((_Atomic(intptr_t)*)obj, value);
0272 }
0273
0274 static inline unsigned int
0275 _Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
0276 {
0277     _Py_USING_STD;
0278     return atomic_exchange((_Atomic(unsigned int)*)obj, value);
0279 }
0280
0281 static inline uint8_t
0282 _Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
0283 {
0284     _Py_USING_STD;
0285     return atomic_exchange((_Atomic(uint8_t)*)obj, value);
0286 }
0287
0288 static inline uint16_t
0289 _Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
0290 {
0291     _Py_USING_STD;
0292     return atomic_exchange((_Atomic(uint16_t)*)obj, value);
0293 }
0294
0295 static inline uint32_t
0296 _Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
0297 {
0298     _Py_USING_STD;
0299     return atomic_exchange((_Atomic(uint32_t)*)obj, value);
0300 }
0301
0302 static inline uint64_t
0303 _Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
0304 {
0305     _Py_USING_STD;
0306     return atomic_exchange((_Atomic(uint64_t)*)obj, value);
0307 }
0308
0309 static inline uintptr_t
0310 _Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
0311 {
0312     _Py_USING_STD;
0313     return atomic_exchange((_Atomic(uintptr_t)*)obj, value);
0314 }
0315
0316 static inline Py_ssize_t
0317 _Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
0318 {
0319     _Py_USING_STD;
0320     return atomic_exchange((_Atomic(Py_ssize_t)*)obj, value);
0321 }
0322
0323 static inline void*
0324 _Py_atomic_exchange_ptr(void *obj, void *value)
0325 {
0326     _Py_USING_STD;
0327     return atomic_exchange((_Atomic(void *)*)obj, value);
0328 }
0329
0330
0331 // --- _Py_atomic_and --------------------------------------------------------
0332
0333 static inline uint8_t
0334 _Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
0335 {
0336     _Py_USING_STD;
0337     return atomic_fetch_and((_Atomic(uint8_t)*)obj, value);
0338 }
0339
0340 static inline uint16_t
0341 _Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
0342 {
0343     _Py_USING_STD;
0344     return atomic_fetch_and((_Atomic(uint16_t)*)obj, value);
0345 }
0346
0347 static inline uint32_t
0348 _Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
0349 {
0350     _Py_USING_STD;
0351     return atomic_fetch_and((_Atomic(uint32_t)*)obj, value);
0352 }
0353
0354 static inline uint64_t
0355 _Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
0356 {
0357     _Py_USING_STD;
0358     return atomic_fetch_and((_Atomic(uint64_t)*)obj, value);
0359 }
0360
0361 static inline uintptr_t
0362 _Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
0363 {
0364     _Py_USING_STD;
0365     return atomic_fetch_and((_Atomic(uintptr_t)*)obj, value);
0366 }
0367
0368
0369 // --- _Py_atomic_or ---------------------------------------------------------
0370
0371 static inline uint8_t
0372 _Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
0373 {
0374     _Py_USING_STD;
0375     return atomic_fetch_or((_Atomic(uint8_t)*)obj, value);
0376 }
0377
0378 static inline uint16_t
0379 _Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
0380 {
0381     _Py_USING_STD;
0382     return atomic_fetch_or((_Atomic(uint16_t)*)obj, value);
0383 }
0384
0385 static inline uint32_t
0386 _Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
0387 {
0388     _Py_USING_STD;
0389     return atomic_fetch_or((_Atomic(uint32_t)*)obj, value);
0390 }
0391
0392 static inline uint64_t
0393 _Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
0394 {
0395     _Py_USING_STD;
0396     return atomic_fetch_or((_Atomic(uint64_t)*)obj, value);
0397 }
0398
0399 static inline uintptr_t
0400 _Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
0401 {
0402     _Py_USING_STD;
0403     return atomic_fetch_or((_Atomic(uintptr_t)*)obj, value);
0404 }
0405
0406
0407 // --- _Py_atomic_load -------------------------------------------------------
0408
0409 static inline int
0410 _Py_atomic_load_int(const int *obj)
0411 {
0412     _Py_USING_STD;
0413     return atomic_load((const _Atomic(int)*)obj);
0414 }
0415
0416 static inline int8_t
0417 _Py_atomic_load_int8(const int8_t *obj)
0418 {
0419     _Py_USING_STD;
0420     return atomic_load((const _Atomic(int8_t)*)obj);
0421 }
0422
0423 static inline int16_t
0424 _Py_atomic_load_int16(const int16_t *obj)
0425 {
0426     _Py_USING_STD;
0427     return atomic_load((const _Atomic(int16_t)*)obj);
0428 }
0429
0430 static inline int32_t
0431 _Py_atomic_load_int32(const int32_t *obj)
0432 {
0433     _Py_USING_STD;
0434     return atomic_load((const _Atomic(int32_t)*)obj);
0435 }
0436
0437 static inline int64_t
0438 _Py_atomic_load_int64(const int64_t *obj)
0439 {
0440     _Py_USING_STD;
0441     return atomic_load((const _Atomic(int64_t)*)obj);
0442 }
0443
0444 static inline intptr_t
0445 _Py_atomic_load_intptr(const intptr_t *obj)
0446 {
0447     _Py_USING_STD;
0448     return atomic_load((const _Atomic(intptr_t)*)obj);
0449 }
0450
0451 static inline uint8_t
0452 _Py_atomic_load_uint8(const uint8_t *obj)
0453 {
0454     _Py_USING_STD;
0455     return atomic_load((const _Atomic(uint8_t)*)obj);
0456 }
0457
0458 static inline uint16_t
0459 _Py_atomic_load_uint16(const uint16_t *obj)
0460 {
0461     _Py_USING_STD;
0462     return atomic_load((const _Atomic(uint32_t)*)obj);
0463 }
0464
0465 static inline uint32_t
0466 _Py_atomic_load_uint32(const uint32_t *obj)
0467 {
0468     _Py_USING_STD;
0469     return atomic_load((const _Atomic(uint32_t)*)obj);
0470 }
0471
0472 static inline uint64_t
0473 _Py_atomic_load_uint64(const uint64_t *obj)
0474 {
0475     _Py_USING_STD;
0476     return atomic_load((const _Atomic(uint64_t)*)obj);
0477 }
0478
0479 static inline uintptr_t
0480 _Py_atomic_load_uintptr(const uintptr_t *obj)
0481 {
0482     _Py_USING_STD;
0483     return atomic_load((const _Atomic(uintptr_t)*)obj);
0484 }
0485
0486 static inline unsigned int
0487 _Py_atomic_load_uint(const unsigned int *obj)
0488 {
0489     _Py_USING_STD;
0490     return atomic_load((const _Atomic(unsigned int)*)obj);
0491 }
0492
0493 static inline Py_ssize_t
0494 _Py_atomic_load_ssize(const Py_ssize_t *obj)
0495 {
0496     _Py_USING_STD;
0497     return atomic_load((const _Atomic(Py_ssize_t)*)obj);
0498 }
0499
0500 static inline void*
0501 _Py_atomic_load_ptr(const void *obj)
0502 {
0503     _Py_USING_STD;
0504     return atomic_load((const _Atomic(void*)*)obj);
0505 }
0506
0507
0508 // --- _Py_atomic_load_relaxed -----------------------------------------------
0509
0510 static inline int
0511 _Py_atomic_load_int_relaxed(const int *obj)
0512 {
0513     _Py_USING_STD;
0514     return atomic_load_explicit((const _Atomic(int)*)obj,
0515                                 memory_order_relaxed);
0516 }
0517
0518 static inline int8_t
0519 _Py_atomic_load_int8_relaxed(const int8_t *obj)
0520 {
0521     _Py_USING_STD;
0522     return atomic_load_explicit((const _Atomic(int8_t)*)obj,
0523                                 memory_order_relaxed);
0524 }
0525
0526 static inline int16_t
0527 _Py_atomic_load_int16_relaxed(const int16_t *obj)
0528 {
0529     _Py_USING_STD;
0530     return atomic_load_explicit((const _Atomic(int16_t)*)obj,
0531                                 memory_order_relaxed);
0532 }
0533
0534 static inline int32_t
0535 _Py_atomic_load_int32_relaxed(const int32_t *obj)
0536 {
0537     _Py_USING_STD;
0538     return atomic_load_explicit((const _Atomic(int32_t)*)obj,
0539                                 memory_order_relaxed);
0540 }
0541
0542 static inline int64_t
0543 _Py_atomic_load_int64_relaxed(const int64_t *obj)
0544 {
0545     _Py_USING_STD;
0546     return atomic_load_explicit((const _Atomic(int64_t)*)obj,
0547                                 memory_order_relaxed);
0548 }
0549
0550 static inline intptr_t
0551 _Py_atomic_load_intptr_relaxed(const intptr_t *obj)
0552 {
0553     _Py_USING_STD;
0554     return atomic_load_explicit((const _Atomic(intptr_t)*)obj,
0555                                 memory_order_relaxed);
0556 }
0557
0558 static inline uint8_t
0559 _Py_atomic_load_uint8_relaxed(const uint8_t *obj)
0560 {
0561     _Py_USING_STD;
0562     return atomic_load_explicit((const _Atomic(uint8_t)*)obj,
0563                                 memory_order_relaxed);
0564 }
0565
0566 static inline uint16_t
0567 _Py_atomic_load_uint16_relaxed(const uint16_t *obj)
0568 {
0569     _Py_USING_STD;
0570     return atomic_load_explicit((const _Atomic(uint16_t)*)obj,
0571                                 memory_order_relaxed);
0572 }
0573
0574 static inline uint32_t
0575 _Py_atomic_load_uint32_relaxed(const uint32_t *obj)
0576 {
0577     _Py_USING_STD;
0578     return atomic_load_explicit((const _Atomic(uint32_t)*)obj,
0579                                 memory_order_relaxed);
0580 }
0581
0582 static inline uint64_t
0583 _Py_atomic_load_uint64_relaxed(const uint64_t *obj)
0584 {
0585     _Py_USING_STD;
0586     return atomic_load_explicit((const _Atomic(uint64_t)*)obj,
0587                                 memory_order_relaxed);
0588 }
0589
0590 static inline uintptr_t
0591 _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
0592 {
0593     _Py_USING_STD;
0594     return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
0595                                 memory_order_relaxed);
0596 }
0597
0598 static inline unsigned int
0599 _Py_atomic_load_uint_relaxed(const unsigned int *obj)
0600 {
0601     _Py_USING_STD;
0602     return atomic_load_explicit((const _Atomic(unsigned int)*)obj,
0603                                 memory_order_relaxed);
0604 }
0605
0606 static inline Py_ssize_t
0607 _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
0608 {
0609     _Py_USING_STD;
0610     return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj,
0611                                 memory_order_relaxed);
0612 }
0613
0614 static inline void*
0615 _Py_atomic_load_ptr_relaxed(const void *obj)
0616 {
0617     _Py_USING_STD;
0618     return atomic_load_explicit((const _Atomic(void*)*)obj,
0619                                 memory_order_relaxed);
0620 }
0621
0622 static inline unsigned long long
0623 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
0624 {
0625     _Py_USING_STD;
0626     return atomic_load_explicit((const _Atomic(unsigned long long)*)obj,
0627                                 memory_order_relaxed);
0628 }
0629
0630
0631 // --- _Py_atomic_store ------------------------------------------------------
0632
0633 static inline void
0634 _Py_atomic_store_int(int *obj, int value)
0635 {
0636     _Py_USING_STD;
0637     atomic_store((_Atomic(int)*)obj, value);
0638 }
0639
0640 static inline void
0641 _Py_atomic_store_int8(int8_t *obj, int8_t value)
0642 {
0643     _Py_USING_STD;
0644     atomic_store((_Atomic(int8_t)*)obj, value);
0645 }
0646
0647 static inline void
0648 _Py_atomic_store_int16(int16_t *obj, int16_t value)
0649 {
0650     _Py_USING_STD;
0651     atomic_store((_Atomic(int16_t)*)obj, value);
0652 }
0653
0654 static inline void
0655 _Py_atomic_store_int32(int32_t *obj, int32_t value)
0656 {
0657     _Py_USING_STD;
0658     atomic_store((_Atomic(int32_t)*)obj, value);
0659 }
0660
0661 static inline void
0662 _Py_atomic_store_int64(int64_t *obj, int64_t value)
0663 {
0664     _Py_USING_STD;
0665     atomic_store((_Atomic(int64_t)*)obj, value);
0666 }
0667
0668 static inline void
0669 _Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
0670 {
0671     _Py_USING_STD;
0672     atomic_store((_Atomic(intptr_t)*)obj, value);
0673 }
0674
0675 static inline void
0676 _Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
0677 {
0678     _Py_USING_STD;
0679     atomic_store((_Atomic(uint8_t)*)obj, value);
0680 }
0681
0682 static inline void
0683 _Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
0684 {
0685     _Py_USING_STD;
0686     atomic_store((_Atomic(uint16_t)*)obj, value);
0687 }
0688
0689 static inline void
0690 _Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
0691 {
0692     _Py_USING_STD;
0693     atomic_store((_Atomic(uint32_t)*)obj, value);
0694 }
0695
0696 static inline void
0697 _Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
0698 {
0699     _Py_USING_STD;
0700     atomic_store((_Atomic(uint64_t)*)obj, value);
0701 }
0702
0703 static inline void
0704 _Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
0705 {
0706     _Py_USING_STD;
0707     atomic_store((_Atomic(uintptr_t)*)obj, value);
0708 }
0709
0710 static inline void
0711 _Py_atomic_store_uint(unsigned int *obj, unsigned int value)
0712 {
0713     _Py_USING_STD;
0714     atomic_store((_Atomic(unsigned int)*)obj, value);
0715 }
0716
0717 static inline void
0718 _Py_atomic_store_ptr(void *obj, void *value)
0719 {
0720     _Py_USING_STD;
0721     atomic_store((_Atomic(void*)*)obj, value);
0722 }
0723
0724 static inline void
0725 _Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
0726 {
0727     _Py_USING_STD;
0728     atomic_store((_Atomic(Py_ssize_t)*)obj, value);
0729 }
0730
0731
0732 // --- _Py_atomic_store_relaxed ----------------------------------------------
0733
0734 static inline void
0735 _Py_atomic_store_int_relaxed(int *obj, int value)
0736 {
0737     _Py_USING_STD;
0738     atomic_store_explicit((_Atomic(int)*)obj, value,
0739                           memory_order_relaxed);
0740 }
0741
0742 static inline void
0743 _Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
0744 {
0745     _Py_USING_STD;
0746     atomic_store_explicit((_Atomic(int8_t)*)obj, value,
0747                           memory_order_relaxed);
0748 }
0749
0750 static inline void
0751 _Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
0752 {
0753     _Py_USING_STD;
0754     atomic_store_explicit((_Atomic(int16_t)*)obj, value,
0755                           memory_order_relaxed);
0756 }
0757
0758 static inline void
0759 _Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
0760 {
0761     _Py_USING_STD;
0762     atomic_store_explicit((_Atomic(int32_t)*)obj, value,
0763                           memory_order_relaxed);
0764 }
0765
0766 static inline void
0767 _Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
0768 {
0769     _Py_USING_STD;
0770     atomic_store_explicit((_Atomic(int64_t)*)obj, value,
0771                           memory_order_relaxed);
0772 }
0773
0774 static inline void
0775 _Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
0776 {
0777     _Py_USING_STD;
0778     atomic_store_explicit((_Atomic(intptr_t)*)obj, value,
0779                           memory_order_relaxed);
0780 }
0781
0782 static inline void
0783 _Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
0784 {
0785     _Py_USING_STD;
0786     atomic_store_explicit((_Atomic(uint8_t)*)obj, value,
0787                           memory_order_relaxed);
0788 }
0789
0790 static inline void
0791 _Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
0792 {
0793     _Py_USING_STD;
0794     atomic_store_explicit((_Atomic(uint16_t)*)obj, value,
0795                           memory_order_relaxed);
0796 }
0797
0798 static inline void
0799 _Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
0800 {
0801     _Py_USING_STD;
0802     atomic_store_explicit((_Atomic(uint32_t)*)obj, value,
0803                           memory_order_relaxed);
0804 }
0805
0806 static inline void
0807 _Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
0808 {
0809     _Py_USING_STD;
0810     atomic_store_explicit((_Atomic(uint64_t)*)obj, value,
0811                           memory_order_relaxed);
0812 }
0813
0814 static inline void
0815 _Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
0816 {
0817     _Py_USING_STD;
0818     atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
0819                           memory_order_relaxed);
0820 }
0821
0822 static inline void
0823 _Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
0824 {
0825     _Py_USING_STD;
0826     atomic_store_explicit((_Atomic(unsigned int)*)obj, value,
0827                           memory_order_relaxed);
0828 }
0829
0830 static inline void
0831 _Py_atomic_store_ptr_relaxed(void *obj, void *value)
0832 {
0833     _Py_USING_STD;
0834     atomic_store_explicit((_Atomic(void*)*)obj, value,
0835                           memory_order_relaxed);
0836 }
0837
0838 static inline void
0839 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
0840 {
0841     _Py_USING_STD;
0842     atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value,
0843                           memory_order_relaxed);
0844 }
0845
0846 static inline void
0847 _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
0848                                 unsigned long long value)
0849 {
0850     _Py_USING_STD;
0851     atomic_store_explicit((_Atomic(unsigned long long)*)obj, value,
0852                           memory_order_relaxed);
0853 }
0854
0855
0856 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
0857
0858 static inline void *
0859 _Py_atomic_load_ptr_acquire(const void *obj)
0860 {
0861     _Py_USING_STD;
0862     return atomic_load_explicit((const _Atomic(void*)*)obj,
0863                                 memory_order_acquire);
0864 }
0865
0866 static inline uintptr_t
0867 _Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
0868 {
0869     _Py_USING_STD;
0870     return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
0871                                 memory_order_acquire);
0872 }
0873
0874 static inline void
0875 _Py_atomic_store_ptr_release(void *obj, void *value)
0876 {
0877     _Py_USING_STD;
0878     atomic_store_explicit((_Atomic(void*)*)obj, value,
0879                           memory_order_release);
0880 }
0881
0882 static inline void
0883 _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
0884 {
0885     _Py_USING_STD;
0886     atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
0887                           memory_order_release);
0888 }
0889
0890 static inline void
0891 _Py_atomic_store_int_release(int *obj, int value)
0892 {
0893     _Py_USING_STD;
0894     atomic_store_explicit((_Atomic(int)*)obj, value,
0895                           memory_order_release);
0896 }
0897
0898 static inline void
0899 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
0900 {
0901     _Py_USING_STD;
0902     atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value,
0903                           memory_order_release);
0904 }
0905
0906 static inline int
0907 _Py_atomic_load_int_acquire(const int *obj)
0908 {
0909     _Py_USING_STD;
0910     return atomic_load_explicit((const _Atomic(int)*)obj,
0911                                 memory_order_acquire);
0912 }
0913
0914 static inline void
0915 _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)
0916 {
0917     _Py_USING_STD;
0918     atomic_store_explicit((_Atomic(uint32_t)*)obj, value,
0919                           memory_order_release);
0920 }
0921
0922 static inline void
0923 _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
0924 {
0925     _Py_USING_STD;
0926     atomic_store_explicit((_Atomic(uint64_t)*)obj, value,
0927                           memory_order_release);
0928 }
0929
0930 static inline uint64_t
0931 _Py_atomic_load_uint64_acquire(const uint64_t *obj)
0932 {
0933     _Py_USING_STD;
0934     return atomic_load_explicit((const _Atomic(uint64_t)*)obj,
0935                                 memory_order_acquire);
0936 }
0937
0938 static inline uint32_t
0939 _Py_atomic_load_uint32_acquire(const uint32_t *obj)
0940 {
0941     _Py_USING_STD;
0942     return atomic_load_explicit((const _Atomic(uint32_t)*)obj,
0943                                 memory_order_acquire);
0944 }
0945
0946 static inline Py_ssize_t
0947 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)
0948 {
0949     _Py_USING_STD;
0950     return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj,
0951                                 memory_order_acquire);
0952 }
0953
0954
0955 // --- _Py_atomic_fence ------------------------------------------------------
0956
0957  static inline void
0958 _Py_atomic_fence_seq_cst(void)
0959 {
0960     _Py_USING_STD;
0961     atomic_thread_fence(memory_order_seq_cst);
0962 }
0963
0964  static inline void
0965 _Py_atomic_fence_acquire(void)
0966 {
0967     _Py_USING_STD;
0968     atomic_thread_fence(memory_order_acquire);
0969 }
0970
0971  static inline void
0972 _Py_atomic_fence_release(void)
0973 {
0974     _Py_USING_STD;
0975     atomic_thread_fence(memory_order_release);
0976 }