python3.13/cpython/pyatomic_msc.h

0001 // This is the implementation of Python atomic operations for MSVC if the
0002 // compiler does not support C11 or C++11 atomics.
0003 //
0004 // MSVC intrinsics are defined on char, short, long, __int64, and pointer
0005 // types. Note that long and int are both 32-bits even on 64-bit Windows,
0006 // so operations on int are cast to long.
0007 //
0008 // The volatile keyword has additional memory ordering semantics on MSVC. On
0009 // x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64,
0010 // volatile accesses behave like C11's memory_order_relaxed.
0011
0012 #ifndef Py_ATOMIC_MSC_H
0013 #  error "this header file must not be included directly"
0014 #endif
0015
0016 #include <intrin.h>
0017
0018 #define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \
0019     Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE))
0020
0021
0022 // --- _Py_atomic_add --------------------------------------------------------
0023
0024 static inline int8_t
0025 _Py_atomic_add_int8(int8_t *obj, int8_t value)
0026 {
0027     _Py_atomic_ASSERT_ARG_TYPE(char);
0028     return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value);
0029 }
0030
0031 static inline int16_t
0032 _Py_atomic_add_int16(int16_t *obj, int16_t value)
0033 {
0034     _Py_atomic_ASSERT_ARG_TYPE(short);
0035     return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value);
0036 }
0037
0038 static inline int32_t
0039 _Py_atomic_add_int32(int32_t *obj, int32_t value)
0040 {
0041     _Py_atomic_ASSERT_ARG_TYPE(long);
0042     return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value);
0043 }
0044
0045 static inline int64_t
0046 _Py_atomic_add_int64(int64_t *obj, int64_t value)
0047 {
0048 #if defined(_M_X64) || defined(_M_ARM64)
0049     _Py_atomic_ASSERT_ARG_TYPE(__int64);
0050     return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value);
0051 #else
0052     int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
0053     for (;;) {
0054         int64_t new_value = old_value + value;
0055         if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) {
0056             return old_value;
0057         }
0058     }
0059 #endif
0060 }
0061
0062
0063 static inline uint8_t
0064 _Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
0065 {
0066     return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value);
0067 }
0068
0069 static inline uint16_t
0070 _Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
0071 {
0072     return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value);
0073 }
0074
0075 static inline uint32_t
0076 _Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
0077 {
0078     return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
0079 }
0080
0081 static inline int
0082 _Py_atomic_add_int(int *obj, int value)
0083 {
0084     _Py_atomic_ASSERT_ARG_TYPE(int32_t);
0085     return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
0086 }
0087
0088 static inline unsigned int
0089 _Py_atomic_add_uint(unsigned int *obj, unsigned int value)
0090 {
0091     _Py_atomic_ASSERT_ARG_TYPE(int32_t);
0092     return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
0093 }
0094
0095 static inline uint64_t
0096 _Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
0097 {
0098     return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
0099 }
0100
0101 static inline intptr_t
0102 _Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
0103 {
0104 #if SIZEOF_VOID_P == 8
0105     _Py_atomic_ASSERT_ARG_TYPE(int64_t);
0106     return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
0107 #else
0108     _Py_atomic_ASSERT_ARG_TYPE(int32_t);
0109     return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
0110 #endif
0111 }
0112
0113 static inline uintptr_t
0114 _Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
0115 {
0116     _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
0117     return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
0118 }
0119
0120 static inline Py_ssize_t
0121 _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
0122 {
0123     _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
0124     return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
0125 }
0126
0127
0128 // --- _Py_atomic_compare_exchange -------------------------------------------
0129
0130 static inline int
0131 _Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value)
0132 {
0133     _Py_atomic_ASSERT_ARG_TYPE(char);
0134     int8_t initial = (int8_t)_InterlockedCompareExchange8(
0135                                        (volatile char *)obj,
0136                                        (char)value,
0137                                        (char)*expected);
0138     if (initial == *expected) {
0139         return 1;
0140     }
0141     *expected = initial;
0142     return 0;
0143 }
0144
0145 static inline int
0146 _Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value)
0147 {
0148     _Py_atomic_ASSERT_ARG_TYPE(short);
0149     int16_t initial = (int16_t)_InterlockedCompareExchange16(
0150                                        (volatile short *)obj,
0151                                        (short)value,
0152                                        (short)*expected);
0153     if (initial == *expected) {
0154         return 1;
0155     }
0156     *expected = initial;
0157     return 0;
0158 }
0159
0160 static inline int
0161 _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value)
0162 {
0163     _Py_atomic_ASSERT_ARG_TYPE(long);
0164     int32_t initial = (int32_t)_InterlockedCompareExchange(
0165                                        (volatile long *)obj,
0166                                        (long)value,
0167                                        (long)*expected);
0168     if (initial == *expected) {
0169         return 1;
0170     }
0171     *expected = initial;
0172     return 0;
0173 }
0174
0175 static inline int
0176 _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value)
0177 {
0178     _Py_atomic_ASSERT_ARG_TYPE(__int64);
0179     int64_t initial = (int64_t)_InterlockedCompareExchange64(
0180                                        (volatile __int64 *)obj,
0181                                        (__int64)value,
0182                                        (__int64)*expected);
0183     if (initial == *expected) {
0184         return 1;
0185     }
0186     *expected = initial;
0187     return 0;
0188 }
0189
0190 static inline int
0191 _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value)
0192 {
0193     void *initial = _InterlockedCompareExchangePointer(
0194                                        (void**)obj,
0195                                        value,
0196                                        *(void**)expected);
0197     if (initial == *(void**)expected) {
0198         return 1;
0199     }
0200     *(void**)expected = initial;
0201     return 0;
0202 }
0203
0204
0205 static inline int
0206 _Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value)
0207 {
0208     return _Py_atomic_compare_exchange_int8((int8_t *)obj,
0209                                             (int8_t *)expected,
0210                                             (int8_t)value);
0211 }
0212
0213 static inline int
0214 _Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value)
0215 {
0216     return _Py_atomic_compare_exchange_int16((int16_t *)obj,
0217                                              (int16_t *)expected,
0218                                              (int16_t)value);
0219 }
0220
0221 static inline int
0222 _Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value)
0223 {
0224     return _Py_atomic_compare_exchange_int32((int32_t *)obj,
0225                                              (int32_t *)expected,
0226                                              (int32_t)value);
0227 }
0228
0229 static inline int
0230 _Py_atomic_compare_exchange_int(int *obj, int *expected, int value)
0231 {
0232     _Py_atomic_ASSERT_ARG_TYPE(int32_t);
0233     return _Py_atomic_compare_exchange_int32((int32_t *)obj,
0234                                              (int32_t *)expected,
0235                                              (int32_t)value);
0236 }
0237
0238 static inline int
0239 _Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value)
0240 {
0241     _Py_atomic_ASSERT_ARG_TYPE(int32_t);
0242     return _Py_atomic_compare_exchange_int32((int32_t *)obj,
0243                                              (int32_t *)expected,
0244                                              (int32_t)value);
0245 }
0246
0247 static inline int
0248 _Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value)
0249 {
0250     return _Py_atomic_compare_exchange_int64((int64_t *)obj,
0251                                              (int64_t *)expected,
0252                                              (int64_t)value);
0253 }
0254
0255 static inline int
0256 _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value)
0257 {
0258     _Py_atomic_ASSERT_ARG_TYPE(void*);
0259     return _Py_atomic_compare_exchange_ptr((void**)obj,
0260                                            (void**)expected,
0261                                            (void*)value);
0262 }
0263
0264 static inline int
0265 _Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value)
0266 {
0267     _Py_atomic_ASSERT_ARG_TYPE(void*);
0268     return _Py_atomic_compare_exchange_ptr((void**)obj,
0269                                            (void**)expected,
0270                                            (void*)value);
0271 }
0272
0273 static inline int
0274 _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value)
0275 {
0276     _Py_atomic_ASSERT_ARG_TYPE(void*);
0277     return _Py_atomic_compare_exchange_ptr((void**)obj,
0278                                            (void**)expected,
0279                                            (void*)value);
0280 }
0281
0282
0283 // --- _Py_atomic_exchange ---------------------------------------------------
0284
0285 static inline int8_t
0286 _Py_atomic_exchange_int8(int8_t *obj, int8_t value)
0287 {
0288     _Py_atomic_ASSERT_ARG_TYPE(char);
0289     return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value);
0290 }
0291
0292 static inline int16_t
0293 _Py_atomic_exchange_int16(int16_t *obj, int16_t value)
0294 {
0295     _Py_atomic_ASSERT_ARG_TYPE(short);
0296     return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value);
0297 }
0298
0299 static inline int32_t
0300 _Py_atomic_exchange_int32(int32_t *obj, int32_t value)
0301 {
0302     _Py_atomic_ASSERT_ARG_TYPE(long);
0303     return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value);
0304 }
0305
0306 static inline int64_t
0307 _Py_atomic_exchange_int64(int64_t *obj, int64_t value)
0308 {
0309 #if defined(_M_X64) || defined(_M_ARM64)
0310     _Py_atomic_ASSERT_ARG_TYPE(__int64);
0311     return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value);
0312 #else
0313     int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
0314     for (;;) {
0315         if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) {
0316             return old_value;
0317         }
0318     }
0319 #endif
0320 }
0321
0322 static inline void*
0323 _Py_atomic_exchange_ptr(void *obj, void *value)
0324 {
0325     return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value);
0326 }
0327
0328
0329 static inline uint8_t
0330 _Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
0331 {
0332     return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj,
0333                                              (int8_t)value);
0334 }
0335
0336 static inline uint16_t
0337 _Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
0338 {
0339     return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj,
0340                                                (int16_t)value);
0341 }
0342
0343 static inline uint32_t
0344 _Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
0345 {
0346     return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj,
0347                                                (int32_t)value);
0348 }
0349
0350 static inline int
0351 _Py_atomic_exchange_int(int *obj, int value)
0352 {
0353     _Py_atomic_ASSERT_ARG_TYPE(int32_t);
0354     return (int)_Py_atomic_exchange_int32((int32_t *)obj,
0355                                            (int32_t)value);
0356 }
0357
0358 static inline unsigned int
0359 _Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
0360 {
0361     _Py_atomic_ASSERT_ARG_TYPE(int32_t);
0362     return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj,
0363                                                    (int32_t)value);
0364 }
0365
0366 static inline uint64_t
0367 _Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
0368 {
0369     return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj,
0370                                                (int64_t)value);
0371 }
0372
0373 static inline intptr_t
0374 _Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
0375 {
0376     _Py_atomic_ASSERT_ARG_TYPE(void*);
0377     return (intptr_t)_Py_atomic_exchange_ptr((void**)obj,
0378                                              (void*)value);
0379 }
0380
0381 static inline uintptr_t
0382 _Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
0383 {
0384     _Py_atomic_ASSERT_ARG_TYPE(void*);
0385     return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj,
0386                                               (void*)value);
0387 }
0388
0389 static inline Py_ssize_t
0390 _Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
0391 {
0392     _Py_atomic_ASSERT_ARG_TYPE(void*);
0393     return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj,
0394                                                (void*)value);
0395 }
0396
0397
0398 // --- _Py_atomic_and --------------------------------------------------------
0399
0400 static inline uint8_t
0401 _Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
0402 {
0403     _Py_atomic_ASSERT_ARG_TYPE(char);
0404     return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value);
0405 }
0406
0407 static inline uint16_t
0408 _Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
0409 {
0410     _Py_atomic_ASSERT_ARG_TYPE(short);
0411     return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value);
0412 }
0413
0414 static inline uint32_t
0415 _Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
0416 {
0417     _Py_atomic_ASSERT_ARG_TYPE(long);
0418     return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value);
0419 }
0420
0421 static inline uint64_t
0422 _Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
0423 {
0424 #if defined(_M_X64) || defined(_M_ARM64)
0425     _Py_atomic_ASSERT_ARG_TYPE(__int64);
0426     return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value);
0427 #else
0428     uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
0429     for (;;) {
0430         uint64_t new_value = old_value & value;
0431         if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
0432             return old_value;
0433         }
0434     }
0435 #endif
0436 }
0437
0438 static inline uintptr_t
0439 _Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
0440 {
0441 #if SIZEOF_VOID_P == 8
0442     _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
0443     return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj,
0444                                             (uint64_t)value);
0445 #else
0446     _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
0447     return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj,
0448                                             (uint32_t)value);
0449 #endif
0450 }
0451
0452
0453 // --- _Py_atomic_or ---------------------------------------------------------
0454
0455 static inline uint8_t
0456 _Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
0457 {
0458     _Py_atomic_ASSERT_ARG_TYPE(char);
0459     return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value);
0460 }
0461
0462 static inline uint16_t
0463 _Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
0464 {
0465     _Py_atomic_ASSERT_ARG_TYPE(short);
0466     return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value);
0467 }
0468
0469 static inline uint32_t
0470 _Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
0471 {
0472     _Py_atomic_ASSERT_ARG_TYPE(long);
0473     return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value);
0474 }
0475
0476 static inline uint64_t
0477 _Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
0478 {
0479 #if defined(_M_X64) || defined(_M_ARM64)
0480     _Py_atomic_ASSERT_ARG_TYPE(__int64);
0481     return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value);
0482 #else
0483     uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
0484     for (;;) {
0485         uint64_t new_value = old_value | value;
0486         if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
0487             return old_value;
0488         }
0489     }
0490 #endif
0491 }
0492
0493
0494 static inline uintptr_t
0495 _Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
0496 {
0497 #if SIZEOF_VOID_P == 8
0498     _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
0499     return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj,
0500                                            (uint64_t)value);
0501 #else
0502     _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
0503     return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj,
0504                                            (uint32_t)value);
0505 #endif
0506 }
0507
0508
0509 // --- _Py_atomic_load -------------------------------------------------------
0510
0511 static inline uint8_t
0512 _Py_atomic_load_uint8(const uint8_t *obj)
0513 {
0514 #if defined(_M_X64) || defined(_M_IX86)
0515     return *(volatile uint8_t *)obj;
0516 #elif defined(_M_ARM64)
0517     return (uint8_t)__ldar8((unsigned __int8 volatile *)obj);
0518 #else
0519 #  error "no implementation of _Py_atomic_load_uint8"
0520 #endif
0521 }
0522
0523 static inline uint16_t
0524 _Py_atomic_load_uint16(const uint16_t *obj)
0525 {
0526 #if defined(_M_X64) || defined(_M_IX86)
0527     return *(volatile uint16_t *)obj;
0528 #elif defined(_M_ARM64)
0529     return (uint16_t)__ldar16((unsigned __int16 volatile *)obj);
0530 #else
0531 #  error "no implementation of _Py_atomic_load_uint16"
0532 #endif
0533 }
0534
0535 static inline uint32_t
0536 _Py_atomic_load_uint32(const uint32_t *obj)
0537 {
0538 #if defined(_M_X64) || defined(_M_IX86)
0539     return *(volatile uint32_t *)obj;
0540 #elif defined(_M_ARM64)
0541     return (uint32_t)__ldar32((unsigned __int32 volatile *)obj);
0542 #else
0543 #  error "no implementation of _Py_atomic_load_uint32"
0544 #endif
0545 }
0546
0547 static inline uint64_t
0548 _Py_atomic_load_uint64(const uint64_t *obj)
0549 {
0550 #if defined(_M_X64) || defined(_M_IX86)
0551     return *(volatile uint64_t *)obj;
0552 #elif defined(_M_ARM64)
0553     return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);
0554 #else
0555 #  error "no implementation of _Py_atomic_load_uint64"
0556 #endif
0557 }
0558
0559 static inline int8_t
0560 _Py_atomic_load_int8(const int8_t *obj)
0561 {
0562     return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj);
0563 }
0564
0565 static inline int16_t
0566 _Py_atomic_load_int16(const int16_t *obj)
0567 {
0568     return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj);
0569 }
0570
0571 static inline int32_t
0572 _Py_atomic_load_int32(const int32_t *obj)
0573 {
0574     return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj);
0575 }
0576
0577 static inline int
0578 _Py_atomic_load_int(const int *obj)
0579 {
0580     _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
0581     return (int)_Py_atomic_load_uint32((uint32_t *)obj);
0582 }
0583
0584 static inline unsigned int
0585 _Py_atomic_load_uint(const unsigned int *obj)
0586 {
0587     _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
0588     return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj);
0589 }
0590
0591 static inline int64_t
0592 _Py_atomic_load_int64(const int64_t *obj)
0593 {
0594     return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj);
0595 }
0596
0597 static inline void*
0598 _Py_atomic_load_ptr(const void *obj)
0599 {
0600 #if SIZEOF_VOID_P == 8
0601     return (void*)_Py_atomic_load_uint64((const uint64_t *)obj);
0602 #else
0603     return (void*)_Py_atomic_load_uint32((const uint32_t *)obj);
0604 #endif
0605 }
0606
0607 static inline intptr_t
0608 _Py_atomic_load_intptr(const intptr_t *obj)
0609 {
0610     _Py_atomic_ASSERT_ARG_TYPE(void*);
0611     return (intptr_t)_Py_atomic_load_ptr((void*)obj);
0612 }
0613
0614 static inline uintptr_t
0615 _Py_atomic_load_uintptr(const uintptr_t *obj)
0616 {
0617     _Py_atomic_ASSERT_ARG_TYPE(void*);
0618     return (uintptr_t)_Py_atomic_load_ptr((void*)obj);
0619 }
0620
0621 static inline Py_ssize_t
0622 _Py_atomic_load_ssize(const Py_ssize_t *obj)
0623 {
0624     _Py_atomic_ASSERT_ARG_TYPE(void*);
0625     return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj);
0626 }
0627
0628
0629 // --- _Py_atomic_load_relaxed -----------------------------------------------
0630
0631 static inline int
0632 _Py_atomic_load_int_relaxed(const int *obj)
0633 {
0634     return *(volatile int *)obj;
0635 }
0636
0637 static inline int8_t
0638 _Py_atomic_load_int8_relaxed(const int8_t *obj)
0639 {
0640     return *(volatile int8_t *)obj;
0641 }
0642
0643 static inline int16_t
0644 _Py_atomic_load_int16_relaxed(const int16_t *obj)
0645 {
0646     return *(volatile int16_t *)obj;
0647 }
0648
0649 static inline int32_t
0650 _Py_atomic_load_int32_relaxed(const int32_t *obj)
0651 {
0652     return *(volatile int32_t *)obj;
0653 }
0654
0655 static inline int64_t
0656 _Py_atomic_load_int64_relaxed(const int64_t *obj)
0657 {
0658     return *(volatile int64_t *)obj;
0659 }
0660
0661 static inline intptr_t
0662 _Py_atomic_load_intptr_relaxed(const intptr_t *obj)
0663 {
0664     return *(volatile intptr_t *)obj;
0665 }
0666
0667 static inline uint8_t
0668 _Py_atomic_load_uint8_relaxed(const uint8_t *obj)
0669 {
0670     return *(volatile uint8_t *)obj;
0671 }
0672
0673 static inline uint16_t
0674 _Py_atomic_load_uint16_relaxed(const uint16_t *obj)
0675 {
0676     return *(volatile uint16_t *)obj;
0677 }
0678
0679 static inline uint32_t
0680 _Py_atomic_load_uint32_relaxed(const uint32_t *obj)
0681 {
0682     return *(volatile uint32_t *)obj;
0683 }
0684
0685 static inline uint64_t
0686 _Py_atomic_load_uint64_relaxed(const uint64_t *obj)
0687 {
0688     return *(volatile uint64_t *)obj;
0689 }
0690
0691 static inline uintptr_t
0692 _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
0693 {
0694     return *(volatile uintptr_t *)obj;
0695 }
0696
0697 static inline unsigned int
0698 _Py_atomic_load_uint_relaxed(const unsigned int *obj)
0699 {
0700     return *(volatile unsigned int *)obj;
0701 }
0702
0703 static inline Py_ssize_t
0704 _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
0705 {
0706     return *(volatile Py_ssize_t *)obj;
0707 }
0708
0709 static inline void*
0710 _Py_atomic_load_ptr_relaxed(const void *obj)
0711 {
0712     return *(void * volatile *)obj;
0713 }
0714
0715 static inline unsigned long long
0716 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
0717 {
0718     return *(volatile unsigned long long *)obj;
0719 }
0720
0721
0722 // --- _Py_atomic_store ------------------------------------------------------
0723
0724 static inline void
0725 _Py_atomic_store_int(int *obj, int value)
0726 {
0727     (void)_Py_atomic_exchange_int(obj, value);
0728 }
0729
0730 static inline void
0731 _Py_atomic_store_int8(int8_t *obj, int8_t value)
0732 {
0733     (void)_Py_atomic_exchange_int8(obj, value);
0734 }
0735
0736 static inline void
0737 _Py_atomic_store_int16(int16_t *obj, int16_t value)
0738 {
0739     (void)_Py_atomic_exchange_int16(obj, value);
0740 }
0741
0742 static inline void
0743 _Py_atomic_store_int32(int32_t *obj, int32_t value)
0744 {
0745     (void)_Py_atomic_exchange_int32(obj, value);
0746 }
0747
0748 static inline void
0749 _Py_atomic_store_int64(int64_t *obj, int64_t value)
0750 {
0751     (void)_Py_atomic_exchange_int64(obj, value);
0752 }
0753
0754 static inline void
0755 _Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
0756 {
0757     (void)_Py_atomic_exchange_intptr(obj, value);
0758 }
0759
0760 static inline void
0761 _Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
0762 {
0763     (void)_Py_atomic_exchange_uint8(obj, value);
0764 }
0765
0766 static inline void
0767 _Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
0768 {
0769     (void)_Py_atomic_exchange_uint16(obj, value);
0770 }
0771
0772 static inline void
0773 _Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
0774 {
0775     (void)_Py_atomic_exchange_uint32(obj, value);
0776 }
0777
0778 static inline void
0779 _Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
0780 {
0781     (void)_Py_atomic_exchange_uint64(obj, value);
0782 }
0783
0784 static inline void
0785 _Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
0786 {
0787     (void)_Py_atomic_exchange_uintptr(obj, value);
0788 }
0789
0790 static inline void
0791 _Py_atomic_store_uint(unsigned int *obj, unsigned int value)
0792 {
0793     (void)_Py_atomic_exchange_uint(obj, value);
0794 }
0795
0796 static inline void
0797 _Py_atomic_store_ptr(void *obj, void *value)
0798 {
0799     (void)_Py_atomic_exchange_ptr(obj, value);
0800 }
0801
0802 static inline void
0803 _Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
0804 {
0805     (void)_Py_atomic_exchange_ssize(obj, value);
0806 }
0807
0808
0809 // --- _Py_atomic_store_relaxed ----------------------------------------------
0810
0811 static inline void
0812 _Py_atomic_store_int_relaxed(int *obj, int value)
0813 {
0814     *(volatile int *)obj = value;
0815 }
0816
0817 static inline void
0818 _Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
0819 {
0820     *(volatile int8_t *)obj = value;
0821 }
0822
0823 static inline void
0824 _Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
0825 {
0826     *(volatile int16_t *)obj = value;
0827 }
0828
0829 static inline void
0830 _Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
0831 {
0832     *(volatile int32_t *)obj = value;
0833 }
0834
0835 static inline void
0836 _Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
0837 {
0838     *(volatile int64_t *)obj = value;
0839 }
0840
0841 static inline void
0842 _Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
0843 {
0844     *(volatile intptr_t *)obj = value;
0845 }
0846
0847 static inline void
0848 _Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
0849 {
0850     *(volatile uint8_t *)obj = value;
0851 }
0852
0853 static inline void
0854 _Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
0855 {
0856     *(volatile uint16_t *)obj = value;
0857 }
0858
0859 static inline void
0860 _Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
0861 {
0862     *(volatile uint32_t *)obj = value;
0863 }
0864
0865 static inline void
0866 _Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
0867 {
0868     *(volatile uint64_t *)obj = value;
0869 }
0870
0871 static inline void
0872 _Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
0873 {
0874     *(volatile uintptr_t *)obj = value;
0875 }
0876
0877 static inline void
0878 _Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
0879 {
0880     *(volatile unsigned int *)obj = value;
0881 }
0882
0883 static inline void
0884 _Py_atomic_store_ptr_relaxed(void *obj, void* value)
0885 {
0886     *(void * volatile *)obj = value;
0887 }
0888
0889 static inline void
0890 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
0891 {
0892     *(volatile Py_ssize_t *)obj = value;
0893 }
0894
0895 static inline void
0896 _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
0897                                 unsigned long long value)
0898 {
0899     *(volatile unsigned long long *)obj = value;
0900 }
0901
0902
0903 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
0904
0905 static inline void *
0906 _Py_atomic_load_ptr_acquire(const void *obj)
0907 {
0908 #if defined(_M_X64) || defined(_M_IX86)
0909     return *(void * volatile *)obj;
0910 #elif defined(_M_ARM64)
0911     return (void *)__ldar64((unsigned __int64 volatile *)obj);
0912 #else
0913 #  error "no implementation of _Py_atomic_load_ptr_acquire"
0914 #endif
0915 }
0916
0917 static inline uintptr_t
0918 _Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
0919 {
0920 #if defined(_M_X64) || defined(_M_IX86)
0921     return *(uintptr_t volatile *)obj;
0922 #elif defined(_M_ARM64)
0923     return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj);
0924 #else
0925 #  error "no implementation of _Py_atomic_load_uintptr_acquire"
0926 #endif
0927 }
0928
0929 static inline void
0930 _Py_atomic_store_ptr_release(void *obj, void *value)
0931 {
0932 #if defined(_M_X64) || defined(_M_IX86)
0933     *(void * volatile *)obj = value;
0934 #elif defined(_M_ARM64)
0935     __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value);
0936 #else
0937 #  error "no implementation of _Py_atomic_store_ptr_release"
0938 #endif
0939 }
0940
0941 static inline void
0942 _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
0943 {
0944 #if defined(_M_X64) || defined(_M_IX86)
0945     *(uintptr_t volatile *)obj = value;
0946 #elif defined(_M_ARM64)
0947     _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
0948     __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
0949 #else
0950 #  error "no implementation of _Py_atomic_store_uintptr_release"
0951 #endif
0952 }
0953
0954 static inline void
0955 _Py_atomic_store_int_release(int *obj, int value)
0956 {
0957 #if defined(_M_X64) || defined(_M_IX86)
0958     *(int volatile *)obj = value;
0959 #elif defined(_M_ARM64)
0960     _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
0961     __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);
0962 #else
0963 #  error "no implementation of _Py_atomic_store_int_release"
0964 #endif
0965 }
0966
0967 static inline void
0968 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
0969 {
0970 #if defined(_M_X64) || defined(_M_IX86)
0971     *(Py_ssize_t volatile *)obj = value;
0972 #elif defined(_M_ARM64)
0973     __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
0974 #else
0975 #  error "no implementation of _Py_atomic_store_ssize_release"
0976 #endif
0977 }
0978
0979 static inline int
0980 _Py_atomic_load_int_acquire(const int *obj)
0981 {
0982 #if defined(_M_X64) || defined(_M_IX86)
0983     return *(int volatile *)obj;
0984 #elif defined(_M_ARM64)
0985     _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
0986     return (int)__ldar32((unsigned __int32 volatile *)obj);
0987 #else
0988 #  error "no implementation of _Py_atomic_load_int_acquire"
0989 #endif
0990 }
0991
0992 static inline void
0993 _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)
0994 {
0995 #if defined(_M_X64) || defined(_M_IX86)
0996     *(uint32_t volatile *)obj = value;
0997 #elif defined(_M_ARM64)
0998     _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
0999     __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);
1000 #else
1001 #  error "no implementation of _Py_atomic_store_uint32_release"
1002 #endif
1003 }
1004
1005 static inline void
1006 _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
1007 {
1008 #if defined(_M_X64) || defined(_M_IX86)
1009     *(uint64_t volatile *)obj = value;
1010 #elif defined(_M_ARM64)
1011     _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
1012     __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
1013 #else
1014 #  error "no implementation of _Py_atomic_store_uint64_release"
1015 #endif
1016 }
1017
1018 static inline uint64_t
1019 _Py_atomic_load_uint64_acquire(const uint64_t *obj)
1020 {
1021 #if defined(_M_X64) || defined(_M_IX86)
1022     return *(uint64_t volatile *)obj;
1023 #elif defined(_M_ARM64)
1024     _Py_atomic_ASSERT_ARG_TYPE(__int64);
1025     return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);
1026 #else
1027 #  error "no implementation of _Py_atomic_load_uint64_acquire"
1028 #endif
1029 }
1030
1031 static inline uint32_t
1032 _Py_atomic_load_uint32_acquire(const uint32_t *obj)
1033 {
1034 #if defined(_M_X64) || defined(_M_IX86)
1035     return *(uint32_t volatile *)obj;
1036 #elif defined(_M_ARM64)
1037     return (uint32_t)__ldar32((uint32_t volatile *)obj);
1038 #else
1039 #  error "no implementation of _Py_atomic_load_uint32_acquire"
1040 #endif
1041 }
1042
1043 static inline Py_ssize_t
1044 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)
1045 {
1046 #if defined(_M_X64) || defined(_M_IX86)
1047     return *(Py_ssize_t volatile *)obj;
1048 #elif defined(_M_ARM64)
1049     return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj);
1050 #else
1051 #  error "no implementation of _Py_atomic_load_ssize_acquire"
1052 #endif
1053 }
1054
1055 // --- _Py_atomic_fence ------------------------------------------------------
1056
1057  static inline void
1058 _Py_atomic_fence_seq_cst(void)
1059 {
1060 #if defined(_M_ARM64)
1061     __dmb(_ARM64_BARRIER_ISH);
1062 #elif defined(_M_X64)
1063     __faststorefence();
1064 #elif defined(_M_IX86)
1065     _mm_mfence();
1066 #else
1067 #  error "no implementation of _Py_atomic_fence_seq_cst"
1068 #endif
1069 }
1070
1071  static inline void
1072 _Py_atomic_fence_acquire(void)
1073 {
1074 #if defined(_M_ARM64)
1075     __dmb(_ARM64_BARRIER_ISHLD);
1076 #elif defined(_M_X64) || defined(_M_IX86)
1077     _ReadBarrier();
1078 #else
1079 #  error "no implementation of _Py_atomic_fence_acquire"
1080 #endif
1081 }
1082
1083  static inline void
1084 _Py_atomic_fence_release(void)
1085 {
1086 #if defined(_M_ARM64)
1087     __dmb(_ARM64_BARRIER_ISH);
1088 #elif defined(_M_X64) || defined(_M_IX86)
1089     _ReadWriteBarrier();
1090 #else
1091 #  error "no implementation of _Py_atomic_fence_release"
1092 #endif
1093 }
1094
1095 #undef _Py_atomic_ASSERT_ARG_TYPE