atomic/detail/core_arch_ops_gcc_aarch32.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2020 Andrey Semashev
0007  */
0008 /*!
0009  * \file   atomic/detail/core_arch_ops_gcc_aarch32.hpp
0010  *
0011  * This header contains implementation of the \c core_arch_operations template.
0012  */
0013
0014 #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_AARCH32_HPP_INCLUDED_
0015 #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_AARCH32_HPP_INCLUDED_
0016
0017 #include <cstddef>
0018 #include <boost/cstdint.hpp>
0019 #include <boost/memory_order.hpp>
0020 #include <boost/atomic/detail/config.hpp>
0021 #include <boost/atomic/detail/storage_traits.hpp>
0022 #include <boost/atomic/detail/core_arch_operations_fwd.hpp>
0023 #include <boost/atomic/detail/capabilities.hpp>
0024 #include <boost/atomic/detail/ops_gcc_aarch32_common.hpp>
0025 #include <boost/atomic/detail/header.hpp>
0026
0027 #ifdef BOOST_HAS_PRAGMA_ONCE
0028 #pragma once
0029 #endif
0030
0031 namespace boost {
0032 namespace atomics {
0033 namespace detail {
0034
0035 // ARMv8 (AArch32) instruction set is similar to ARMv7, but adds
0036 // lda(b/h) and ldaex(b/h/d) instructions for acquire loads and
0037 // stl(b/h) and stlex(b/h/d) instructions for release stores. This
0038 // makes explicit memory fences unnecessary for implementation of
0039 // the majority of the atomic operations.
0040 //
0041 // ARMv8 deprecates applying "it" hints to some instructions, including
0042 // strex. It also deprecates "it" hints applying to more than one
0043 // of the following conditional instructions. This means we have to
0044 // use conditional jumps instead of making other instructions conditional.
0045
0046 struct core_arch_operations_gcc_aarch32_base
0047 {
0048     static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = false;
0049     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
0050 };
0051
0052 template< bool Signed, bool Interprocess >
0053 struct core_arch_operations< 1u, Signed, Interprocess > :
0054     public core_arch_operations_gcc_aarch32_base
0055 {
0056     typedef typename storage_traits< 1u >::type storage_type;
0057
0058     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
0059     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 1u;
0060     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0061     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0062
0063     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0064     {
0065         if ((static_cast< unsigned int >(order) & static_cast< unsigned int >(memory_order_release)) != 0u)
0066         {
0067             __asm__ __volatile__
0068             (
0069                 "stlb %[value], %[storage]\n\t"
0070                 : [storage] "=Q" (storage)
0071                 : [value] "r" (v)
0072                 : "memory"
0073             );
0074         }
0075         else
0076         {
0077             storage = v;
0078         }
0079     }
0080
0081     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0082     {
0083         storage_type v;
0084         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0085         {
0086             __asm__ __volatile__
0087             (
0088                 "ldab %[value], %[storage]\n\t"
0089                 : [value] "=r" (v)
0090                 : [storage] "Q" (storage)
0091                 : "memory"
0092             );
0093         }
0094         else
0095         {
0096             v = storage;
0097         }
0098
0099         return v;
0100     }
0101
0102     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0103     {
0104         storage_type original;
0105         uint32_t tmp;
0106 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0107         __asm__ __volatile__\
0108         (\
0109             "1:\n\t"\
0110             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0111             "st" st_mo "exb %[tmp], %[value], %[storage]\n\t"\
0112             "teq %[tmp], #0\n\t"\
0113             "bne 1b\n\t"\
0114             : [tmp] "=&r" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)\
0115             : [value] "r" (v)\
0116             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0117         );
0118
0119         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0120 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0121
0122         return original;
0123     }
0124
0125     static BOOST_FORCEINLINE bool compare_exchange_weak(
0126         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0127     {
0128         storage_type original;
0129         bool success;
0130 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0131         __asm__ __volatile__\
0132         (\
0133             "uxtb %[expected], %[expected]\n\t"\
0134             "mov %[success], #0\n\t"\
0135             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0136             "cmp %[original], %[expected]\n\t"\
0137             "bne 1f\n\t"\
0138             "st" st_mo "exb %[success], %[desired], %[storage]\n\t"\
0139             "eor %[success], %[success], #1\n\t"\
0140             "1:\n\t"\
0141             : [original] "=&r" (original), [success] "=&r" (success), [storage] "+Q" (storage)\
0142             : [expected] "r" (expected), [desired] "r" (desired)\
0143             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0144         );
0145
0146         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0147 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0148
0149         expected = original;
0150         return success;
0151     }
0152
0153     static BOOST_FORCEINLINE bool compare_exchange_strong(
0154         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0155     {
0156         storage_type original;
0157         bool success;
0158 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0159         __asm__ __volatile__\
0160         (\
0161             "uxtb %[expected], %[expected]\n\t"\
0162             "mov %[success], #0\n\t"\
0163             "1:\n\t"\
0164             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0165             "cmp %[original], %[expected]\n\t"\
0166             "bne 2f\n\t"\
0167             "st" st_mo "exb %[success], %[desired], %[storage]\n\t"\
0168             "eors %[success], %[success], #1\n\t"\
0169             "beq 1b\n\t"\
0170             "2:\n\t"\
0171             : [original] "=&r" (original), [success] "=&r" (success), [storage] "+Q" (storage)\
0172             : [expected] "r" (expected), [desired] "r" (desired)\
0173             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0174         );
0175
0176         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0177 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0178
0179         expected = original;
0180         return success;
0181     }
0182
0183     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0184     {
0185         storage_type original, result;
0186         uint32_t tmp;
0187 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0188         __asm__ __volatile__\
0189         (\
0190             "1:\n\t"\
0191             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0192             "add %[result], %[original], %[value]\n\t"\
0193             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0194             "teq %[tmp], #0\n\t"\
0195             "bne 1b\n\t"\
0196             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0197             : [value] "Ir" (v)\
0198             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0199         );
0200
0201         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0202 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0203
0204         return original;
0205     }
0206
0207     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0208     {
0209         storage_type original, result;
0210         uint32_t tmp;
0211 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0212         __asm__ __volatile__\
0213         (\
0214             "1:\n\t"\
0215             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0216             "sub %[result], %[original], %[value]\n\t"\
0217             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0218             "teq %[tmp], #0\n\t"\
0219             "bne 1b\n\t"\
0220             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0221             : [value] "Ir" (v)\
0222             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0223         );
0224
0225         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0226 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0227
0228         return original;
0229     }
0230
0231     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0232     {
0233         storage_type original, result;
0234         uint32_t tmp;
0235 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0236         __asm__ __volatile__\
0237         (\
0238             "1:\n\t"\
0239             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0240             "and %[result], %[original], %[value]\n\t"\
0241             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0242             "teq %[tmp], #0\n\t"\
0243             "bne 1b\n\t"\
0244             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0245             : [value] "Ir" (v)\
0246             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0247         );
0248
0249         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0250 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0251
0252         return original;
0253     }
0254
0255     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0256     {
0257         storage_type original, result;
0258         uint32_t tmp;
0259 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0260         __asm__ __volatile__\
0261         (\
0262             "1:\n\t"\
0263             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0264             "orr %[result], %[original], %[value]\n\t"\
0265             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0266             "teq %[tmp], #0\n\t"\
0267             "bne 1b\n\t"\
0268             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0269             : [value] "Ir" (v)\
0270             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0271         );
0272
0273         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0274 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0275
0276         return original;
0277     }
0278
0279     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0280     {
0281         storage_type original, result;
0282         uint32_t tmp;
0283 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0284         __asm__ __volatile__\
0285         (\
0286             "1:\n\t"\
0287             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0288             "eor %[result], %[original], %[value]\n\t"\
0289             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0290             "teq %[tmp], #0\n\t"\
0291             "bne 1b\n\t"\
0292             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0293             : [value] "Ir" (v)\
0294             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0295         );
0296
0297         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0298 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0299
0300         return original;
0301     }
0302
0303     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0304     {
0305         return !!exchange(storage, (storage_type)1, order);
0306     }
0307
0308     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0309     {
0310         store(storage, (storage_type)0, order);
0311     }
0312 };
0313
0314 template< bool Signed, bool Interprocess >
0315 struct core_arch_operations< 2u, Signed, Interprocess > :
0316     public core_arch_operations_gcc_aarch32_base
0317 {
0318     typedef typename storage_traits< 2u >::type storage_type;
0319
0320     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
0321     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 2u;
0322     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0323     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0324
0325     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0326     {
0327         if ((static_cast< unsigned int >(order) & static_cast< unsigned int >(memory_order_release)) != 0u)
0328         {
0329             __asm__ __volatile__
0330             (
0331                 "stlh %[value], %[storage]\n\t"
0332                 : [storage] "=Q" (storage)
0333                 : [value] "r" (v)
0334                 : "memory"
0335             );
0336         }
0337         else
0338         {
0339             storage = v;
0340         }
0341     }
0342
0343     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0344     {
0345         storage_type v;
0346         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0347         {
0348             __asm__ __volatile__
0349             (
0350                 "ldah %[value], %[storage]\n\t"
0351                 : [value] "=r" (v)
0352                 : [storage] "Q" (storage)
0353                 : "memory"
0354             );
0355         }
0356         else
0357         {
0358             v = storage;
0359         }
0360
0361         return v;
0362     }
0363
0364     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0365     {
0366         storage_type original;
0367         uint32_t tmp;
0368 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0369         __asm__ __volatile__\
0370         (\
0371             "1:\n\t"\
0372             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0373             "st" st_mo "exh %[tmp], %[value], %[storage]\n\t"\
0374             "teq %[tmp], #0\n\t"\
0375             "bne 1b\n\t"\
0376             : [tmp] "=&r" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)\
0377             : [value] "r" (v)\
0378             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0379         );
0380
0381         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0382 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0383
0384         return original;
0385     }
0386
0387     static BOOST_FORCEINLINE bool compare_exchange_weak(
0388         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0389     {
0390         storage_type original;
0391         bool success;
0392 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0393         __asm__ __volatile__\
0394         (\
0395             "uxth %[expected], %[expected]\n\t"\
0396             "mov %[success], #0\n\t"\
0397             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0398             "cmp %[original], %[expected]\n\t"\
0399             "bne 1f\n\t"\
0400             "st" st_mo "exh %[success], %[desired], %[storage]\n\t"\
0401             "eor %[success], %[success], #1\n\t"\
0402             "1:\n\t"\
0403             : [original] "=&r" (original), [success] "=&r" (success), [storage] "+Q" (storage)\
0404             : [expected] "r" (expected), [desired] "r" (desired)\
0405             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0406         );
0407
0408         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0409 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0410
0411         expected = original;
0412         return success;
0413     }
0414
0415     static BOOST_FORCEINLINE bool compare_exchange_strong(
0416         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0417     {
0418         storage_type original;
0419         bool success;
0420 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0421         __asm__ __volatile__\
0422         (\
0423             "uxth %[expected], %[expected]\n\t"\
0424             "mov %[success], #0\n\t"\
0425             "1:\n\t"\
0426             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0427             "cmp %[original], %[expected]\n\t"\
0428             "bne 2f\n\t"\
0429             "st" st_mo "exh %[success], %[desired], %[storage]\n\t"\
0430             "eors %[success], %[success], #1\n\t"\
0431             "beq 1b\n\t"\
0432             "2:\n\t"\
0433             : [original] "=&r" (original), [success] "=&r" (success), [storage] "+Q" (storage)\
0434             : [expected] "r" (expected), [desired] "r" (desired)\
0435             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0436         );
0437
0438         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0439 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0440
0441         expected = original;
0442         return success;
0443     }
0444
0445     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0446     {
0447         storage_type original, result;
0448         uint32_t tmp;
0449 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0450         __asm__ __volatile__\
0451         (\
0452             "1:\n\t"\
0453             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0454             "add %[result], %[original], %[value]\n\t"\
0455             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0456             "teq %[tmp], #0\n\t"\
0457             "bne 1b\n\t"\
0458             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0459             : [value] "Ir" (v)\
0460             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0461         );
0462
0463         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0464 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0465
0466         return original;
0467     }
0468
0469     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0470     {
0471         storage_type original, result;
0472         uint32_t tmp;
0473 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0474         __asm__ __volatile__\
0475         (\
0476             "1:\n\t"\
0477             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0478             "sub %[result], %[original], %[value]\n\t"\
0479             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0480             "teq %[tmp], #0\n\t"\
0481             "bne 1b\n\t"\
0482             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0483             : [value] "Ir" (v)\
0484             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0485         );
0486
0487         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0488 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0489
0490         return original;
0491     }
0492
0493     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0494     {
0495         storage_type original, result;
0496         uint32_t tmp;
0497 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0498         __asm__ __volatile__\
0499         (\
0500             "1:\n\t"\
0501             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0502             "and %[result], %[original], %[value]\n\t"\
0503             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0504             "teq %[tmp], #0\n\t"\
0505             "bne 1b\n\t"\
0506             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0507             : [value] "Ir" (v)\
0508             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0509         );
0510
0511         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0512 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0513
0514         return original;
0515     }
0516
0517     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0518     {
0519         storage_type original, result;
0520         uint32_t tmp;
0521 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0522         __asm__ __volatile__\
0523         (\
0524             "1:\n\t"\
0525             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0526             "orr %[result], %[original], %[value]\n\t"\
0527             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0528             "teq %[tmp], #0\n\t"\
0529             "bne 1b\n\t"\
0530             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0531             : [value] "Ir" (v)\
0532             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0533         );
0534
0535         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0536 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0537
0538         return original;
0539     }
0540
0541     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0542     {
0543         storage_type original, result;
0544         uint32_t tmp;
0545 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0546         __asm__ __volatile__\
0547         (\
0548             "1:\n\t"\
0549             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0550             "eor %[result], %[original], %[value]\n\t"\
0551             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0552             "teq %[tmp], #0\n\t"\
0553             "bne 1b\n\t"\
0554             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0555             : [value] "Ir" (v)\
0556             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0557         );
0558
0559         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0560 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0561
0562         return original;
0563     }
0564
0565     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0566     {
0567         return !!exchange(storage, (storage_type)1, order);
0568     }
0569
0570     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0571     {
0572         store(storage, (storage_type)0, order);
0573     }
0574 };
0575
0576 template< bool Signed, bool Interprocess >
0577 struct core_arch_operations< 4u, Signed, Interprocess > :
0578     public core_arch_operations_gcc_aarch32_base
0579 {
0580     typedef typename storage_traits< 4u >::type storage_type;
0581
0582     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
0583     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u;
0584     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0585     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0586
0587     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0588     {
0589         if ((static_cast< unsigned int >(order) & static_cast< unsigned int >(memory_order_release)) != 0u)
0590         {
0591             __asm__ __volatile__
0592             (
0593                 "stl %[value], %[storage]\n\t"
0594                 : [storage] "=Q" (storage)
0595                 : [value] "r" (v)
0596                 : "memory"
0597             );
0598         }
0599         else
0600         {
0601             storage = v;
0602         }
0603     }
0604
0605     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0606     {
0607         storage_type v;
0608         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0609         {
0610             __asm__ __volatile__
0611             (
0612                 "lda %[value], %[storage]\n\t"
0613                 : [value] "=r" (v)
0614                 : [storage] "Q" (storage)
0615                 : "memory"
0616             );
0617         }
0618         else
0619         {
0620             v = storage;
0621         }
0622
0623         return v;
0624     }
0625
0626     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0627     {
0628         storage_type original;
0629         uint32_t tmp;
0630 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0631         __asm__ __volatile__\
0632         (\
0633             "1:\n\t"\
0634             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0635             "st" st_mo "ex %[tmp], %[value], %[storage]\n\t"\
0636             "teq %[tmp], #0\n\t"\
0637             "bne 1b\n\t"\
0638             : [tmp] "=&r" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)\
0639             : [value] "r" (v)\
0640             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0641         );
0642
0643         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0644 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0645
0646         return original;
0647     }
0648
0649     static BOOST_FORCEINLINE bool compare_exchange_weak(
0650         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0651     {
0652         storage_type original;
0653         bool success;
0654 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0655         __asm__ __volatile__\
0656         (\
0657             "mov %[success], #0\n\t"\
0658             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0659             "cmp %[original], %[expected]\n\t"\
0660             "bne 1f\n\t"\
0661             "st" st_mo "ex %[success], %[desired], %[storage]\n\t"\
0662             "eor %[success], %[success], #1\n\t"\
0663             "1:\n\t"\
0664             : [original] "=&r" (original), [success] "=&r" (success), [storage] "+Q" (storage)\
0665             : [expected] "Ir" (expected), [desired] "r" (desired)\
0666             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0667         );
0668
0669         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0670 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0671
0672         expected = original;
0673         return success;
0674     }
0675
0676     static BOOST_FORCEINLINE bool compare_exchange_strong(
0677         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0678     {
0679         storage_type original;
0680         bool success;
0681 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0682         __asm__ __volatile__\
0683         (\
0684             "mov %[success], #0\n\t"\
0685             "1:\n\t"\
0686             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0687             "cmp %[original], %[expected]\n\t"\
0688             "bne 2f\n\t"\
0689             "st" st_mo "ex %[success], %[desired], %[storage]\n\t"\
0690             "eors %[success], %[success], #1\n\t"\
0691             "beq 1b\n\t"\
0692             "2:\n\t"\
0693             : [original] "=&r" (original), [success] "=&r" (success), [storage] "+Q" (storage)\
0694             : [expected] "Ir" (expected), [desired] "r" (desired)\
0695             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0696         );
0697
0698         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0699 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0700
0701         expected = original;
0702         return success;
0703     }
0704
0705     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0706     {
0707         storage_type original, result;
0708         uint32_t tmp;
0709 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0710         __asm__ __volatile__\
0711         (\
0712             "1:\n\t"\
0713             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0714             "add %[result], %[original], %[value]\n\t"\
0715             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0716             "teq %[tmp], #0\n\t"\
0717             "bne 1b\n\t"\
0718             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0719             : [value] "Ir" (v)\
0720             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0721         );
0722
0723         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0724 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0725
0726         return original;
0727     }
0728
0729     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0730     {
0731         storage_type original, result;
0732         uint32_t tmp;
0733 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0734         __asm__ __volatile__\
0735         (\
0736             "1:\n\t"\
0737             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0738             "sub %[result], %[original], %[value]\n\t"\
0739             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0740             "teq %[tmp], #0\n\t"\
0741             "bne 1b\n\t"\
0742             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0743             : [value] "Ir" (v)\
0744             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0745         );
0746
0747         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0748 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0749
0750         return original;
0751     }
0752
0753     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0754     {
0755         storage_type original, result;
0756         uint32_t tmp;
0757 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0758         __asm__ __volatile__\
0759         (\
0760             "1:\n\t"\
0761             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0762             "and %[result], %[original], %[value]\n\t"\
0763             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0764             "teq %[tmp], #0\n\t"\
0765             "bne 1b\n\t"\
0766             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0767             : [value] "Ir" (v)\
0768             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0769         );
0770
0771         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0772 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0773
0774         return original;
0775     }
0776
0777     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0778     {
0779         storage_type original, result;
0780         uint32_t tmp;
0781 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0782         __asm__ __volatile__\
0783         (\
0784             "1:\n\t"\
0785             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0786             "orr %[result], %[original], %[value]\n\t"\
0787             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0788             "teq %[tmp], #0\n\t"\
0789             "bne 1b\n\t"\
0790             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0791             : [value] "Ir" (v)\
0792             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0793         );
0794
0795         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0796 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0797
0798         return original;
0799     }
0800
0801     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0802     {
0803         storage_type original, result;
0804         uint32_t tmp;
0805 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0806         __asm__ __volatile__\
0807         (\
0808             "1:\n\t"\
0809             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0810             "eor %[result], %[original], %[value]\n\t"\
0811             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0812             "teq %[tmp], #0\n\t"\
0813             "bne 1b\n\t"\
0814             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0815             : [value] "Ir" (v)\
0816             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0817         );
0818
0819         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0820 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0821
0822         return original;
0823     }
0824
0825     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0826     {
0827         return !!exchange(storage, (storage_type)1, order);
0828     }
0829
0830     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0831     {
0832         store(storage, (storage_type)0, order);
0833     }
0834 };
0835
0836
0837 // Unlike 32-bit operations, for 64-bit loads and stores we must use ldrexd/strexd.
0838 // Other instructions result in a non-atomic sequence of 32-bit or more fine-grained accesses.
0839 // See "ARM Architecture Reference Manual ARMv8, for ARMv8-A architecture profile", Section E2.2 "Atomicity in the ARM architecture".
0840 // Section E2.3.7 "Memory barriers", subsection "Load-Acquire, Store-Release" extends atomicity guarantees given for ldrexd/strexd
0841 // to the new ldaexd/stlexd instructions with acquire/release semantics.
0842 //
0843 // In the asm blocks below we have to use 32-bit register pairs to compose 64-bit values. In order to pass the 64-bit operands
0844 // to/from asm blocks, we use undocumented gcc feature: the lower half (Rt) of the operand is accessible normally, via the numbered
0845 // placeholder (e.g. %0), and the upper half (Rt2) - via the same placeholder with an 'H' after the '%' sign (e.g. %H0).
0846 // See: http://hardwarebug.org/2010/07/06/arm-inline-asm-secrets/
0847 //
0848 // The ldrexd and strexd instructions operate on pairs of registers, meaning that each load loads two integers from memory in
0849 // successive address order, to the first and second registers in the pair, respectively, and store similarly stores two integers.
0850 // The order of these integers does not depend on the active endianness mode (although the byte order in the integers themselves
0851 // obviously does depend on endianness). This means we need to account for the current endianness mode ourselves, where it matters.
0852
0853 template< bool Signed, bool Interprocess >
0854 struct core_arch_operations< 8u, Signed, Interprocess > :
0855     public core_arch_operations_gcc_aarch32_base
0856 {
0857     typedef typename storage_traits< 8u >::type storage_type;
0858
0859     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
0860     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u;
0861     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0862     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0863
0864     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0865     {
0866         exchange(storage, v, order);
0867     }
0868
0869     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0870     {
0871         storage_type original;
0872         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0873         {
0874             __asm__ __volatile__
0875             (
0876                 "ldaexd %0, %H0, %1\n\t"
0877                 : "=&r" (original)   // %0
0878                 : "Q" (storage)      // %1
0879             );
0880         }
0881         else
0882         {
0883             __asm__ __volatile__
0884             (
0885                 "ldrexd %0, %H0, %1\n\t"
0886                 : "=&r" (original)   // %0
0887                 : "Q" (storage)      // %1
0888             );
0889         }
0890
0891         return original;
0892     }
0893
0894     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0895     {
0896         storage_type original;
0897         uint32_t tmp;
0898 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0899         __asm__ __volatile__\
0900         (\
0901             "1:\n\t"\
0902             "ld" ld_mo "exd %1, %H1, %2\n\t"\
0903             "st" st_mo "exd %0, %3, %H3, %2\n\t"\
0904             "teq %0, #0\n\t"\
0905             "bne 1b\n\t"\
0906             : "=&r" (tmp), "=&r" (original), "+Q" (storage)\
0907             : "r" (v)\
0908             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0909         );
0910
0911         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0912 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0913
0914         return original;
0915     }
0916
0917     static BOOST_FORCEINLINE bool compare_exchange_weak(
0918         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0919     {
0920         storage_type original;
0921         bool success;
0922 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0923         __asm__ __volatile__\
0924         (\
0925             "mov %1, #0\n\t"\
0926             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0927             "cmp %0, %3\n\t"\
0928             "it eq\n\t"\
0929             "cmpeq %H0, %H3\n\t"\
0930             "bne 1f\n\t"\
0931             "st" st_mo "exd %1, %4, %H4, %2\n\t"\
0932             "eor %1, %1, #1\n\t"\
0933             "1:\n\t"\
0934             : "=&r" (original), "=&r" (success), "+Q" (storage)\
0935             : "r" (expected), "r" (desired)\
0936             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0937         );
0938
0939         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0940 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0941
0942         expected = original;
0943
0944         return success;
0945     }
0946
0947     static BOOST_FORCEINLINE bool compare_exchange_strong(
0948         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0949     {
0950         storage_type original;
0951         bool success;
0952 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0953         __asm__ __volatile__\
0954         (\
0955             "mov %1, #0\n\t"\
0956             "1:\n\t"\
0957             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0958             "cmp %0, %3\n\t"\
0959             "it eq\n\t"\
0960             "cmpeq %H0, %H3\n\t"\
0961             "bne 2f\n\t"\
0962             "st" st_mo "exd %1, %4, %H4, %2\n\t"\
0963             "eors %1, %1, #1\n\t"\
0964             "beq 1b\n\t"\
0965             "2:\n\t"\
0966             : "=&r" (original), "=&r" (success), "+Q" (storage)\
0967             : "r" (expected), "r" (desired)\
0968             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0969         );
0970
0971         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(success_order)
0972 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0973
0974         expected = original;
0975
0976         return success;
0977     }
0978
0979     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0980     {
0981         storage_type original, result;
0982         uint32_t tmp;
0983 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0984         __asm__ __volatile__\
0985         (\
0986             "1:\n\t"\
0987             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0988             "adds " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(3) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(4) "\n\t"\
0989             "adc " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(3) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(4) "\n\t"\
0990             "st" st_mo "exd %1, %3, %H3, %2\n\t"\
0991             "teq %1, #0\n\t"\
0992             "bne 1b\n\t"\
0993             : "=&r" (original), "=&r" (tmp), "+Q" (storage), "=&r" (result)\
0994             : "r" (v)\
0995             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0996         );
0997
0998         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0999 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
1000
1001         return original;
1002     }
1003
1004     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1005     {
1006         storage_type original, result;
1007         uint32_t tmp;
1008 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
1009         __asm__ __volatile__\
1010         (\
1011             "1:\n\t"\
1012             "ld" ld_mo "exd %0, %H0, %2\n\t"\
1013             "subs " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(3) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(4) "\n\t"\
1014             "sbc " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(3) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(4) "\n\t"\
1015             "st" st_mo "exd %1, %3, %H3, %2\n\t"\
1016             "teq %1, #0\n\t"\
1017             "bne 1b\n\t"\
1018             : "=&r" (original), "=&r" (tmp), "+Q" (storage), "=&r" (result)\
1019             : "r" (v)\
1020             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1021         );
1022
1023         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
1024 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
1025
1026         return original;
1027     }
1028
1029     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1030     {
1031         storage_type original, result;
1032         uint32_t tmp;
1033 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
1034         __asm__ __volatile__\
1035         (\
1036             "1:\n\t"\
1037             "ld" ld_mo "exd %0, %H0, %2\n\t"\
1038             "and %3, %0, %4\n\t"\
1039             "and %H3, %H0, %H4\n\t"\
1040             "st" st_mo "exd %1, %3, %H3, %2\n\t"\
1041             "teq %1, #0\n\t"\
1042             "bne 1b\n\t"\
1043             : "=&r" (original), "=&r" (tmp), "+Q" (storage), "=&r" (result)\
1044             : "r" (v)\
1045             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1046         );
1047
1048         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
1049 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
1050
1051         return original;
1052     }
1053
1054     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1055     {
1056         storage_type original, result;
1057         uint32_t tmp;
1058 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
1059         __asm__ __volatile__\
1060         (\
1061             "1:\n\t"\
1062             "ld" ld_mo "exd %0, %H0, %2\n\t"\
1063             "orr %3, %0, %4\n\t"\
1064             "orr %H3, %H0, %H4\n\t"\
1065             "st" st_mo "exd %1, %3, %H3, %2\n\t"\
1066             "teq %1, #0\n\t"\
1067             "bne 1b\n\t"\
1068             : "=&r" (original), "=&r" (tmp), "+Q" (storage), "=&r" (result)\
1069             : "r" (v)\
1070             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1071         );
1072
1073         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
1074 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
1075
1076         return original;
1077     }
1078
1079     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1080     {
1081         storage_type original, result;
1082         uint32_t tmp;
1083 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
1084         __asm__ __volatile__\
1085         (\
1086             "1:\n\t"\
1087             "ld" ld_mo "exd %0, %H0, %2\n\t"\
1088             "eor %3, %0, %4\n\t"\
1089             "eor %H3, %H0, %H4\n\t"\
1090             "st" st_mo "exd %1, %3, %H3, %2\n\t"\
1091             "teq %1, #0\n\t"\
1092             "bne 1b\n\t"\
1093             : "=&r" (original), "=&r" (tmp), "+Q" (storage), "=&r" (result)\
1094             : "r" (v)\
1095             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1096         );
1097
1098         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
1099 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
1100
1101         return original;
1102     }
1103
1104     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1105     {
1106         return !!exchange(storage, (storage_type)1, order);
1107     }
1108
1109     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1110     {
1111         store(storage, (storage_type)0, order);
1112     }
1113 };
1114
1115 } // namespace detail
1116 } // namespace atomics
1117 } // namespace boost
1118
1119 #include <boost/atomic/detail/footer.hpp>
1120
1121 #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_AARCH32_HPP_INCLUDED_