atomic/detail/extra_ops_gcc_aarch32.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2020 Andrey Semashev
0007  */
0008 /*!
0009  * \file   atomic/detail/extra_ops_gcc_aarch32.hpp
0010  *
0011  * This header contains implementation of the extra atomic operations for AArch32.
0012  */
0013
0014 #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_AARCH32_HPP_INCLUDED_
0015 #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_AARCH32_HPP_INCLUDED_
0016
0017 #include <cstddef>
0018 #include <boost/cstdint.hpp>
0019 #include <boost/memory_order.hpp>
0020 #include <boost/atomic/detail/config.hpp>
0021 #include <boost/atomic/detail/platform.hpp>
0022 #include <boost/atomic/detail/storage_traits.hpp>
0023 #include <boost/atomic/detail/extra_operations_fwd.hpp>
0024 #include <boost/atomic/detail/extra_ops_generic.hpp>
0025 #include <boost/atomic/detail/ops_gcc_aarch32_common.hpp>
0026 #include <boost/atomic/detail/capabilities.hpp>
0027 #include <boost/atomic/detail/header.hpp>
0028
0029 #ifdef BOOST_HAS_PRAGMA_ONCE
0030 #pragma once
0031 #endif
0032
0033 namespace boost {
0034 namespace atomics {
0035 namespace detail {
0036
0037 template< typename Base >
0038 struct extra_operations_gcc_aarch32_common :
0039     public Base
0040 {
0041     typedef Base base_type;
0042     typedef typename base_type::storage_type storage_type;
0043
0044     // Note: For opaque operations prefer operations returning the resulting values instead of the original values
0045     //       as these operations require less registers.
0046     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0047     {
0048         base_type::negate(storage, order);
0049     }
0050
0051     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0052     {
0053         base_type::bitwise_complement(storage, order);
0054     }
0055
0056     static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0057     {
0058         base_type::add(storage, v, order);
0059     }
0060
0061     static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0062     {
0063         base_type::sub(storage, v, order);
0064     }
0065
0066     static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0067     {
0068         base_type::bitwise_and(storage, v, order);
0069     }
0070
0071     static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0072     {
0073         base_type::bitwise_or(storage, v, order);
0074     }
0075
0076     static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0077     {
0078         base_type::bitwise_xor(storage, v, order);
0079     }
0080
0081     static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0082     {
0083         return !!base_type::negate(storage, order);
0084     }
0085
0086     static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0087     {
0088         return !!base_type::add(storage, v, order);
0089     }
0090
0091     static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0092     {
0093         return !!base_type::sub(storage, v, order);
0094     }
0095
0096     static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0097     {
0098         return !!base_type::bitwise_and(storage, v, order);
0099     }
0100
0101     static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0102     {
0103         return !!base_type::bitwise_or(storage, v, order);
0104     }
0105
0106     static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0107     {
0108         return !!base_type::bitwise_xor(storage, v, order);
0109     }
0110
0111     static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0112     {
0113         return !!base_type::bitwise_complement(storage, order);
0114     }
0115 };
0116
0117 template< typename Base, std::size_t Size, bool Signed >
0118 struct extra_operations_gcc_aarch32;
0119
0120 template< typename Base, bool Signed >
0121 struct extra_operations_gcc_aarch32< Base, 1u, Signed > :
0122     public extra_operations_generic< Base, 1u, Signed >
0123 {
0124     typedef extra_operations_generic< Base, 1u, Signed > base_type;
0125     typedef typename base_type::storage_type storage_type;
0126
0127     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0128     {
0129         storage_type original, result;
0130         uint32_t tmp;
0131 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0132         __asm__ __volatile__\
0133         (\
0134             "1:\n\t"\
0135             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0136             "rsb %[result], %[original], #0\n\t"\
0137             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0138             "teq %[tmp], #0\n\t"\
0139             "bne 1b\n\t"\
0140             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0141             : \
0142             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0143         );
0144
0145         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0146 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0147
0148         return original;
0149     }
0150
0151     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0152     {
0153         storage_type result;
0154         uint32_t tmp;
0155 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0156         __asm__ __volatile__\
0157         (\
0158             "1:\n\t"\
0159             "ld" ld_mo "exb %[result], %[storage]\n\t"\
0160             "rsb %[result], #0\n\t"\
0161             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0162             "teq %[tmp], #0\n\t"\
0163             "bne 1b\n\t"\
0164             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0165             : \
0166             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0167         );
0168
0169         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0170 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0171
0172         return result;
0173     }
0174
0175     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0176     {
0177         storage_type result;
0178         uint32_t tmp;
0179 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0180         __asm__ __volatile__\
0181         (\
0182             "1:\n\t"\
0183             "ld" ld_mo "exb %[result], %[storage]\n\t"\
0184             "add %[result], %[value]\n\t"\
0185             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0186             "teq %[tmp], #0\n\t"\
0187             "bne 1b\n\t"\
0188             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0189             : [value] "Ir" (v)\
0190             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0191         );
0192
0193         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0194 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0195
0196         return result;
0197     }
0198
0199     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0200     {
0201         storage_type result;
0202         uint32_t tmp;
0203 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0204         __asm__ __volatile__\
0205         (\
0206             "1:\n\t"\
0207             "ld" ld_mo "exb %[result], %[storage]\n\t"\
0208             "sub %[result], %[value]\n\t"\
0209             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0210             "teq %[tmp], #0\n\t"\
0211             "bne 1b\n\t"\
0212             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0213             : [value] "Ir" (v)\
0214             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0215         );
0216
0217         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0218 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0219
0220         return result;
0221     }
0222
0223     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0224     {
0225         storage_type result;
0226         uint32_t tmp;
0227 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0228         __asm__ __volatile__\
0229         (\
0230             "1:\n\t"\
0231             "ld" ld_mo "exb %[result], %[storage]\n\t"\
0232             "and %[result], %[value]\n\t"\
0233             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0234             "teq %[tmp], #0\n\t"\
0235             "bne 1b\n\t"\
0236             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0237             : [value] "Ir" (v)\
0238             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0239         );
0240
0241         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0242 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0243
0244         return result;
0245     }
0246
0247     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0248     {
0249         storage_type result;
0250         uint32_t tmp;
0251 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0252         __asm__ __volatile__\
0253         (\
0254             "1:\n\t"\
0255             "ld" ld_mo "exb %[result], %[storage]\n\t"\
0256             "orr %[result], %[value]\n\t"\
0257             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0258             "teq %[tmp], #0\n\t"\
0259             "bne 1b\n\t"\
0260             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0261             : [value] "Ir" (v)\
0262             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0263         );
0264
0265         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0266 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0267
0268         return result;
0269     }
0270
0271     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0272     {
0273         storage_type result;
0274         uint32_t tmp;
0275 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0276         __asm__ __volatile__\
0277         (\
0278             "1:\n\t"\
0279             "ld" ld_mo "exb %[result], %[storage]\n\t"\
0280             "eor %[result], %[value]\n\t"\
0281             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0282             "teq %[tmp], #0\n\t"\
0283             "bne 1b\n\t"\
0284             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0285             : [value] "Ir" (v)\
0286             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0287         );
0288
0289         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0290 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0291
0292         return result;
0293     }
0294
0295     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0296     {
0297         storage_type original, result;
0298         uint32_t tmp;
0299 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0300         __asm__ __volatile__\
0301         (\
0302             "1:\n\t"\
0303             "ld" ld_mo "exb %[original], %[storage]\n\t"\
0304             "mvn %[result], %[original]\n\t"\
0305             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0306             "teq %[tmp], #0\n\t"\
0307             "bne 1b\n\t"\
0308             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0309             : \
0310             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0311         );
0312
0313         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0314 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0315
0316         return original;
0317     }
0318
0319     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0320     {
0321         storage_type result;
0322         uint32_t tmp;
0323 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0324         __asm__ __volatile__\
0325         (\
0326             "1:\n\t"\
0327             "ld" ld_mo "exb %[result], %[storage]\n\t"\
0328             "mvn %[result], %[result]\n\t"\
0329             "st" st_mo "exb %[tmp], %[result], %[storage]\n\t"\
0330             "teq %[tmp], #0\n\t"\
0331             "bne 1b\n\t"\
0332             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0333             : \
0334             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0335         );
0336
0337         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0338 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0339
0340         return result;
0341     }
0342 };
0343
0344 template< typename Base, bool Signed >
0345 struct extra_operations< Base, 1u, Signed, true > :
0346     public extra_operations_gcc_aarch32_common< extra_operations_gcc_aarch32< Base, 1u, Signed > >
0347 {
0348 };
0349
0350
0351 template< typename Base, bool Signed >
0352 struct extra_operations_gcc_aarch32< Base, 2u, Signed > :
0353     public extra_operations_generic< Base, 2u, Signed >
0354 {
0355     typedef extra_operations_generic< Base, 2u, Signed > base_type;
0356     typedef typename base_type::storage_type storage_type;
0357
0358     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0359     {
0360         storage_type original, result;
0361         uint32_t tmp;
0362 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0363         __asm__ __volatile__\
0364         (\
0365             "1:\n\t"\
0366             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0367             "rsb %[result], %[original], #0\n\t"\
0368             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0369             "teq %[tmp], #0\n\t"\
0370             "bne 1b\n\t"\
0371             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0372             : \
0373             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0374         );
0375
0376         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0377 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0378
0379         return original;
0380     }
0381
0382     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0383     {
0384         storage_type result;
0385         uint32_t tmp;
0386 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0387         __asm__ __volatile__\
0388         (\
0389             "1:\n\t"\
0390             "ld" ld_mo "exh %[result], %[storage]\n\t"\
0391             "rsb %[result], #0\n\t"\
0392             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0393             "teq %[tmp], #0\n\t"\
0394             "bne 1b\n\t"\
0395             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0396             : \
0397             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0398         );
0399
0400         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0401 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0402
0403         return result;
0404     }
0405
0406     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0407     {
0408         storage_type result;
0409         uint32_t tmp;
0410 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0411         __asm__ __volatile__\
0412         (\
0413             "1:\n\t"\
0414             "ld" ld_mo "exh %[result], %[storage]\n\t"\
0415             "add %[result], %[value]\n\t"\
0416             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0417             "teq %[tmp], #0\n\t"\
0418             "bne 1b\n\t"\
0419             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0420             : [value] "Ir" (v)\
0421             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0422         );
0423
0424         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0425 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0426
0427         return result;
0428     }
0429
0430     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0431     {
0432         storage_type result;
0433         uint32_t tmp;
0434 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0435         __asm__ __volatile__\
0436         (\
0437             "1:\n\t"\
0438             "ld" ld_mo "exh %[result], %[storage]\n\t"\
0439             "sub %[result], %[value]\n\t"\
0440             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0441             "teq %[tmp], #0\n\t"\
0442             "bne 1b\n\t"\
0443             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0444             : [value] "Ir" (v)\
0445             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0446         );
0447
0448         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0449 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0450
0451         return result;
0452     }
0453
0454     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0455     {
0456         storage_type result;
0457         uint32_t tmp;
0458 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0459         __asm__ __volatile__\
0460         (\
0461             "1:\n\t"\
0462             "ld" ld_mo "exh %[result], %[storage]\n\t"\
0463             "and %[result], %[value]\n\t"\
0464             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0465             "teq %[tmp], #0\n\t"\
0466             "bne 1b\n\t"\
0467             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0468             : [value] "Ir" (v)\
0469             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0470         );
0471
0472         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0473 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0474
0475         return result;
0476     }
0477
0478     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0479     {
0480         storage_type result;
0481         uint32_t tmp;
0482 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0483         __asm__ __volatile__\
0484         (\
0485             "1:\n\t"\
0486             "ld" ld_mo "exh %[result], %[storage]\n\t"\
0487             "orr %[result], %[value]\n\t"\
0488             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0489             "teq %[tmp], #0\n\t"\
0490             "bne 1b\n\t"\
0491             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0492             : [value] "Ir" (v)\
0493             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0494         );
0495
0496         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0497 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0498
0499         return result;
0500     }
0501
0502     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0503     {
0504         storage_type result;
0505         uint32_t tmp;
0506 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0507         __asm__ __volatile__\
0508         (\
0509             "1:\n\t"\
0510             "ld" ld_mo "exh %[result], %[storage]\n\t"\
0511             "eor %[result], %[value]\n\t"\
0512             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0513             "teq %[tmp], #0\n\t"\
0514             "bne 1b\n\t"\
0515             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0516             : [value] "Ir" (v)\
0517             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0518         );
0519
0520         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0521 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0522
0523         return result;
0524     }
0525
0526     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0527     {
0528         storage_type original, result;
0529         uint32_t tmp;
0530 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0531         __asm__ __volatile__\
0532         (\
0533             "1:\n\t"\
0534             "ld" ld_mo "exh %[original], %[storage]\n\t"\
0535             "mvn %[result], %[original]\n\t"\
0536             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0537             "teq %[tmp], #0\n\t"\
0538             "bne 1b\n\t"\
0539             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0540             : \
0541             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0542         );
0543
0544         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0545 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0546
0547         return original;
0548     }
0549
0550     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0551     {
0552         storage_type result;
0553         uint32_t tmp;
0554 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0555         __asm__ __volatile__\
0556         (\
0557             "1:\n\t"\
0558             "ld" ld_mo "exh %[result], %[storage]\n\t"\
0559             "mvn %[result], %[result]\n\t"\
0560             "st" st_mo "exh %[tmp], %[result], %[storage]\n\t"\
0561             "teq %[tmp], #0\n\t"\
0562             "bne 1b\n\t"\
0563             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0564             : \
0565             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0566         );
0567
0568         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0569 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0570
0571         return result;
0572     }
0573 };
0574
0575 template< typename Base, bool Signed >
0576 struct extra_operations< Base, 2u, Signed, true > :
0577     public extra_operations_gcc_aarch32_common< extra_operations_gcc_aarch32< Base, 2u, Signed > >
0578 {
0579 };
0580
0581 template< typename Base, bool Signed >
0582 struct extra_operations_gcc_aarch32< Base, 4u, Signed > :
0583     public extra_operations_generic< Base, 4u, Signed >
0584 {
0585     typedef extra_operations_generic< Base, 4u, Signed > base_type;
0586     typedef typename base_type::storage_type storage_type;
0587
0588     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0589     {
0590         storage_type original, result;
0591         uint32_t tmp;
0592 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0593         __asm__ __volatile__\
0594         (\
0595             "1:\n\t"\
0596             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0597             "rsb %[result], %[original], #0\n\t"\
0598             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0599             "teq %[tmp], #0\n\t"\
0600             "bne 1b\n\t"\
0601             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0602             : \
0603             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0604         );
0605
0606         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0607 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0608
0609         return original;
0610     }
0611
0612     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0613     {
0614         storage_type result;
0615         uint32_t tmp;
0616 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0617         __asm__ __volatile__\
0618         (\
0619             "1:\n\t"\
0620             "ld" ld_mo "ex %[result], %[storage]\n\t"\
0621             "rsb %[result], #0\n\t"\
0622             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0623             "teq %[tmp], #0\n\t"\
0624             "bne 1b\n\t"\
0625             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0626             : \
0627             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0628         );
0629
0630         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0631 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0632
0633         return result;
0634     }
0635
0636     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0637     {
0638         storage_type result;
0639         uint32_t tmp;
0640 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0641         __asm__ __volatile__\
0642         (\
0643             "1:\n\t"\
0644             "ld" ld_mo "ex %[result], %[storage]\n\t"\
0645             "add %[result], %[value]\n\t"\
0646             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0647             "teq %[tmp], #0\n\t"\
0648             "bne 1b\n\t"\
0649             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0650             : [value] "Ir" (v)\
0651             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0652         );
0653
0654         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0655 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0656
0657         return result;
0658     }
0659
0660     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0661     {
0662         storage_type result;
0663         uint32_t tmp;
0664 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0665         __asm__ __volatile__\
0666         (\
0667             "1:\n\t"\
0668             "ld" ld_mo "ex %[result], %[storage]\n\t"\
0669             "sub %[result], %[value]\n\t"\
0670             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0671             "teq %[tmp], #0\n\t"\
0672             "bne 1b\n\t"\
0673             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0674             : [value] "Ir" (v)\
0675             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0676         );
0677
0678         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0679 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0680
0681         return result;
0682     }
0683
0684     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0685     {
0686         storage_type result;
0687         uint32_t tmp;
0688 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0689         __asm__ __volatile__\
0690         (\
0691             "1:\n\t"\
0692             "ld" ld_mo "ex %[result], %[storage]\n\t"\
0693             "and %[result], %[value]\n\t"\
0694             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0695             "teq %[tmp], #0\n\t"\
0696             "bne 1b\n\t"\
0697             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0698             : [value] "Ir" (v)\
0699             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0700         );
0701
0702         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0703 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0704
0705         return result;
0706     }
0707
0708     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0709     {
0710         storage_type result;
0711         uint32_t tmp;
0712 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0713         __asm__ __volatile__\
0714         (\
0715             "1:\n\t"\
0716             "ld" ld_mo "ex %[result], %[storage]\n\t"\
0717             "orr %[result], %[value]\n\t"\
0718             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0719             "teq %[tmp], #0\n\t"\
0720             "bne 1b\n\t"\
0721             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0722             : [value] "Ir" (v)\
0723             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0724         );
0725
0726         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0727 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0728
0729         return result;
0730     }
0731
0732     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0733     {
0734         storage_type result;
0735         uint32_t tmp;
0736 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0737         __asm__ __volatile__\
0738         (\
0739             "1:\n\t"\
0740             "ld" ld_mo "ex %[result], %[storage]\n\t"\
0741             "eor %[result], %[value]\n\t"\
0742             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0743             "teq %[tmp], #0\n\t"\
0744             "bne 1b\n\t"\
0745             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0746             : [value] "Ir" (v)\
0747             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0748         );
0749
0750         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0751 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0752
0753         return result;
0754     }
0755
0756     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0757     {
0758         storage_type original, result;
0759         uint32_t tmp;
0760 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0761         __asm__ __volatile__\
0762         (\
0763             "1:\n\t"\
0764             "ld" ld_mo "ex %[original], %[storage]\n\t"\
0765             "mvn %[result], %[original]\n\t"\
0766             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0767             "teq %[tmp], #0\n\t"\
0768             "bne 1b\n\t"\
0769             : [original] "=&r" (original), [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0770             : \
0771             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0772         );
0773
0774         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0775 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0776
0777         return original;
0778     }
0779
0780     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0781     {
0782         storage_type result;
0783         uint32_t tmp;
0784 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0785         __asm__ __volatile__\
0786         (\
0787             "1:\n\t"\
0788             "ld" ld_mo "ex %[result], %[storage]\n\t"\
0789             "mvn %[result], %[result]\n\t"\
0790             "st" st_mo "ex %[tmp], %[result], %[storage]\n\t"\
0791             "teq %[tmp], #0\n\t"\
0792             "bne 1b\n\t"\
0793             : [result] "=&r" (result), [tmp] "=&r" (tmp), [storage] "+Q" (storage)\
0794             : \
0795             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0796         );
0797
0798         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0799 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0800
0801         return result;
0802     }
0803 };
0804
0805 template< typename Base, bool Signed >
0806 struct extra_operations< Base, 4u, Signed, true > :
0807     public extra_operations_gcc_aarch32_common< extra_operations_gcc_aarch32< Base, 4u, Signed > >
0808 {
0809 };
0810
0811 template< typename Base, bool Signed >
0812 struct extra_operations_gcc_aarch32< Base, 8u, Signed > :
0813     public extra_operations_generic< Base, 8u, Signed >
0814 {
0815     typedef extra_operations_generic< Base, 8u, Signed > base_type;
0816     typedef typename base_type::storage_type storage_type;
0817
0818     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0819     {
0820         storage_type original, result;
0821         uint32_t tmp;
0822 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0823         __asm__ __volatile__\
0824         (\
0825             "1:\n\t"\
0826             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0827             "mvn %3, %0\n\t"\
0828             "mvn %H3, %H0\n\t"\
0829             "adds " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(3) ", #1\n\t"\
0830             "adc " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(3) ", #0\n\t"\
0831             "st" st_mo "exd %1, %3, %H3, %2\n\t"\
0832             "teq %1, #0\n\t"\
0833             "bne 1b\n\t"\
0834             : "=&r" (original), "=&r" (tmp), "+Q" (storage), "=&r" (result)\
0835             : \
0836             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0837         );
0838
0839         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0840 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0841
0842         return original;
0843     }
0844
0845     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0846     {
0847         storage_type result;
0848         uint32_t tmp;
0849 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0850         __asm__ __volatile__\
0851         (\
0852             "1:\n\t"\
0853             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0854             "mvn %0, %0\n\t"\
0855             "mvn %H0, %H0\n\t"\
0856             "adds " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(0) ", #1\n\t"\
0857             "adc " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(0) ", #0\n\t"\
0858             "st" st_mo "exd %1, %0, %H0, %2\n\t"\
0859             "teq %1, #0\n\t"\
0860             "bne 1b\n\t"\
0861             : "=&r" (result), "=&r" (tmp), "+Q" (storage)\
0862             : \
0863             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0864         );
0865
0866         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0867 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0868
0869         return result;
0870     }
0871
0872     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0873     {
0874         storage_type result;
0875         uint32_t tmp;
0876 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0877         __asm__ __volatile__\
0878         (\
0879             "1:\n\t"\
0880             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0881             "adds " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(3) "\n\t"\
0882             "adc " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(3) "\n\t"\
0883             "st" st_mo "exd %1, %0, %H0, %2\n\t"\
0884             "teq %1, #0\n\t"\
0885             "bne 1b\n\t"\
0886             : "=&r" (result), "=&r" (tmp), "+Q" (storage)\
0887             : "r" (v)\
0888             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0889         );
0890
0891         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0892 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0893
0894         return result;
0895     }
0896
0897     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0898     {
0899         storage_type result;
0900         uint32_t tmp;
0901 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0902         __asm__ __volatile__\
0903         (\
0904             "1:\n\t"\
0905             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0906             "subs " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_LO(3) "\n\t"\
0907             "sbc " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(0) ", " BOOST_ATOMIC_DETAIL_AARCH32_ASM_ARG_HI(3) "\n\t"\
0908             "st" st_mo "exd %1, %0, %H0, %2\n\t"\
0909             "teq %1, #0\n\t"\
0910             "bne 1b\n\t"\
0911             : "=&r" (result), "=&r" (tmp), "+Q" (storage)\
0912             : "r" (v)\
0913             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0914         );
0915
0916         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0917 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0918
0919         return result;
0920     }
0921
0922     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0923     {
0924         storage_type result;
0925         uint32_t tmp;
0926 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0927         __asm__ __volatile__\
0928         (\
0929             "1:\n\t"\
0930             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0931             "and %0, %3\n\t"\
0932             "and %H0, %H3\n\t"\
0933             "st" st_mo "exd %1, %0, %H0, %2\n\t"\
0934             "teq %1, #0\n\t"\
0935             "bne 1b\n\t"\
0936             : "=&r" (result), "=&r" (tmp), "+Q" (storage)\
0937             : "r" (v)\
0938             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0939         );
0940
0941         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0942 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0943
0944         return result;
0945     }
0946
0947     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0948     {
0949         storage_type result;
0950         uint32_t tmp;
0951 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0952         __asm__ __volatile__\
0953         (\
0954             "1:\n\t"\
0955             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0956             "orr %0, %3\n\t"\
0957             "orr %H0, %H3\n\t"\
0958             "st" st_mo "exd %1, %0, %H0, %2\n\t"\
0959             "teq %1, #0\n\t"\
0960             "bne 1b\n\t"\
0961             : "=&r" (result), "=&r" (tmp), "+Q" (storage)\
0962             : "r" (v)\
0963             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0964         );
0965
0966         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0967 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0968
0969         return result;
0970     }
0971
0972     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0973     {
0974         storage_type result;
0975         uint32_t tmp;
0976 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
0977         __asm__ __volatile__\
0978         (\
0979             "1:\n\t"\
0980             "ld" ld_mo "exd %0, %H0, %2\n\t"\
0981             "eor %0, %3\n\t"\
0982             "eor %H0, %H3\n\t"\
0983             "st" st_mo "exd %1, %0, %H0, %2\n\t"\
0984             "teq %1, #0\n\t"\
0985             "bne 1b\n\t"\
0986             : "=&r" (result), "=&r" (tmp), "+Q" (storage)\
0987             : "r" (v)\
0988             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0989         );
0990
0991         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
0992 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
0993
0994         return result;
0995     }
0996
0997     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0998     {
0999         storage_type original, result;
1000         uint32_t tmp;
1001 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
1002         __asm__ __volatile__\
1003         (\
1004             "1:\n\t"\
1005             "ld" ld_mo "exd %0, %H0, %2\n\t"\
1006             "mvn %3, %0\n\t"\
1007             "mvn %H3, %H0\n\t"\
1008             "st" st_mo "exd %1, %3, %H3, %2\n\t"\
1009             "teq %1, #0\n\t"\
1010             "bne 1b\n\t"\
1011             : "=&r" (original), "=&r" (tmp), "+Q" (storage), "=&r" (result)\
1012             : \
1013             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1014         );
1015
1016         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
1017 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
1018
1019         return original;
1020     }
1021
1022     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1023     {
1024         storage_type result;
1025         uint32_t tmp;
1026 #define BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN(ld_mo, st_mo)\
1027         __asm__ __volatile__\
1028         (\
1029             "1:\n\t"\
1030             "ld" ld_mo "exd %0, %H0, %2\n\t"\
1031             "mvn %0, %0\n\t"\
1032             "mvn %H0, %H0\n\t"\
1033             "st" st_mo "exd %1, %0, %H0, %2\n\t"\
1034             "teq %1, #0\n\t"\
1035             "bne 1b\n\t"\
1036             : "=&r" (result), "=&r" (tmp), "+Q" (storage)\
1037             : \
1038             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1039         );
1040
1041         BOOST_ATOMIC_DETAIL_AARCH32_MO_SWITCH(order)
1042 #undef BOOST_ATOMIC_DETAIL_AARCH32_MO_INSN
1043
1044         return result;
1045     }
1046 };
1047
1048 template< typename Base, bool Signed >
1049 struct extra_operations< Base, 8u, Signed, true > :
1050     public extra_operations_gcc_aarch32_common< extra_operations_gcc_aarch32< Base, 8u, Signed > >
1051 {
1052 };
1053
1054 } // namespace detail
1055 } // namespace atomics
1056 } // namespace boost
1057
1058 #include <boost/atomic/detail/footer.hpp>
1059
1060 #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_AARCH32_HPP_INCLUDED_