atomic/detail/extra_ops_gcc_arm.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2017 - 2018 Andrey Semashev
0007  */
0008 /*!
0009  * \file   atomic/detail/extra_ops_gcc_arm.hpp
0010  *
0011  * This header contains implementation of the extra atomic operations for ARM.
0012  */
0013
0014 #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_ARM_HPP_INCLUDED_
0015 #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_ARM_HPP_INCLUDED_
0016
0017 #include <cstddef>
0018 #include <boost/cstdint.hpp>
0019 #include <boost/memory_order.hpp>
0020 #include <boost/atomic/detail/config.hpp>
0021 #include <boost/atomic/detail/platform.hpp>
0022 #include <boost/atomic/detail/storage_traits.hpp>
0023 #include <boost/atomic/detail/extra_operations_fwd.hpp>
0024 #include <boost/atomic/detail/extra_ops_generic.hpp>
0025 #include <boost/atomic/detail/ops_gcc_arm_common.hpp>
0026 #include <boost/atomic/detail/gcc_arm_asm_common.hpp>
0027 #include <boost/atomic/detail/capabilities.hpp>
0028 #include <boost/atomic/detail/header.hpp>
0029
0030 #ifdef BOOST_HAS_PRAGMA_ONCE
0031 #pragma once
0032 #endif
0033
0034 namespace boost {
0035 namespace atomics {
0036 namespace detail {
0037
0038 template< typename Base >
0039 struct extra_operations_gcc_arm_common :
0040     public Base
0041 {
0042     typedef Base base_type;
0043     typedef typename base_type::storage_type storage_type;
0044
0045     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0046     {
0047         base_type::fetch_negate(storage, order);
0048     }
0049
0050     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0051     {
0052         base_type::fetch_complement(storage, order);
0053     }
0054
0055     static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0056     {
0057         return !!base_type::negate(storage, order);
0058     }
0059
0060     static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0061     {
0062         return !!base_type::add(storage, v, order);
0063     }
0064
0065     static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0066     {
0067         return !!base_type::sub(storage, v, order);
0068     }
0069
0070     static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0071     {
0072         return !!base_type::bitwise_and(storage, v, order);
0073     }
0074
0075     static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0076     {
0077         return !!base_type::bitwise_or(storage, v, order);
0078     }
0079
0080     static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0081     {
0082         return !!base_type::bitwise_xor(storage, v, order);
0083     }
0084
0085     static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0086     {
0087         return !!base_type::bitwise_complement(storage, order);
0088     }
0089 };
0090
0091 template< typename Base, std::size_t Size, bool Signed >
0092 struct extra_operations_gcc_arm;
0093
0094 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
0095
0096 template< typename Base, bool Signed >
0097 struct extra_operations_gcc_arm< Base, 1u, Signed > :
0098     public extra_operations_generic< Base, 1u, Signed >
0099 {
0100     typedef extra_operations_generic< Base, 1u, Signed > base_type;
0101     typedef typename base_type::storage_type storage_type;
0102     typedef typename storage_traits< 4u >::type extended_storage_type;
0103
0104     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0105     {
0106         core_arch_operations_gcc_arm_base::fence_before(order);
0107         uint32_t tmp;
0108         extended_storage_type original, result;
0109         __asm__ __volatile__
0110         (
0111             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0112             "1:\n\t"
0113             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0114             "rsb      %[result], %[original], #0\n\t"        // result = 0 - original
0115             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0116             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0117             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0118             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0119             : [original] "=&r" (original),  // %0
0120               [result] "=&r" (result),      // %1
0121               [tmp] "=&l" (tmp),            // %2
0122               [storage] "+Q" (storage)      // %3
0123             :
0124             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0125         );
0126         core_arch_operations_gcc_arm_base::fence_after(order);
0127         return static_cast< storage_type >(original);
0128     }
0129
0130     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0131     {
0132         core_arch_operations_gcc_arm_base::fence_before(order);
0133         uint32_t tmp;
0134         extended_storage_type original, result;
0135         __asm__ __volatile__
0136         (
0137             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0138             "1:\n\t"
0139             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0140             "rsb      %[result], %[original], #0\n\t"        // result = 0 - original
0141             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0142             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0143             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0144             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0145             : [original] "=&r" (original),  // %0
0146               [result] "=&r" (result),      // %1
0147               [tmp] "=&l" (tmp),            // %2
0148               [storage] "+Q" (storage)      // %3
0149             :
0150             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0151         );
0152         core_arch_operations_gcc_arm_base::fence_after(order);
0153         return static_cast< storage_type >(result);
0154     }
0155
0156     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0157     {
0158         core_arch_operations_gcc_arm_base::fence_before(order);
0159         uint32_t tmp;
0160         extended_storage_type original, result;
0161         __asm__ __volatile__
0162         (
0163             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0164             "1:\n\t"
0165             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0166             "add      %[result], %[original], %[value]\n\t"  // result = original + value
0167             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0168             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0169             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0170             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0171             : [original] "=&r" (original),  // %0
0172               [result] "=&r" (result),      // %1
0173               [tmp] "=&l" (tmp),            // %2
0174               [storage] "+Q" (storage)      // %3
0175             : [value] "Ir" (v)              // %4
0176             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0177         );
0178         core_arch_operations_gcc_arm_base::fence_after(order);
0179         return static_cast< storage_type >(result);
0180     }
0181
0182     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0183     {
0184         core_arch_operations_gcc_arm_base::fence_before(order);
0185         uint32_t tmp;
0186         extended_storage_type original, result;
0187         __asm__ __volatile__
0188         (
0189             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0190             "1:\n\t"
0191             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0192             "sub      %[result], %[original], %[value]\n\t"  // result = original - value
0193             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0194             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0195             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0196             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0197             : [original] "=&r" (original),  // %0
0198               [result] "=&r" (result),      // %1
0199               [tmp] "=&l" (tmp),            // %2
0200               [storage] "+Q" (storage)      // %3
0201             : [value] "Ir" (v)              // %4
0202             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0203         );
0204         core_arch_operations_gcc_arm_base::fence_after(order);
0205         return static_cast< storage_type >(result);
0206     }
0207
0208     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0209     {
0210         core_arch_operations_gcc_arm_base::fence_before(order);
0211         uint32_t tmp;
0212         extended_storage_type original, result;
0213         __asm__ __volatile__
0214         (
0215             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0216             "1:\n\t"
0217             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0218             "and      %[result], %[original], %[value]\n\t"  // result = original & value
0219             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0220             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0221             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0222             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0223             : [original] "=&r" (original),  // %0
0224               [result] "=&r" (result),      // %1
0225               [tmp] "=&l" (tmp),            // %2
0226               [storage] "+Q" (storage)      // %3
0227             : [value] "Ir" (v)              // %4
0228             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0229         );
0230         core_arch_operations_gcc_arm_base::fence_after(order);
0231         return static_cast< storage_type >(result);
0232     }
0233
0234     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0235     {
0236         core_arch_operations_gcc_arm_base::fence_before(order);
0237         uint32_t tmp;
0238         extended_storage_type original, result;
0239         __asm__ __volatile__
0240         (
0241             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0242             "1:\n\t"
0243             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0244             "orr      %[result], %[original], %[value]\n\t"  // result = original | value
0245             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0246             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0247             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0248             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0249             : [original] "=&r" (original),  // %0
0250               [result] "=&r" (result),      // %1
0251               [tmp] "=&l" (tmp),            // %2
0252               [storage] "+Q" (storage)      // %3
0253             : [value] "Ir" (v)              // %4
0254             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0255         );
0256         core_arch_operations_gcc_arm_base::fence_after(order);
0257         return static_cast< storage_type >(result);
0258     }
0259
0260     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0261     {
0262         core_arch_operations_gcc_arm_base::fence_before(order);
0263         uint32_t tmp;
0264         extended_storage_type original, result;
0265         __asm__ __volatile__
0266         (
0267             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0268             "1:\n\t"
0269             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0270             "eor      %[result], %[original], %[value]\n\t"  // result = original ^ value
0271             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0272             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0273             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0274             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0275             : [original] "=&r" (original),  // %0
0276               [result] "=&r" (result),      // %1
0277               [tmp] "=&l" (tmp),            // %2
0278               [storage] "+Q" (storage)      // %3
0279             : [value] "Ir" (v)              // %4
0280             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0281         );
0282         core_arch_operations_gcc_arm_base::fence_after(order);
0283         return static_cast< storage_type >(result);
0284     }
0285
0286     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0287     {
0288         core_arch_operations_gcc_arm_base::fence_before(order);
0289         uint32_t tmp;
0290         extended_storage_type original, result;
0291         __asm__ __volatile__
0292         (
0293             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0294             "1:\n\t"
0295             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0296             "mvn      %[result], %[original]\n\t"            // result = NOT original
0297             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0298             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0299             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0300             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0301             : [original] "=&r" (original),  // %0
0302               [result] "=&r" (result),      // %1
0303               [tmp] "=&l" (tmp),            // %2
0304               [storage] "+Q" (storage)      // %3
0305             :
0306             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0307         );
0308         core_arch_operations_gcc_arm_base::fence_after(order);
0309         return static_cast< storage_type >(original);
0310     }
0311
0312     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0313     {
0314         core_arch_operations_gcc_arm_base::fence_before(order);
0315         uint32_t tmp;
0316         extended_storage_type original, result;
0317         __asm__ __volatile__
0318         (
0319             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0320             "1:\n\t"
0321             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0322             "mvn      %[result], %[original]\n\t"            // result = NOT original
0323             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0324             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0325             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0326             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0327             : [original] "=&r" (original),  // %0
0328               [result] "=&r" (result),      // %1
0329               [tmp] "=&l" (tmp),            // %2
0330               [storage] "+Q" (storage)      // %3
0331             :
0332             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0333         );
0334         core_arch_operations_gcc_arm_base::fence_after(order);
0335         return static_cast< storage_type >(result);
0336     }
0337 };
0338
0339 template< typename Base, bool Signed >
0340 struct extra_operations< Base, 1u, Signed, true > :
0341     public extra_operations_gcc_arm_common< extra_operations_gcc_arm< Base, 1u, Signed > >
0342 {
0343 };
0344
0345 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
0346
0347 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
0348
0349 template< typename Base, bool Signed >
0350 struct extra_operations_gcc_arm< Base, 2u, Signed > :
0351     public extra_operations_generic< Base, 2u, Signed >
0352 {
0353     typedef extra_operations_generic< Base, 2u, Signed > base_type;
0354     typedef typename base_type::storage_type storage_type;
0355     typedef typename storage_traits< 4u >::type extended_storage_type;
0356
0357     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0358     {
0359         core_arch_operations_gcc_arm_base::fence_before(order);
0360         uint32_t tmp;
0361         extended_storage_type original, result;
0362         __asm__ __volatile__
0363         (
0364             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0365             "1:\n\t"
0366             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0367             "rsb      %[result], %[original], #0\n\t"        // result = 0 - original
0368             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0369             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0370             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0371             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0372             : [original] "=&r" (original),  // %0
0373               [result] "=&r" (result),      // %1
0374               [tmp] "=&l" (tmp),            // %2
0375               [storage] "+Q" (storage)      // %3
0376             :
0377             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0378         );
0379         core_arch_operations_gcc_arm_base::fence_after(order);
0380         return static_cast< storage_type >(original);
0381     }
0382
0383     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0384     {
0385         core_arch_operations_gcc_arm_base::fence_before(order);
0386         uint32_t tmp;
0387         extended_storage_type original, result;
0388         __asm__ __volatile__
0389         (
0390             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0391             "1:\n\t"
0392             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0393             "rsb      %[result], %[original], #0\n\t"        // result = 0 - original
0394             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0395             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0396             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0397             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0398             : [original] "=&r" (original),  // %0
0399               [result] "=&r" (result),      // %1
0400               [tmp] "=&l" (tmp),            // %2
0401               [storage] "+Q" (storage)      // %3
0402             :
0403             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0404         );
0405         core_arch_operations_gcc_arm_base::fence_after(order);
0406         return static_cast< storage_type >(result);
0407     }
0408
0409     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0410     {
0411         core_arch_operations_gcc_arm_base::fence_before(order);
0412         uint32_t tmp;
0413         extended_storage_type original, result;
0414         __asm__ __volatile__
0415         (
0416             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0417             "1:\n\t"
0418             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0419             "add      %[result], %[original], %[value]\n\t"  // result = original + value
0420             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0421             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0422             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0423             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0424             : [original] "=&r" (original),  // %0
0425               [result] "=&r" (result),      // %1
0426               [tmp] "=&l" (tmp),            // %2
0427               [storage] "+Q" (storage)      // %3
0428             : [value] "Ir" (v)              // %4
0429             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0430         );
0431         core_arch_operations_gcc_arm_base::fence_after(order);
0432         return static_cast< storage_type >(result);
0433     }
0434
0435     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0436     {
0437         core_arch_operations_gcc_arm_base::fence_before(order);
0438         uint32_t tmp;
0439         extended_storage_type original, result;
0440         __asm__ __volatile__
0441         (
0442             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0443             "1:\n\t"
0444             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0445             "sub      %[result], %[original], %[value]\n\t"  // result = original - value
0446             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0447             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0448             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0449             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0450             : [original] "=&r" (original),  // %0
0451               [result] "=&r" (result),      // %1
0452               [tmp] "=&l" (tmp),            // %2
0453               [storage] "+Q" (storage)      // %3
0454             : [value] "Ir" (v)              // %4
0455             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0456         );
0457         core_arch_operations_gcc_arm_base::fence_after(order);
0458         return static_cast< storage_type >(result);
0459     }
0460
0461     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0462     {
0463         core_arch_operations_gcc_arm_base::fence_before(order);
0464         uint32_t tmp;
0465         extended_storage_type original, result;
0466         __asm__ __volatile__
0467         (
0468             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0469             "1:\n\t"
0470             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0471             "and      %[result], %[original], %[value]\n\t"  // result = original & value
0472             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0473             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0474             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0475             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0476             : [original] "=&r" (original),  // %0
0477               [result] "=&r" (result),      // %1
0478               [tmp] "=&l" (tmp),            // %2
0479               [storage] "+Q" (storage)      // %3
0480             : [value] "Ir" (v)              // %4
0481             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0482         );
0483         core_arch_operations_gcc_arm_base::fence_after(order);
0484         return static_cast< storage_type >(result);
0485     }
0486
0487     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0488     {
0489         core_arch_operations_gcc_arm_base::fence_before(order);
0490         uint32_t tmp;
0491         extended_storage_type original, result;
0492         __asm__ __volatile__
0493         (
0494             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0495             "1:\n\t"
0496             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0497             "orr      %[result], %[original], %[value]\n\t"  // result = original | value
0498             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0499             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0500             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0501             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0502             : [original] "=&r" (original),  // %0
0503               [result] "=&r" (result),      // %1
0504               [tmp] "=&l" (tmp),            // %2
0505               [storage] "+Q" (storage)      // %3
0506             : [value] "Ir" (v)              // %4
0507             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0508         );
0509         core_arch_operations_gcc_arm_base::fence_after(order);
0510         return static_cast< storage_type >(result);
0511     }
0512
0513     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0514     {
0515         core_arch_operations_gcc_arm_base::fence_before(order);
0516         uint32_t tmp;
0517         extended_storage_type original, result;
0518         __asm__ __volatile__
0519         (
0520             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0521             "1:\n\t"
0522             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0523             "eor      %[result], %[original], %[value]\n\t"  // result = original ^ value
0524             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0525             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0526             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0527             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0528             : [original] "=&r" (original),  // %0
0529               [result] "=&r" (result),      // %1
0530               [tmp] "=&l" (tmp),            // %2
0531               [storage] "+Q" (storage)      // %3
0532             : [value] "Ir" (v)              // %4
0533             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0534         );
0535         core_arch_operations_gcc_arm_base::fence_after(order);
0536         return static_cast< storage_type >(result);
0537     }
0538
0539     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0540     {
0541         core_arch_operations_gcc_arm_base::fence_before(order);
0542         uint32_t tmp;
0543         extended_storage_type original, result;
0544         __asm__ __volatile__
0545         (
0546             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0547             "1:\n\t"
0548             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0549             "mvn      %[result], %[original]\n\t"            // result = NOT original
0550             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0551             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0552             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0553             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0554             : [original] "=&r" (original),  // %0
0555               [result] "=&r" (result),      // %1
0556               [tmp] "=&l" (tmp),            // %2
0557               [storage] "+Q" (storage)      // %3
0558             :
0559             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0560         );
0561         core_arch_operations_gcc_arm_base::fence_after(order);
0562         return static_cast< storage_type >(original);
0563     }
0564
0565     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0566     {
0567         core_arch_operations_gcc_arm_base::fence_before(order);
0568         uint32_t tmp;
0569         extended_storage_type original, result;
0570         __asm__ __volatile__
0571         (
0572             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0573             "1:\n\t"
0574             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0575             "mvn      %[result], %[original]\n\t"            // result = NOT original
0576             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0577             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0578             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0579             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0580             : [original] "=&r" (original),  // %0
0581               [result] "=&r" (result),      // %1
0582               [tmp] "=&l" (tmp),            // %2
0583               [storage] "+Q" (storage)      // %3
0584             :
0585             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0586         );
0587         core_arch_operations_gcc_arm_base::fence_after(order);
0588         return static_cast< storage_type >(result);
0589     }
0590 };
0591
0592 template< typename Base, bool Signed >
0593 struct extra_operations< Base, 2u, Signed, true > :
0594     public extra_operations_gcc_arm_common< extra_operations_gcc_arm< Base, 2u, Signed > >
0595 {
0596 };
0597
0598 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
0599
0600 template< typename Base, bool Signed >
0601 struct extra_operations_gcc_arm< Base, 4u, Signed > :
0602     public extra_operations_generic< Base, 4u, Signed >
0603 {
0604     typedef extra_operations_generic< Base, 4u, Signed > base_type;
0605     typedef typename base_type::storage_type storage_type;
0606
0607     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0608     {
0609         core_arch_operations_gcc_arm_base::fence_before(order);
0610         uint32_t tmp;
0611         storage_type original, result;
0612         __asm__ __volatile__
0613         (
0614             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0615             "1:\n\t"
0616             "ldrex    %[original], %[storage]\n\t"           // original = *(&storage)
0617             "rsb      %[result], %[original], #0\n\t"        // result = 0 - original
0618             "strex    %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0619             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0620             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0621             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0622             : [original] "=&r" (original),  // %0
0623               [result] "=&r" (result),      // %1
0624               [tmp] "=&l" (tmp),            // %2
0625               [storage] "+Q" (storage)      // %3
0626             :
0627             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0628         );
0629         core_arch_operations_gcc_arm_base::fence_after(order);
0630         return original;
0631     }
0632
0633     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0634     {
0635         core_arch_operations_gcc_arm_base::fence_before(order);
0636         uint32_t tmp;
0637         storage_type original, result;
0638         __asm__ __volatile__
0639         (
0640             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0641             "1:\n\t"
0642             "ldrex    %[original], %[storage]\n\t"           // original = *(&storage)
0643             "rsb      %[result], %[original], #0\n\t"        // result = 0 - original
0644             "strex    %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0645             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0646             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0647             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0648             : [original] "=&r" (original),  // %0
0649               [result] "=&r" (result),      // %1
0650               [tmp] "=&l" (tmp),            // %2
0651               [storage] "+Q" (storage)      // %3
0652             :
0653             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0654         );
0655         core_arch_operations_gcc_arm_base::fence_after(order);
0656         return result;
0657     }
0658
0659     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0660     {
0661         core_arch_operations_gcc_arm_base::fence_before(order);
0662         uint32_t tmp;
0663         storage_type original, result;
0664         __asm__ __volatile__
0665         (
0666             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0667             "1:\n\t"
0668             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0669             "add     %[result], %[original], %[value]\n\t"  // result = original + value
0670             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0671             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0672             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0673             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0674             : [original] "=&r" (original),  // %0
0675               [result] "=&r" (result),      // %1
0676               [tmp] "=&l" (tmp),            // %2
0677               [storage] "+Q" (storage)      // %3
0678             : [value] "Ir" (v)              // %4
0679             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0680         );
0681         core_arch_operations_gcc_arm_base::fence_after(order);
0682         return result;
0683     }
0684
0685     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0686     {
0687         core_arch_operations_gcc_arm_base::fence_before(order);
0688         uint32_t tmp;
0689         storage_type original, result;
0690         __asm__ __volatile__
0691         (
0692             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0693             "1:\n\t"
0694             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0695             "sub     %[result], %[original], %[value]\n\t"  // result = original - value
0696             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0697             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0698             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0699             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0700             : [original] "=&r" (original),  // %0
0701               [result] "=&r" (result),      // %1
0702               [tmp] "=&l" (tmp),            // %2
0703               [storage] "+Q" (storage)      // %3
0704             : [value] "Ir" (v)              // %4
0705             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0706         );
0707         core_arch_operations_gcc_arm_base::fence_after(order);
0708         return result;
0709     }
0710
0711     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0712     {
0713         core_arch_operations_gcc_arm_base::fence_before(order);
0714         uint32_t tmp;
0715         storage_type original, result;
0716         __asm__ __volatile__
0717         (
0718             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0719             "1:\n\t"
0720             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0721             "and     %[result], %[original], %[value]\n\t"  // result = original & value
0722             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0723             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0724             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0725             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0726             : [original] "=&r" (original),  // %0
0727               [result] "=&r" (result),      // %1
0728               [tmp] "=&l" (tmp),            // %2
0729               [storage] "+Q" (storage)      // %3
0730             : [value] "Ir" (v)              // %4
0731             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0732         );
0733         core_arch_operations_gcc_arm_base::fence_after(order);
0734         return result;
0735     }
0736
0737     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0738     {
0739         core_arch_operations_gcc_arm_base::fence_before(order);
0740         uint32_t tmp;
0741         storage_type original, result;
0742         __asm__ __volatile__
0743         (
0744             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0745             "1:\n\t"
0746             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0747             "orr     %[result], %[original], %[value]\n\t"  // result = original | value
0748             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0749             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0750             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0751             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0752             : [original] "=&r" (original),  // %0
0753               [result] "=&r" (result),      // %1
0754               [tmp] "=&l" (tmp),            // %2
0755               [storage] "+Q" (storage)      // %3
0756             : [value] "Ir" (v)              // %4
0757             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0758         );
0759         core_arch_operations_gcc_arm_base::fence_after(order);
0760         return result;
0761     }
0762
0763     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0764     {
0765         core_arch_operations_gcc_arm_base::fence_before(order);
0766         uint32_t tmp;
0767         storage_type original, result;
0768         __asm__ __volatile__
0769         (
0770             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0771             "1:\n\t"
0772             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0773             "eor     %[result], %[original], %[value]\n\t"  // result = original ^ value
0774             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0775             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0776             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0777             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0778             : [original] "=&r" (original),  // %0
0779               [result] "=&r" (result),      // %1
0780               [tmp] "=&l" (tmp),            // %2
0781               [storage] "+Q" (storage)      // %3
0782             : [value] "Ir" (v)              // %4
0783             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0784         );
0785         core_arch_operations_gcc_arm_base::fence_after(order);
0786         return result;
0787     }
0788
0789     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0790     {
0791         core_arch_operations_gcc_arm_base::fence_before(order);
0792         uint32_t tmp;
0793         storage_type original, result;
0794         __asm__ __volatile__
0795         (
0796             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0797             "1:\n\t"
0798             "ldrex    %[original], %[storage]\n\t"           // original = *(&storage)
0799             "mvn      %[result], %[original]\n\t"            // result = NOT original
0800             "strex    %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0801             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0802             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0803             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0804             : [original] "=&r" (original),  // %0
0805               [result] "=&r" (result),      // %1
0806               [tmp] "=&l" (tmp),            // %2
0807               [storage] "+Q" (storage)      // %3
0808             :
0809             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0810         );
0811         core_arch_operations_gcc_arm_base::fence_after(order);
0812         return original;
0813     }
0814
0815     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0816     {
0817         core_arch_operations_gcc_arm_base::fence_before(order);
0818         uint32_t tmp;
0819         storage_type original, result;
0820         __asm__ __volatile__
0821         (
0822             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0823             "1:\n\t"
0824             "ldrex    %[original], %[storage]\n\t"           // original = *(&storage)
0825             "mvn      %[result], %[original]\n\t"            // result = NOT original
0826             "strex    %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0827             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0828             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0829             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0830             : [original] "=&r" (original),  // %0
0831               [result] "=&r" (result),      // %1
0832               [tmp] "=&l" (tmp),            // %2
0833               [storage] "+Q" (storage)      // %3
0834             :
0835             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0836         );
0837         core_arch_operations_gcc_arm_base::fence_after(order);
0838         return result;
0839     }
0840 };
0841
0842 template< typename Base, bool Signed >
0843 struct extra_operations< Base, 4u, Signed, true > :
0844     public extra_operations_gcc_arm_common< extra_operations_gcc_arm< Base, 4u, Signed > >
0845 {
0846 };
0847
0848 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
0849
0850 template< typename Base, bool Signed >
0851 struct extra_operations_gcc_arm< Base, 8u, Signed > :
0852     public extra_operations_generic< Base, 8u, Signed >
0853 {
0854     typedef extra_operations_generic< Base, 8u, Signed > base_type;
0855     typedef typename base_type::storage_type storage_type;
0856
0857     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0858     {
0859         core_arch_operations_gcc_arm_base::fence_before(order);
0860         storage_type original, result;
0861         uint32_t tmp;
0862         __asm__ __volatile__
0863         (
0864             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
0865             "1:\n\t"
0866             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
0867             "mvn     %2, %1\n\t"                      // result = NOT original
0868             "mvn     %H2, %H1\n\t"
0869             "adds   " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", #1\n\t" // result = result + 1
0870             "adc    " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", #0\n\t"
0871             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
0872             "teq     %0, #0\n\t"                      // flags = tmp==0
0873             "bne     1b\n\t"                          // if (!flags.equal) goto retry
0874             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
0875             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
0876               "=&r" (original),  // %1
0877               "=&r" (result),    // %2
0878               "+Q" (storage)     // %3
0879             :
0880             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0881         );
0882         core_arch_operations_gcc_arm_base::fence_after(order);
0883         return original;
0884     }
0885
0886     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0887     {
0888         core_arch_operations_gcc_arm_base::fence_before(order);
0889         storage_type original, result;
0890         uint32_t tmp;
0891         __asm__ __volatile__
0892         (
0893             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
0894             "1:\n\t"
0895             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
0896             "mvn     %2, %1\n\t"                      // result = NOT original
0897             "mvn     %H2, %H1\n\t"
0898             "adds   " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", #1\n\t" // result = result + 1
0899             "adc    " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", #0\n\t"
0900             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
0901             "teq     %0, #0\n\t"                      // flags = tmp==0
0902             "bne     1b\n\t"                          // if (!flags.equal) goto retry
0903             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
0904             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
0905               "=&r" (original),  // %1
0906               "=&r" (result),    // %2
0907               "+Q" (storage)     // %3
0908             :
0909             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0910         );
0911         core_arch_operations_gcc_arm_base::fence_after(order);
0912         return result;
0913     }
0914
0915     static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0916     {
0917         core_arch_operations_gcc_arm_base::fence_before(order);
0918         storage_type original, result;
0919         uint32_t tmp;
0920         __asm__ __volatile__
0921         (
0922             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
0923             "1:\n\t"
0924             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
0925             "adds   " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(4) "\n\t" // result = original + value
0926             "adc    " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(4) "\n\t"
0927             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
0928             "teq     %0, #0\n\t"                      // flags = tmp==0
0929             "bne     1b\n\t"                          // if (!flags.equal) goto retry
0930             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
0931             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
0932               "=&r" (original),  // %1
0933               "=&r" (result),    // %2
0934               "+Q" (storage)     // %3
0935             : "r" (v)            // %4
0936             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0937         );
0938         core_arch_operations_gcc_arm_base::fence_after(order);
0939         return result;
0940     }
0941
0942     static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0943     {
0944         core_arch_operations_gcc_arm_base::fence_before(order);
0945         storage_type original, result;
0946         uint32_t tmp;
0947         __asm__ __volatile__
0948         (
0949             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
0950             "1:\n\t"
0951             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
0952             "subs   " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(4) "\n\t" // result = original - value
0953             "sbc    " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(4) "\n\t"
0954             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
0955             "teq     %0, #0\n\t"                      // flags = tmp==0
0956             "bne     1b\n\t"                          // if (!flags.equal) goto retry
0957             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
0958             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
0959               "=&r" (original),  // %1
0960               "=&r" (result),    // %2
0961               "+Q" (storage)     // %3
0962             : "r" (v)            // %4
0963             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0964         );
0965         core_arch_operations_gcc_arm_base::fence_after(order);
0966         return result;
0967     }
0968
0969     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0970     {
0971         core_arch_operations_gcc_arm_base::fence_before(order);
0972         storage_type original, result;
0973         uint32_t tmp;
0974         __asm__ __volatile__
0975         (
0976             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
0977             "1:\n\t"
0978             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
0979             "and     %2, %1, %4\n\t"                  // result = original & value
0980             "and     %H2, %H1, %H4\n\t"
0981             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
0982             "teq     %0, #0\n\t"                      // flags = tmp==0
0983             "bne     1b\n\t"                          // if (!flags.equal) goto retry
0984             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
0985             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
0986               "=&r" (original),  // %1
0987               "=&r" (result),    // %2
0988               "+Q" (storage)     // %3
0989             : "r" (v)            // %4
0990             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0991         );
0992         core_arch_operations_gcc_arm_base::fence_after(order);
0993         return result;
0994     }
0995
0996     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0997     {
0998         core_arch_operations_gcc_arm_base::fence_before(order);
0999         storage_type original, result;
1000         uint32_t tmp;
1001         __asm__ __volatile__
1002         (
1003             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1004             "1:\n\t"
1005             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1006             "orr     %2, %1, %4\n\t"                  // result = original | value
1007             "orr     %H2, %H1, %H4\n\t"
1008             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1009             "teq     %0, #0\n\t"                      // flags = tmp==0
1010             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1011             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1012             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1013               "=&r" (original),  // %1
1014               "=&r" (result),    // %2
1015               "+Q" (storage)     // %3
1016             : "r" (v)            // %4
1017             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1018         );
1019         core_arch_operations_gcc_arm_base::fence_after(order);
1020         return result;
1021     }
1022
1023     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1024     {
1025         core_arch_operations_gcc_arm_base::fence_before(order);
1026         storage_type original, result;
1027         uint32_t tmp;
1028         __asm__ __volatile__
1029         (
1030             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1031             "1:\n\t"
1032             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1033             "eor     %2, %1, %4\n\t"                  // result = original ^ value
1034             "eor     %H2, %H1, %H4\n\t"
1035             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1036             "teq     %0, #0\n\t"                      // flags = tmp==0
1037             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1038             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1039             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1040               "=&r" (original),  // %1
1041               "=&r" (result),    // %2
1042               "+Q" (storage)     // %3
1043             : "r" (v)            // %4
1044             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1045         );
1046         core_arch_operations_gcc_arm_base::fence_after(order);
1047         return result;
1048     }
1049
1050     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1051     {
1052         core_arch_operations_gcc_arm_base::fence_before(order);
1053         storage_type original, result;
1054         uint32_t tmp;
1055         __asm__ __volatile__
1056         (
1057             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1058             "1:\n\t"
1059             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1060             "mvn     %2, %1\n\t"                      // result = NOT original
1061             "mvn     %H2, %H1\n\t"
1062             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1063             "teq     %0, #0\n\t"                      // flags = tmp==0
1064             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1065             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1066             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1067               "=&r" (original),  // %1
1068               "=&r" (result),    // %2
1069               "+Q" (storage)     // %3
1070             :
1071             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1072         );
1073         core_arch_operations_gcc_arm_base::fence_after(order);
1074         return original;
1075     }
1076
1077     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1078     {
1079         core_arch_operations_gcc_arm_base::fence_before(order);
1080         storage_type original, result;
1081         uint32_t tmp;
1082         __asm__ __volatile__
1083         (
1084             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1085             "1:\n\t"
1086             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1087             "mvn     %2, %1\n\t"                      // result = NOT original
1088             "mvn     %H2, %H1\n\t"
1089             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1090             "teq     %0, #0\n\t"                      // flags = tmp==0
1091             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1092             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1093             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1094               "=&r" (original),  // %1
1095               "=&r" (result),    // %2
1096               "+Q" (storage)     // %3
1097             :
1098             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1099         );
1100         core_arch_operations_gcc_arm_base::fence_after(order);
1101         return result;
1102     }
1103 };
1104
1105 template< typename Base, bool Signed >
1106 struct extra_operations< Base, 8u, Signed, true > :
1107     public extra_operations_gcc_arm_common< extra_operations_gcc_arm< Base, 8u, Signed > >
1108 {
1109 };
1110
1111 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
1112
1113 } // namespace detail
1114 } // namespace atomics
1115 } // namespace boost
1116
1117 #include <boost/atomic/detail/footer.hpp>
1118
1119 #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_ARM_HPP_INCLUDED_