atomic/detail/core_arch_ops_gcc_arm.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2009 Helge Bahmann
0007  * Copyright (c) 2013 Tim Blechmann
0008  * Copyright (c) 2014, 2020 Andrey Semashev
0009  */
0010 /*!
0011  * \file   atomic/detail/core_arch_ops_gcc_arm.hpp
0012  *
0013  * This header contains implementation of the \c core_arch_operations template.
0014  */
0015
0016 #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ARM_HPP_INCLUDED_
0017 #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ARM_HPP_INCLUDED_
0018
0019 #include <cstddef>
0020 #include <boost/cstdint.hpp>
0021 #include <boost/memory_order.hpp>
0022 #include <boost/atomic/detail/config.hpp>
0023 #include <boost/atomic/detail/storage_traits.hpp>
0024 #include <boost/atomic/detail/integral_conversions.hpp>
0025 #include <boost/atomic/detail/core_arch_operations_fwd.hpp>
0026 #include <boost/atomic/detail/ops_gcc_arm_common.hpp>
0027 #include <boost/atomic/detail/gcc_arm_asm_common.hpp>
0028 #include <boost/atomic/detail/capabilities.hpp>
0029 #include <boost/atomic/detail/header.hpp>
0030
0031 #ifdef BOOST_HAS_PRAGMA_ONCE
0032 #pragma once
0033 #endif
0034
0035 namespace boost {
0036 namespace atomics {
0037 namespace detail {
0038
0039 // From the ARM Architecture Reference Manual for architecture v6:
0040 //
0041 // LDREX{<cond>} <Rd>, [<Rn>]
0042 // <Rd> Specifies the destination register for the memory word addressed by <Rd>
0043 // <Rn> Specifies the register containing the address.
0044 //
0045 // STREX{<cond>} <Rd>, <Rm>, [<Rn>]
0046 // <Rd> Specifies the destination register for the returned status value.
0047 //      0  if the operation updates memory
0048 //      1  if the operation fails to update memory
0049 // <Rm> Specifies the register containing the word to be stored to memory.
0050 // <Rn> Specifies the register containing the address.
0051 // Rd must not be the same register as Rm or Rn.
0052 //
0053 // ARM v7 is like ARM v6 plus:
0054 // There are half-word and byte versions of the LDREX and STREX instructions,
0055 // LDREXH, LDREXB, STREXH and STREXB.
0056 // There are also double-word versions, LDREXD and STREXD.
0057 // (Actually it looks like these are available from version 6k onwards.)
0058
0059 template< bool Signed, bool Interprocess >
0060 struct core_arch_operations< 4u, Signed, Interprocess > :
0061     public core_arch_operations_gcc_arm_base
0062 {
0063     typedef typename storage_traits< 4u >::type storage_type;
0064
0065     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
0066     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u;
0067     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0068     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0069
0070     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0071     {
0072         fence_before(order);
0073         storage = v;
0074         fence_after_store(order);
0075     }
0076
0077     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0078     {
0079         storage_type v = storage;
0080         fence_after(order);
0081         return v;
0082     }
0083
0084     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0085     {
0086         fence_before(order);
0087         storage_type original;
0088         uint32_t tmp;
0089         __asm__ __volatile__
0090         (
0091             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0092             "1:\n\t"
0093             "ldrex %[original], %[storage]\n\t"          // load the original value
0094             "strex %[tmp], %[value], %[storage]\n\t"     // store the replacement, tmp = store failed
0095             "teq   %[tmp], #0\n\t"                       // check if store succeeded
0096             "bne   1b\n\t"
0097             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0098             : [tmp] "=&l" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)
0099             : [value] "r" (v)
0100             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0101         );
0102         fence_after(order);
0103         return original;
0104     }
0105
0106     static BOOST_FORCEINLINE bool compare_exchange_weak(
0107         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0108     {
0109         fence_before(success_order);
0110         bool success = false;
0111 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0112         uint32_t tmp;
0113 #endif
0114         storage_type original;
0115         __asm__ __volatile__
0116         (
0117             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0118             "ldrex   %[original], %[storage]\n\t"             // original = *(&storage)
0119             "cmp     %[original], %[expected]\n\t"            // flags = original==expected
0120             "itt     eq\n\t"                                  // [hint that the following 2 instructions are conditional on flags.equal]
0121             "strexeq %[success], %[desired], %[storage]\n\t"  // if (flags.equal) *(&storage) = desired, success = store failed
0122             "eoreq   %[success], %[success], #1\n\t"          // if (flags.equal) success ^= 1 (i.e. make it 1 if store succeeded)
0123             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0124             : [original] "=&r" (original),
0125               [success] "+r" (success),
0126 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0127               [tmp] "=&l" (tmp),
0128 #endif
0129               [storage] "+Q" (storage)
0130             : [expected] "Ir" (expected),
0131               [desired] "r" (desired)
0132             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0133         );
0134         if (success)
0135             fence_after(success_order);
0136         else
0137             fence_after(failure_order);
0138         expected = original;
0139         return success;
0140     }
0141
0142     static BOOST_FORCEINLINE bool compare_exchange_strong(
0143         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0144     {
0145         fence_before(success_order);
0146         bool success = false;
0147 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0148         uint32_t tmp;
0149 #endif
0150         storage_type original;
0151         __asm__ __volatile__
0152         (
0153             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0154             "1:\n\t"
0155             "ldrex   %[original], %[storage]\n\t"             // original = *(&storage)
0156             "cmp     %[original], %[expected]\n\t"            // flags = original==expected
0157             "bne     2f\n\t"                                  // if (!flags.equal) goto end
0158             "strex   %[success], %[desired], %[storage]\n\t"  // *(&storage) = desired, success = store failed
0159             "eors    %[success], %[success], #1\n\t"          // success ^= 1 (i.e. make it 1 if store succeeded); flags.equal = success == 0
0160             "beq     1b\n\t"                                  // if (flags.equal) goto retry
0161             "2:\n\t"
0162             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0163             : [original] "=&r" (original),
0164               [success] "+r" (success),
0165 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0166               [tmp] "=&l" (tmp),
0167 #endif
0168               [storage] "+Q" (storage)
0169             : [expected] "Ir" (expected),
0170               [desired] "r" (desired)
0171             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0172         );
0173         if (success)
0174             fence_after(success_order);
0175         else
0176             fence_after(failure_order);
0177         expected = original;
0178         return success;
0179     }
0180
0181     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0182     {
0183         fence_before(order);
0184         uint32_t tmp;
0185         storage_type original, result;
0186         __asm__ __volatile__
0187         (
0188             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0189             "1:\n\t"
0190             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0191             "add     %[result], %[original], %[value]\n\t"  // result = original + value
0192             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0193             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0194             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0195             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0196             : [original] "=&r" (original),  // %0
0197               [result] "=&r" (result),      // %1
0198               [tmp] "=&l" (tmp),            // %2
0199               [storage] "+Q" (storage)      // %3
0200             : [value] "Ir" (v)              // %4
0201             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0202         );
0203         fence_after(order);
0204         return original;
0205     }
0206
0207     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0208     {
0209         fence_before(order);
0210         uint32_t tmp;
0211         storage_type original, result;
0212         __asm__ __volatile__
0213         (
0214             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0215             "1:\n\t"
0216             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0217             "sub     %[result], %[original], %[value]\n\t"  // result = original - value
0218             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0219             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0220             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0221             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0222             : [original] "=&r" (original),  // %0
0223               [result] "=&r" (result),      // %1
0224               [tmp] "=&l" (tmp),            // %2
0225               [storage] "+Q" (storage)      // %3
0226             : [value] "Ir" (v)              // %4
0227             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0228         );
0229         fence_after(order);
0230         return original;
0231     }
0232
0233     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0234     {
0235         fence_before(order);
0236         uint32_t tmp;
0237         storage_type original, result;
0238         __asm__ __volatile__
0239         (
0240             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0241             "1:\n\t"
0242             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0243             "and     %[result], %[original], %[value]\n\t"  // result = original & value
0244             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0245             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0246             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0247             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0248             : [original] "=&r" (original),  // %0
0249               [result] "=&r" (result),      // %1
0250               [tmp] "=&l" (tmp),            // %2
0251               [storage] "+Q" (storage)      // %3
0252             : [value] "Ir" (v)              // %4
0253             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0254         );
0255         fence_after(order);
0256         return original;
0257     }
0258
0259     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0260     {
0261         fence_before(order);
0262         uint32_t tmp;
0263         storage_type original, result;
0264         __asm__ __volatile__
0265         (
0266             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0267             "1:\n\t"
0268             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0269             "orr     %[result], %[original], %[value]\n\t"  // result = original | value
0270             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0271             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0272             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0273             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0274             : [original] "=&r" (original),  // %0
0275               [result] "=&r" (result),      // %1
0276               [tmp] "=&l" (tmp),            // %2
0277               [storage] "+Q" (storage)      // %3
0278             : [value] "Ir" (v)              // %4
0279             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0280         );
0281         fence_after(order);
0282         return original;
0283     }
0284
0285     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0286     {
0287         fence_before(order);
0288         uint32_t tmp;
0289         storage_type original, result;
0290         __asm__ __volatile__
0291         (
0292             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0293             "1:\n\t"
0294             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0295             "eor     %[result], %[original], %[value]\n\t"  // result = original ^ value
0296             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0297             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0298             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0299             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0300             : [original] "=&r" (original),  // %0
0301               [result] "=&r" (result),      // %1
0302               [tmp] "=&l" (tmp),            // %2
0303               [storage] "+Q" (storage)      // %3
0304             : [value] "Ir" (v)              // %4
0305             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0306         );
0307         fence_after(order);
0308         return original;
0309     }
0310
0311     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0312     {
0313         return !!exchange(storage, (storage_type)1, order);
0314     }
0315
0316     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0317     {
0318         store(storage, (storage_type)0, order);
0319     }
0320 };
0321
0322 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
0323
0324 template< bool Signed, bool Interprocess >
0325 struct core_arch_operations< 1u, Signed, Interprocess > :
0326     public core_arch_operations_gcc_arm_base
0327 {
0328     typedef typename storage_traits< 1u >::type storage_type;
0329     typedef typename storage_traits< 4u >::type extended_storage_type;
0330
0331     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
0332     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 1u;
0333     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0334     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0335
0336     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0337     {
0338         fence_before(order);
0339         storage = v;
0340         fence_after_store(order);
0341     }
0342
0343     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0344     {
0345         storage_type v = storage;
0346         fence_after(order);
0347         return v;
0348     }
0349
0350     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0351     {
0352         fence_before(order);
0353         extended_storage_type original;
0354         uint32_t tmp;
0355         __asm__ __volatile__
0356         (
0357             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0358             "1:\n\t"
0359             "ldrexb %[original], %[storage]\n\t"          // load the original value and zero-extend to 32 bits
0360             "strexb %[tmp], %[value], %[storage]\n\t"     // store the replacement, tmp = store failed
0361             "teq    %[tmp], #0\n\t"                       // check if store succeeded
0362             "bne    1b\n\t"
0363             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0364             : [tmp] "=&l" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)
0365             : [value] "r" (v)
0366             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0367         );
0368         fence_after(order);
0369         return static_cast< storage_type >(original);
0370     }
0371
0372     static BOOST_FORCEINLINE bool compare_exchange_weak(
0373         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0374     {
0375         fence_before(success_order);
0376         bool success = false;
0377 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0378         uint32_t tmp;
0379 #endif
0380         extended_storage_type original;
0381         __asm__ __volatile__
0382         (
0383             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0384             "ldrexb   %[original], %[storage]\n\t"             // original = zero_extend(*(&storage))
0385             "cmp      %[original], %[expected]\n\t"            // flags = original==expected
0386             "itt      eq\n\t"                                  // [hint that the following 2 instructions are conditional on flags.equal]
0387             "strexbeq %[success], %[desired], %[storage]\n\t"  // if (flags.equal) *(&storage) = desired, success = store failed
0388             "eoreq    %[success], %[success], #1\n\t"          // if (flags.equal) success ^= 1 (i.e. make it 1 if store succeeded)
0389             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0390             : [original] "=&r" (original),
0391               [success] "+r" (success),
0392 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0393               [tmp] "=&l" (tmp),
0394 #endif
0395               [storage] "+Q" (storage)
0396             : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
0397               [desired] "r" (desired)
0398             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0399         );
0400         if (success)
0401             fence_after(success_order);
0402         else
0403             fence_after(failure_order);
0404         expected = static_cast< storage_type >(original);
0405         return success;
0406     }
0407
0408     static BOOST_FORCEINLINE bool compare_exchange_strong(
0409         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0410     {
0411         fence_before(success_order);
0412         bool success = false;
0413 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0414         uint32_t tmp;
0415 #endif
0416         extended_storage_type original;
0417         __asm__ __volatile__
0418         (
0419             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0420             "1:\n\t"
0421             "ldrexb   %[original], %[storage]\n\t"             // original = zero_extend(*(&storage))
0422             "cmp      %[original], %[expected]\n\t"            // flags = original==expected
0423             "bne      2f\n\t"                                  // if (!flags.equal) goto end
0424             "strexb   %[success], %[desired], %[storage]\n\t"  // *(&storage) = desired, success = store failed
0425             "eors     %[success], %[success], #1\n\t"          // success ^= 1 (i.e. make it 1 if store succeeded); flags.equal = success == 0
0426             "beq      1b\n\t"                                  // if (flags.equal) goto retry
0427             "2:\n\t"
0428             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0429             : [original] "=&r" (original),
0430               [success] "+r" (success),
0431 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0432               [tmp] "=&l" (tmp),
0433 #endif
0434               [storage] "+Q" (storage)
0435             : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
0436               [desired] "r" (desired)
0437             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0438         );
0439         if (success)
0440             fence_after(success_order);
0441         else
0442             fence_after(failure_order);
0443         expected = static_cast< storage_type >(original);
0444         return success;
0445     }
0446
0447     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0448     {
0449         fence_before(order);
0450         uint32_t tmp;
0451         extended_storage_type original, result;
0452         __asm__ __volatile__
0453         (
0454             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0455             "1:\n\t"
0456             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0457             "add      %[result], %[original], %[value]\n\t"  // result = original + value
0458             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0459             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0460             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0461             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0462             : [original] "=&r" (original),  // %0
0463               [result] "=&r" (result),      // %1
0464               [tmp] "=&l" (tmp),            // %2
0465               [storage] "+Q" (storage)      // %3
0466             : [value] "Ir" (v)              // %4
0467             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0468         );
0469         fence_after(order);
0470         return static_cast< storage_type >(original);
0471     }
0472
0473     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0474     {
0475         fence_before(order);
0476         uint32_t tmp;
0477         extended_storage_type original, result;
0478         __asm__ __volatile__
0479         (
0480             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0481             "1:\n\t"
0482             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0483             "sub      %[result], %[original], %[value]\n\t"  // result = original - value
0484             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0485             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0486             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0487             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0488             : [original] "=&r" (original),  // %0
0489               [result] "=&r" (result),      // %1
0490               [tmp] "=&l" (tmp),            // %2
0491               [storage] "+Q" (storage)      // %3
0492             : [value] "Ir" (v)              // %4
0493             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0494         );
0495         fence_after(order);
0496         return static_cast< storage_type >(original);
0497     }
0498
0499     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0500     {
0501         fence_before(order);
0502         uint32_t tmp;
0503         extended_storage_type original, result;
0504         __asm__ __volatile__
0505         (
0506             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0507             "1:\n\t"
0508             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0509             "and      %[result], %[original], %[value]\n\t"  // result = original & value
0510             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0511             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0512             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0513             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0514             : [original] "=&r" (original),  // %0
0515               [result] "=&r" (result),      // %1
0516               [tmp] "=&l" (tmp),            // %2
0517               [storage] "+Q" (storage)      // %3
0518             : [value] "Ir" (v)              // %4
0519             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0520         );
0521         fence_after(order);
0522         return static_cast< storage_type >(original);
0523     }
0524
0525     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0526     {
0527         fence_before(order);
0528         uint32_t tmp;
0529         extended_storage_type original, result;
0530         __asm__ __volatile__
0531         (
0532             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0533             "1:\n\t"
0534             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0535             "orr      %[result], %[original], %[value]\n\t"  // result = original | value
0536             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0537             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0538             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0539             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0540             : [original] "=&r" (original),  // %0
0541               [result] "=&r" (result),      // %1
0542               [tmp] "=&l" (tmp),            // %2
0543               [storage] "+Q" (storage)      // %3
0544             : [value] "Ir" (v)              // %4
0545             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0546         );
0547         fence_after(order);
0548         return static_cast< storage_type >(original);
0549     }
0550
0551     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0552     {
0553         fence_before(order);
0554         uint32_t tmp;
0555         extended_storage_type original, result;
0556         __asm__ __volatile__
0557         (
0558             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0559             "1:\n\t"
0560             "ldrexb   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0561             "eor      %[result], %[original], %[value]\n\t"  // result = original ^ value
0562             "strexb   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0563             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0564             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0565             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0566             : [original] "=&r" (original),  // %0
0567               [result] "=&r" (result),      // %1
0568               [tmp] "=&l" (tmp),            // %2
0569               [storage] "+Q" (storage)      // %3
0570             : [value] "Ir" (v)              // %4
0571             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0572         );
0573         fence_after(order);
0574         return static_cast< storage_type >(original);
0575     }
0576
0577     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0578     {
0579         return !!exchange(storage, (storage_type)1, order);
0580     }
0581
0582     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0583     {
0584         store(storage, (storage_type)0, order);
0585     }
0586 };
0587
0588 #else // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
0589
0590 template< bool Interprocess >
0591 struct core_arch_operations< 1u, false, Interprocess > :
0592     public core_arch_operations< 4u, false, Interprocess >
0593 {
0594     typedef core_arch_operations< 4u, false, Interprocess > base_type;
0595     typedef typename base_type::storage_type storage_type;
0596
0597     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0598     {
0599         base_type::fence_before(order);
0600         uint32_t tmp;
0601         storage_type original, result;
0602         __asm__ __volatile__
0603         (
0604             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0605             "1:\n\t"
0606             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0607             "add     %[result], %[original], %[value]\n\t"  // result = original + value
0608             "uxtb    %[result], %[result]\n\t"              // zero extend result from 8 to 32 bits
0609             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0610             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0611             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0612             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0613             : [original] "=&r" (original),  // %0
0614               [result] "=&r" (result),      // %1
0615               [tmp] "=&l" (tmp),            // %2
0616               [storage] "+Q" (storage)      // %3
0617             : [value] "Ir" (v)              // %4
0618             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0619         );
0620         base_type::fence_after(order);
0621         return original;
0622     }
0623
0624     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0625     {
0626         base_type::fence_before(order);
0627         uint32_t tmp;
0628         storage_type original, result;
0629         __asm__ __volatile__
0630         (
0631             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0632             "1:\n\t"
0633             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0634             "sub     %[result], %[original], %[value]\n\t"  // result = original - value
0635             "uxtb    %[result], %[result]\n\t"              // zero extend result from 8 to 32 bits
0636             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0637             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0638             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0639             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0640             : [original] "=&r" (original),  // %0
0641               [result] "=&r" (result),      // %1
0642               [tmp] "=&l" (tmp),            // %2
0643               [storage] "+Q" (storage)      // %3
0644             : [value] "Ir" (v)              // %4
0645             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0646         );
0647         base_type::fence_after(order);
0648         return original;
0649     }
0650 };
0651
0652 template< bool Interprocess >
0653 struct core_arch_operations< 1u, true, Interprocess > :
0654     public core_arch_operations< 4u, true, Interprocess >
0655 {
0656     typedef core_arch_operations< 4u, true, Interprocess > base_type;
0657     typedef typename base_type::storage_type storage_type;
0658
0659     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0660     {
0661         base_type::fence_before(order);
0662         uint32_t tmp;
0663         storage_type original, result;
0664         __asm__ __volatile__
0665         (
0666             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0667             "1:\n\t"
0668             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0669             "add     %[result], %[original], %[value]\n\t"  // result = original + value
0670             "sxtb    %[result], %[result]\n\t"              // sign extend result from 8 to 32 bits
0671             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0672             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0673             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0674             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0675             : [original] "=&r" (original),  // %0
0676               [result] "=&r" (result),      // %1
0677               [tmp] "=&l" (tmp),            // %2
0678               [storage] "+Q" (storage)      // %3
0679             : [value] "Ir" (v)              // %4
0680             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0681         );
0682         base_type::fence_after(order);
0683         return original;
0684     }
0685
0686     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0687     {
0688         base_type::fence_before(order);
0689         uint32_t tmp;
0690         storage_type original, result;
0691         __asm__ __volatile__
0692         (
0693             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0694             "1:\n\t"
0695             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
0696             "sub     %[result], %[original], %[value]\n\t"  // result = original - value
0697             "sxtb    %[result], %[result]\n\t"              // sign extend result from 8 to 32 bits
0698             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0699             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
0700             "bne     1b\n\t"                                // if (!flags.equal) goto retry
0701             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0702             : [original] "=&r" (original),  // %0
0703               [result] "=&r" (result),      // %1
0704               [tmp] "=&l" (tmp),            // %2
0705               [storage] "+Q" (storage)      // %3
0706             : [value] "Ir" (v)              // %4
0707             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0708         );
0709         base_type::fence_after(order);
0710         return original;
0711     }
0712 };
0713
0714 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
0715
0716 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
0717
0718 template< bool Signed, bool Interprocess >
0719 struct core_arch_operations< 2u, Signed, Interprocess > :
0720     public core_arch_operations_gcc_arm_base
0721 {
0722     typedef typename storage_traits< 2u >::type storage_type;
0723     typedef typename storage_traits< 4u >::type extended_storage_type;
0724
0725     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
0726     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 2u;
0727     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0728     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0729
0730     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0731     {
0732         fence_before(order);
0733         storage = v;
0734         fence_after_store(order);
0735     }
0736
0737     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0738     {
0739         storage_type v = storage;
0740         fence_after(order);
0741         return v;
0742     }
0743
0744     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0745     {
0746         fence_before(order);
0747         extended_storage_type original;
0748         uint32_t tmp;
0749         __asm__ __volatile__
0750         (
0751             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0752             "1:\n\t"
0753             "ldrexh %[original], %[storage]\n\t"          // load the original value and zero-extend to 32 bits
0754             "strexh %[tmp], %[value], %[storage]\n\t"     // store the replacement, tmp = store failed
0755             "teq    %[tmp], #0\n\t"                       // check if store succeeded
0756             "bne    1b\n\t"
0757             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0758             : [tmp] "=&l" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)
0759             : [value] "r" (v)
0760             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0761         );
0762         fence_after(order);
0763         return static_cast< storage_type >(original);
0764     }
0765
0766     static BOOST_FORCEINLINE bool compare_exchange_weak(
0767         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0768     {
0769         fence_before(success_order);
0770         bool success = false;
0771 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0772         uint32_t tmp;
0773 #endif
0774         extended_storage_type original;
0775         __asm__ __volatile__
0776         (
0777             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0778             "ldrexh   %[original], %[storage]\n\t"             // original = zero_extend(*(&storage))
0779             "cmp      %[original], %[expected]\n\t"            // flags = original==expected
0780             "itt      eq\n\t"                                  // [hint that the following 2 instructions are conditional on flags.equal]
0781             "strexheq %[success], %[desired], %[storage]\n\t"  // if (flags.equal) *(&storage) = desired, success = store failed
0782             "eoreq    %[success], %[success], #1\n\t"          // if (flags.equal) success ^= 1 (i.e. make it 1 if store succeeded)
0783             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0784             : [original] "=&r" (original),
0785               [success] "+r" (success),
0786 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0787               [tmp] "=&l" (tmp),
0788 #endif
0789               [storage] "+Q" (storage)
0790             : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
0791               [desired] "r" (desired)
0792             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0793         );
0794         if (success)
0795             fence_after(success_order);
0796         else
0797             fence_after(failure_order);
0798         expected = static_cast< storage_type >(original);
0799         return success;
0800     }
0801
0802     static BOOST_FORCEINLINE bool compare_exchange_strong(
0803         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0804     {
0805         fence_before(success_order);
0806         bool success = false;
0807 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0808         uint32_t tmp;
0809 #endif
0810         extended_storage_type original;
0811         __asm__ __volatile__
0812         (
0813             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0814             "1:\n\t"
0815             "ldrexh   %[original], %[storage]\n\t"             // original = zero_extend(*(&storage))
0816             "cmp      %[original], %[expected]\n\t"            // flags = original==expected
0817             "bne      2f\n\t"                                  // if (!flags.equal) goto end
0818             "strexh   %[success], %[desired], %[storage]\n\t"  // *(&storage) = desired, success = store failed
0819             "eors     %[success], %[success], #1\n\t"          // success ^= 1 (i.e. make it 1 if store succeeded); flags.equal = success == 0
0820             "beq      1b\n\t"                                  // if (flags.equal) goto retry
0821             "2:\n\t"
0822             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0823             : [original] "=&r" (original),
0824               [success] "+r" (success),
0825 #if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
0826               [tmp] "=&l" (tmp),
0827 #endif
0828               [storage] "+Q" (storage)
0829             : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
0830               [desired] "r" (desired)
0831             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0832         );
0833         if (success)
0834             fence_after(success_order);
0835         else
0836             fence_after(failure_order);
0837         expected = static_cast< storage_type >(original);
0838         return success;
0839     }
0840
0841     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0842     {
0843         fence_before(order);
0844         uint32_t tmp;
0845         extended_storage_type original, result;
0846         __asm__ __volatile__
0847         (
0848             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0849             "1:\n\t"
0850             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0851             "add      %[result], %[original], %[value]\n\t"  // result = original + value
0852             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0853             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0854             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0855             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0856             : [original] "=&r" (original),  // %0
0857               [result] "=&r" (result),      // %1
0858               [tmp] "=&l" (tmp),            // %2
0859               [storage] "+Q" (storage)      // %3
0860             : [value] "Ir" (v)              // %4
0861             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0862         );
0863         fence_after(order);
0864         return static_cast< storage_type >(original);
0865     }
0866
0867     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0868     {
0869         fence_before(order);
0870         uint32_t tmp;
0871         extended_storage_type original, result;
0872         __asm__ __volatile__
0873         (
0874             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0875             "1:\n\t"
0876             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0877             "sub      %[result], %[original], %[value]\n\t"  // result = original - value
0878             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0879             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0880             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0881             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0882             : [original] "=&r" (original),  // %0
0883               [result] "=&r" (result),      // %1
0884               [tmp] "=&l" (tmp),            // %2
0885               [storage] "+Q" (storage)      // %3
0886             : [value] "Ir" (v)              // %4
0887             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0888         );
0889         fence_after(order);
0890         return static_cast< storage_type >(original);
0891     }
0892
0893     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0894     {
0895         fence_before(order);
0896         uint32_t tmp;
0897         extended_storage_type original, result;
0898         __asm__ __volatile__
0899         (
0900             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0901             "1:\n\t"
0902             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0903             "and      %[result], %[original], %[value]\n\t"  // result = original & value
0904             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0905             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0906             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0907             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0908             : [original] "=&r" (original),  // %0
0909               [result] "=&r" (result),      // %1
0910               [tmp] "=&l" (tmp),            // %2
0911               [storage] "+Q" (storage)      // %3
0912             : [value] "Ir" (v)              // %4
0913             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0914         );
0915         fence_after(order);
0916         return static_cast< storage_type >(original);
0917     }
0918
0919     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0920     {
0921         fence_before(order);
0922         uint32_t tmp;
0923         extended_storage_type original, result;
0924         __asm__ __volatile__
0925         (
0926             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0927             "1:\n\t"
0928             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0929             "orr      %[result], %[original], %[value]\n\t"  // result = original | value
0930             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0931             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0932             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0933             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0934             : [original] "=&r" (original),  // %0
0935               [result] "=&r" (result),      // %1
0936               [tmp] "=&l" (tmp),            // %2
0937               [storage] "+Q" (storage)      // %3
0938             : [value] "Ir" (v)              // %4
0939             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0940         );
0941         fence_after(order);
0942         return static_cast< storage_type >(original);
0943     }
0944
0945     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0946     {
0947         fence_before(order);
0948         uint32_t tmp;
0949         extended_storage_type original, result;
0950         __asm__ __volatile__
0951         (
0952             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0953             "1:\n\t"
0954             "ldrexh   %[original], %[storage]\n\t"           // original = zero_extend(*(&storage))
0955             "eor      %[result], %[original], %[value]\n\t"  // result = original ^ value
0956             "strexh   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
0957             "teq      %[tmp], #0\n\t"                        // flags = tmp==0
0958             "bne      1b\n\t"                                // if (!flags.equal) goto retry
0959             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
0960             : [original] "=&r" (original),  // %0
0961               [result] "=&r" (result),      // %1
0962               [tmp] "=&l" (tmp),            // %2
0963               [storage] "+Q" (storage)      // %3
0964             : [value] "Ir" (v)              // %4
0965             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0966         );
0967         fence_after(order);
0968         return static_cast< storage_type >(original);
0969     }
0970
0971     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0972     {
0973         return !!exchange(storage, (storage_type)1, order);
0974     }
0975
0976     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0977     {
0978         store(storage, (storage_type)0, order);
0979     }
0980 };
0981
0982 #else // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
0983
0984 template< bool Interprocess >
0985 struct core_arch_operations< 2u, false, Interprocess > :
0986     public core_arch_operations< 4u, false, Interprocess >
0987 {
0988     typedef core_arch_operations< 4u, false, Interprocess > base_type;
0989     typedef typename base_type::storage_type storage_type;
0990
0991     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0992     {
0993         base_type::fence_before(order);
0994         uint32_t tmp;
0995         storage_type original, result;
0996         __asm__ __volatile__
0997         (
0998             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
0999             "1:\n\t"
1000             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
1001             "add     %[result], %[original], %[value]\n\t"  // result = original + value
1002             "uxth    %[result], %[result]\n\t"              // zero extend result from 16 to 32 bits
1003             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
1004             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
1005             "bne     1b\n\t"                                // if (!flags.equal) goto retry
1006             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1007             : [original] "=&r" (original),  // %0
1008               [result] "=&r" (result),      // %1
1009               [tmp] "=&l" (tmp),            // %2
1010               [storage] "+Q" (storage)      // %3
1011             : [value] "Ir" (v)              // %4
1012             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1013         );
1014         base_type::fence_after(order);
1015         return original;
1016     }
1017
1018     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1019     {
1020         base_type::fence_before(order);
1021         uint32_t tmp;
1022         storage_type original, result;
1023         __asm__ __volatile__
1024         (
1025             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
1026             "1:\n\t"
1027             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
1028             "sub     %[result], %[original], %[value]\n\t"  // result = original - value
1029             "uxth    %[result], %[result]\n\t"              // zero extend result from 16 to 32 bits
1030             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
1031             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
1032             "bne     1b\n\t"                                // if (!flags.equal) goto retry
1033             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1034             : [original] "=&r" (original),  // %0
1035               [result] "=&r" (result),      // %1
1036               [tmp] "=&l" (tmp),            // %2
1037               [storage] "+Q" (storage)      // %3
1038             : [value] "Ir" (v)              // %4
1039             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1040         );
1041         base_type::fence_after(order);
1042         return original;
1043     }
1044 };
1045
1046 template< bool Interprocess >
1047 struct core_arch_operations< 2u, true, Interprocess > :
1048     public core_arch_operations< 4u, true, Interprocess >
1049 {
1050     typedef core_arch_operations< 4u, true, Interprocess > base_type;
1051     typedef typename base_type::storage_type storage_type;
1052
1053     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1054     {
1055         base_type::fence_before(order);
1056         uint32_t tmp;
1057         storage_type original, result;
1058         __asm__ __volatile__
1059         (
1060             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
1061             "1:\n\t"
1062             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
1063             "add     %[result], %[original], %[value]\n\t"  // result = original + value
1064             "sxth    %[result], %[result]\n\t"              // sign extend result from 16 to 32 bits
1065             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
1066             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
1067             "bne     1b\n\t"                                // if (!flags.equal) goto retry
1068             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1069             : [original] "=&r" (original),  // %0
1070               [result] "=&r" (result),      // %1
1071               [tmp] "=&l" (tmp),            // %2
1072               [storage] "+Q" (storage)      // %3
1073             : [value] "Ir" (v)              // %4
1074             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1075         );
1076         base_type::fence_after(order);
1077         return original;
1078     }
1079
1080     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1081     {
1082         base_type::fence_before(order);
1083         uint32_t tmp;
1084         storage_type original, result;
1085         __asm__ __volatile__
1086         (
1087             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
1088             "1:\n\t"
1089             "ldrex   %[original], %[storage]\n\t"           // original = *(&storage)
1090             "sub     %[result], %[original], %[value]\n\t"  // result = original - value
1091             "sxth    %[result], %[result]\n\t"              // sign extend result from 16 to 32 bits
1092             "strex   %[tmp], %[result], %[storage]\n\t"     // *(&storage) = result, tmp = store failed
1093             "teq     %[tmp], #0\n\t"                        // flags = tmp==0
1094             "bne     1b\n\t"                                // if (!flags.equal) goto retry
1095             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1096             : [original] "=&r" (original),  // %0
1097               [result] "=&r" (result),      // %1
1098               [tmp] "=&l" (tmp),            // %2
1099               [storage] "+Q" (storage)      // %3
1100             : [value] "Ir" (v)              // %4
1101             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1102         );
1103         base_type::fence_after(order);
1104         return original;
1105     }
1106 };
1107
1108 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
1109
1110 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
1111
1112 // Unlike 32-bit operations, for 64-bit loads and stores we must use ldrexd/strexd.
1113 // Any other instructions result in a non-atomic sequence of 32-bit accesses.
1114 // See "ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition",
1115 // Section A3.5.3 "Atomicity in the ARM architecture".
1116
1117 // In the asm blocks below we have to use 32-bit register pairs to compose 64-bit values.
1118 // In order to pass the 64-bit operands to/from asm blocks, we use undocumented gcc feature:
1119 // the lower half (Rt) of the operand is accessible normally, via the numbered placeholder (e.g. %0),
1120 // and the upper half (Rt2) - via the same placeholder with an 'H' after the '%' sign (e.g. %H0).
1121 // See: http://hardwarebug.org/2010/07/06/arm-inline-asm-secrets/
1122
1123 template< bool Signed, bool Interprocess >
1124 struct core_arch_operations< 8u, Signed, Interprocess > :
1125     public core_arch_operations_gcc_arm_base
1126 {
1127     typedef typename storage_traits< 8u >::type storage_type;
1128
1129     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
1130     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u;
1131     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
1132     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
1133
1134     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1135     {
1136         exchange(storage, v, order);
1137     }
1138
1139     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
1140     {
1141         // ARMv7 says ldrex (and other load-exclusive instructions) can be used without a matching strex, see
1142         // "ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition", Section A3.4.5 "Load-Exclusive and Store-Exclusive usage restrictions".
1143         storage_type original;
1144 #if defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
1145         __asm__ __volatile__
1146         (
1147             "ldrexd %0, %H0, %1\n\t"
1148             : "=&r" (original)   // %0
1149             : "Q" (storage)      // %1
1150         );
1151 #else
1152         uint32_t tmp;
1153         __asm__ __volatile__
1154         (
1155             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1156             "ldrexd %1, %H1, %2\n\t"
1157             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1158             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1159               "=&r" (original)   // %1
1160             : "Q" (storage)      // %2
1161         );
1162 #endif
1163         fence_after(order);
1164         return original;
1165     }
1166
1167     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1168     {
1169         fence_before(order);
1170         storage_type original;
1171         uint32_t tmp;
1172         __asm__ __volatile__
1173         (
1174             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1175             "1:\n\t"
1176             "ldrexd %1, %H1, %2\n\t"        // load the original value
1177             "strexd %0, %3, %H3, %2\n\t"    // store the replacement, tmp = store failed
1178             "teq    %0, #0\n\t"               // check if store succeeded
1179             "bne    1b\n\t"
1180             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1181             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1182               "=&r" (original),  // %1
1183               "+Q" (storage)     // %2
1184             : "r" (v)            // %3
1185             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1186         );
1187         fence_after(order);
1188         return original;
1189     }
1190
1191     static BOOST_FORCEINLINE bool compare_exchange_weak(
1192         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1193     {
1194         fence_before(success_order);
1195         storage_type original;
1196         bool success = false;
1197         uint32_t tmp;
1198         __asm__ __volatile__
1199         (
1200             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1201             "ldrexd   %1, %H1, %3\n\t"               // original = *(&storage)
1202             "cmp      %1, %4\n\t"                    // flags = original.lo==expected.lo
1203             "it       eq\n\t"                        // [hint that the following 1 instruction is conditional on flags.equal]
1204             "cmpeq    %H1, %H4\n\t"                  // if (flags.equal) flags = original.hi==expected.hi
1205             "bne      1f\n\t"
1206             "strexd   %2, %5, %H5, %3\n\t"           // *(&storage) = desired, success = store failed
1207             "eor      %2, %2, #1\n\t"                // success ^= 1 (i.e. make it 1 if store succeeded)
1208             "1:\n\t"
1209             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1210             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1211               "=&r" (original),  // %1
1212               "+r" (success),    // %2
1213               "+Q" (storage)     // %3
1214             : "r" (expected),    // %4
1215               "r" (desired)      // %5
1216             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1217         );
1218         if (success)
1219             fence_after(success_order);
1220         else
1221             fence_after(failure_order);
1222         expected = original;
1223         return success;
1224     }
1225
1226     static BOOST_FORCEINLINE bool compare_exchange_strong(
1227         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1228     {
1229         fence_before(success_order);
1230         storage_type original;
1231         bool success = false;
1232         uint32_t tmp;
1233         __asm__ __volatile__
1234         (
1235             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1236             "1:\n\t"
1237             "ldrexd   %1, %H1, %3\n\t"               // original = *(&storage)
1238             "cmp      %1, %4\n\t"                    // flags = original.lo==expected.lo
1239             "it       eq\n\t"                        // [hint that the following 1 instruction is conditional on flags.equal]
1240             "cmpeq    %H1, %H4\n\t"                  // if (flags.equal) flags = original.hi==expected.hi
1241             "bne      2f\n\t"
1242             "strexd   %2, %5, %H5, %3\n\t"           // *(&storage) = desired, success = store failed
1243             "eors     %2, %2, #1\n\t"                // success ^= 1 (i.e. make it 1 if store succeeded), flags.equal = success == 0
1244             "beq      1b\n\t"                        // if (flags.equal) goto retry
1245             "2:\n\t"
1246             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1247             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1248               "=&r" (original),  // %1
1249               "+r" (success),    // %2
1250               "+Q" (storage)     // %3
1251             : "r" (expected),    // %4
1252               "r" (desired)      // %5
1253             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1254         );
1255         if (success)
1256             fence_after(success_order);
1257         else
1258             fence_after(failure_order);
1259         expected = original;
1260         return success;
1261     }
1262
1263     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1264     {
1265         fence_before(order);
1266         storage_type original, result;
1267         uint32_t tmp;
1268         __asm__ __volatile__
1269         (
1270             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1271             "1:\n\t"
1272             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1273             "adds   " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(4) "\n\t" // result = original + value
1274             "adc    " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(4) "\n\t"
1275             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1276             "teq     %0, #0\n\t"                      // flags = tmp==0
1277             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1278             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1279             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1280               "=&r" (original),  // %1
1281               "=&r" (result),    // %2
1282               "+Q" (storage)     // %3
1283             : "r" (v)            // %4
1284             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1285         );
1286         fence_after(order);
1287         return original;
1288     }
1289
1290     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1291     {
1292         fence_before(order);
1293         storage_type original, result;
1294         uint32_t tmp;
1295         __asm__ __volatile__
1296         (
1297             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1298             "1:\n\t"
1299             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1300             "subs   " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(4) "\n\t" // result = original - value
1301             "sbc    " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(4) "\n\t"
1302             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1303             "teq     %0, #0\n\t"                      // flags = tmp==0
1304             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1305             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1306             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1307               "=&r" (original),  // %1
1308               "=&r" (result),    // %2
1309               "+Q" (storage)     // %3
1310             : "r" (v)            // %4
1311             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1312         );
1313         fence_after(order);
1314         return original;
1315     }
1316
1317     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1318     {
1319         fence_before(order);
1320         storage_type original, result;
1321         uint32_t tmp;
1322         __asm__ __volatile__
1323         (
1324             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1325             "1:\n\t"
1326             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1327             "and     %2, %1, %4\n\t"                  // result = original & value
1328             "and     %H2, %H1, %H4\n\t"
1329             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1330             "teq     %0, #0\n\t"                      // flags = tmp==0
1331             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1332             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1333             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1334               "=&r" (original),  // %1
1335               "=&r" (result),    // %2
1336               "+Q" (storage)     // %3
1337             : "r" (v)            // %4
1338             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1339         );
1340         fence_after(order);
1341         return original;
1342     }
1343
1344     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1345     {
1346         fence_before(order);
1347         storage_type original, result;
1348         uint32_t tmp;
1349         __asm__ __volatile__
1350         (
1351             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1352             "1:\n\t"
1353             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1354             "orr     %2, %1, %4\n\t"                  // result = original | value
1355             "orr     %H2, %H1, %H4\n\t"
1356             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1357             "teq     %0, #0\n\t"                      // flags = tmp==0
1358             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1359             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1360             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1361               "=&r" (original),  // %1
1362               "=&r" (result),    // %2
1363               "+Q" (storage)     // %3
1364             : "r" (v)            // %4
1365             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1366         );
1367         fence_after(order);
1368         return original;
1369     }
1370
1371     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1372     {
1373         fence_before(order);
1374         storage_type original, result;
1375         uint32_t tmp;
1376         __asm__ __volatile__
1377         (
1378             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
1379             "1:\n\t"
1380             "ldrexd  %1, %H1, %3\n\t"                 // original = *(&storage)
1381             "eor     %2, %1, %4\n\t"                  // result = original ^ value
1382             "eor     %H2, %H1, %H4\n\t"
1383             "strexd  %0, %2, %H2, %3\n\t"             // *(&storage) = result, tmp = store failed
1384             "teq     %0, #0\n\t"                      // flags = tmp==0
1385             "bne     1b\n\t"                          // if (!flags.equal) goto retry
1386             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1387             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1388               "=&r" (original),  // %1
1389               "=&r" (result),    // %2
1390               "+Q" (storage)     // %3
1391             : "r" (v)            // %4
1392             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1393         );
1394         fence_after(order);
1395         return original;
1396     }
1397
1398     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1399     {
1400         return !!exchange(storage, (storage_type)1, order);
1401     }
1402
1403     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1404     {
1405         store(storage, (storage_type)0, order);
1406     }
1407 };
1408
1409 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
1410
1411 } // namespace detail
1412 } // namespace atomics
1413 } // namespace boost
1414
1415 #include <boost/atomic/detail/footer.hpp>
1416
1417 #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ARM_HPP_INCLUDED_