atomic/detail/core_arch_ops_gcc_ppc.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2009 Helge Bahmann
0007  * Copyright (c) 2013 Tim Blechmann
0008  * Copyright (c) 2014 Andrey Semashev
0009  */
0010 /*!
0011  * \file   atomic/detail/core_arch_ops_gcc_ppc.hpp
0012  *
0013  * This header contains implementation of the \c core_arch_operations template.
0014  */
0015
0016 #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_
0017 #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_
0018
0019 #include <cstddef>
0020 #include <boost/memory_order.hpp>
0021 #include <boost/atomic/detail/config.hpp>
0022 #include <boost/atomic/detail/storage_traits.hpp>
0023 #include <boost/atomic/detail/core_arch_operations_fwd.hpp>
0024 #include <boost/atomic/detail/ops_gcc_ppc_common.hpp>
0025 #include <boost/atomic/detail/gcc_ppc_asm_common.hpp>
0026 #include <boost/atomic/detail/capabilities.hpp>
0027 #include <boost/atomic/detail/header.hpp>
0028
0029 #ifdef BOOST_HAS_PRAGMA_ONCE
0030 #pragma once
0031 #endif
0032
0033 namespace boost {
0034 namespace atomics {
0035 namespace detail {
0036
0037 // The implementation below uses information from this document:
0038 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2010.02.19a.html
0039
0040 /*
0041     Refer to: Motorola: "Programming Environments Manual for 32-Bit
0042     Implementations of the PowerPC Architecture", Appendix E:
0043     "Synchronization Programming Examples" for an explanation of what is
0044     going on here (can be found on the web at various places by the
0045     name "MPCFPE32B.pdf", Google is your friend...)
0046
0047     Most of the atomic operations map to instructions in a relatively
0048     straight-forward fashion, but "load"s may at first glance appear
0049     a bit strange as they map to:
0050
0051             lwz %rX, addr
0052             cmpw %rX, %rX
0053             bne- 1f
0054         1:
0055
0056     That is, the CPU is forced to perform a branch that "formally" depends
0057     on the value retrieved from memory. This scheme has an overhead of
0058     about 1-2 clock cycles per load, but it allows to map "acquire" to
0059     the "isync" instruction instead of "sync" uniformly and for all type
0060     of atomic operations. Since "isync" has a cost of about 15 clock
0061     cycles, while "sync" hast a cost of about 50 clock cycles, the small
0062     penalty to atomic loads more than compensates for this.
0063
0064     Byte- and halfword-sized atomic values are implemented in two ways.
0065     When 8 and 16-bit instructions are available (in Power8 and later),
0066     they are used. Otherwise operations are realized by encoding the
0067     value to be represented into a word, performing sign/zero extension
0068     as appropriate. This means that after add/sub operations the value
0069     needs fixing up to accurately preserve the wrap-around semantic of
0070     the smaller type. (Nothing special needs to be done for the bit-wise
0071     and the "exchange type" operators as the compiler already sees to
0072     it that values carried in registers are extended appropriately and
0073     everything falls into place naturally).
0074
0075     The register constraint "b"  instructs gcc to use any register
0076     except r0; this is sometimes required because the encoding for
0077     r0 is used to signify "constant zero" in a number of instructions,
0078     making r0 unusable in this place. For simplicity this constraint
0079     is used everywhere since I am to lazy to look this up on a
0080     per-instruction basis, and ppc has enough registers for this not
0081     to pose a problem.
0082 */
0083
0084 template< bool Signed, bool Interprocess >
0085 struct core_arch_operations< 4u, Signed, Interprocess > :
0086     public core_arch_operations_gcc_ppc_base
0087 {
0088     typedef typename storage_traits< 4u >::type storage_type;
0089
0090     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
0091     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u;
0092     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0093     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0094
0095     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0096     {
0097         fence_before(order);
0098         __asm__ __volatile__
0099         (
0100             "stw %1, %0\n\t"
0101             : "+m" (storage)
0102             : "r" (v)
0103         );
0104     }
0105
0106     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0107     {
0108         storage_type v;
0109         if (order == memory_order_seq_cst)
0110             __asm__ __volatile__ ("sync" ::: "memory");
0111         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0112         {
0113             __asm__ __volatile__
0114             (
0115                 "lwz %0, %1\n\t"
0116                 "cmpw %0, %0\n\t"
0117                 BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4")
0118                 BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0119                 "isync\n\t"
0120                 : "=&r" (v)
0121                 : "m" (storage)
0122                 : "cr0", "memory"
0123             );
0124         }
0125         else
0126         {
0127             __asm__ __volatile__
0128             (
0129                 "lwz %0, %1\n\t"
0130                 : "=&r" (v)
0131                 : "m" (storage)
0132             );
0133         }
0134         return v;
0135     }
0136
0137     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0138     {
0139         storage_type original;
0140         fence_before(order);
0141         __asm__ __volatile__
0142         (
0143             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0144             "lwarx %0,%y1\n\t"
0145             "stwcx. %2,%y1\n\t"
0146             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8")
0147             : "=&b" (original), "+Z" (storage)
0148             : "b" (v)
0149             : "cr0"
0150         );
0151         fence_after(order);
0152         return original;
0153     }
0154
0155     static BOOST_FORCEINLINE bool compare_exchange_weak(
0156         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0157     {
0158         int success;
0159         fence_before(success_order);
0160         __asm__ __volatile__
0161         (
0162             "li %1, 0\n\t"
0163             "lwarx %0,%y2\n\t"
0164             "cmpw %0, %3\n\t"
0165             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
0166             "stwcx. %4,%y2\n\t"
0167             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8")
0168             "li %1, 1\n\t"
0169             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0170             : "=&b" (expected), "=&b" (success), "+Z" (storage)
0171             : "b" (expected), "b" (desired)
0172             : "cr0"
0173         );
0174         if (success)
0175             fence_after(success_order);
0176         else
0177             fence_after(failure_order);
0178         return !!success;
0179     }
0180
0181     static BOOST_FORCEINLINE bool compare_exchange_strong(
0182         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0183     {
0184         int success;
0185         fence_before(success_order);
0186         __asm__ __volatile__
0187         (
0188             "li %1, 0\n\t"
0189             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0")
0190             "lwarx %0,%y2\n\t"
0191             "cmpw %0, %3\n\t"
0192             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
0193             "stwcx. %4,%y2\n\t"
0194             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16")
0195             "li %1, 1\n\t"
0196             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0197             : "=&b" (expected), "=&b" (success), "+Z" (storage)
0198             : "b" (expected), "b" (desired)
0199             : "cr0"
0200         );
0201         if (success)
0202             fence_after(success_order);
0203         else
0204             fence_after(failure_order);
0205         return !!success;
0206     }
0207
0208     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0209     {
0210         storage_type original, result;
0211         fence_before(order);
0212         __asm__ __volatile__
0213         (
0214             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0215             "lwarx %0,%y2\n\t"
0216             "add %1,%0,%3\n\t"
0217             "stwcx. %1,%y2\n\t"
0218             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0219             : "=&b" (original), "=&b" (result), "+Z" (storage)
0220             : "b" (v)
0221             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0222         );
0223         fence_after(order);
0224         return original;
0225     }
0226
0227     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0228     {
0229         storage_type original, result;
0230         fence_before(order);
0231         __asm__ __volatile__
0232         (
0233             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0234             "lwarx %0,%y2\n\t"
0235             "sub %1,%0,%3\n\t"
0236             "stwcx. %1,%y2\n\t"
0237             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0238             : "=&b" (original), "=&b" (result), "+Z" (storage)
0239             : "b" (v)
0240             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0241         );
0242         fence_after(order);
0243         return original;
0244     }
0245
0246     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0247     {
0248         storage_type original, result;
0249         fence_before(order);
0250         __asm__ __volatile__
0251         (
0252             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0253             "lwarx %0,%y2\n\t"
0254             "and %1,%0,%3\n\t"
0255             "stwcx. %1,%y2\n\t"
0256             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0257             : "=&b" (original), "=&b" (result), "+Z" (storage)
0258             : "b" (v)
0259             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0260         );
0261         fence_after(order);
0262         return original;
0263     }
0264
0265     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0266     {
0267         storage_type original, result;
0268         fence_before(order);
0269         __asm__ __volatile__
0270         (
0271             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0272             "lwarx %0,%y2\n\t"
0273             "or %1,%0,%3\n\t"
0274             "stwcx. %1,%y2\n\t"
0275             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0276             : "=&b" (original), "=&b" (result), "+Z" (storage)
0277             : "b" (v)
0278             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0279         );
0280         fence_after(order);
0281         return original;
0282     }
0283
0284     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0285     {
0286         storage_type original, result;
0287         fence_before(order);
0288         __asm__ __volatile__
0289         (
0290             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0291             "lwarx %0,%y2\n\t"
0292             "xor %1,%0,%3\n\t"
0293             "stwcx. %1,%y2\n\t"
0294             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0295             : "=&b" (original), "=&b" (result), "+Z" (storage)
0296             : "b" (v)
0297             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0298         );
0299         fence_after(order);
0300         return original;
0301     }
0302
0303     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0304     {
0305         return !!exchange(storage, (storage_type)1, order);
0306     }
0307
0308     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0309     {
0310         store(storage, (storage_type)0, order);
0311     }
0312 };
0313
0314 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
0315
0316 template< bool Signed, bool Interprocess >
0317 struct core_arch_operations< 1u, Signed, Interprocess > :
0318     public core_arch_operations_gcc_ppc_base
0319 {
0320     typedef typename storage_traits< 1u >::type storage_type;
0321
0322     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
0323     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 1u;
0324     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0325     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0326
0327     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0328     {
0329         fence_before(order);
0330         __asm__ __volatile__
0331         (
0332             "stb %1, %0\n\t"
0333             : "+m" (storage)
0334             : "r" (v)
0335         );
0336     }
0337
0338     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0339     {
0340         storage_type v;
0341         if (order == memory_order_seq_cst)
0342             __asm__ __volatile__ ("sync" ::: "memory");
0343         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0344         {
0345             __asm__ __volatile__
0346             (
0347                 "lbz %0, %1\n\t"
0348                 "cmpw %0, %0\n\t"
0349                 BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4")
0350                 BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0351                 "isync\n\t"
0352                 : "=&r" (v)
0353                 : "m" (storage)
0354                 : "cr0", "memory"
0355             );
0356         }
0357         else
0358         {
0359             __asm__ __volatile__
0360             (
0361                 "lbz %0, %1\n\t"
0362                 : "=&r" (v)
0363                 : "m" (storage)
0364             );
0365         }
0366         return v;
0367     }
0368
0369     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0370     {
0371         storage_type original;
0372         fence_before(order);
0373         __asm__ __volatile__
0374         (
0375             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0376             "lbarx %0,%y1\n\t"
0377             "stbcx. %2,%y1\n\t"
0378             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8")
0379             : "=&b" (original), "+Z" (storage)
0380             : "b" (v)
0381             : "cr0"
0382         );
0383         fence_after(order);
0384         return original;
0385     }
0386
0387     static BOOST_FORCEINLINE bool compare_exchange_weak(
0388         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0389     {
0390         int success;
0391         fence_before(success_order);
0392         __asm__ __volatile__
0393         (
0394             "li %1, 0\n\t"
0395             "lbarx %0,%y2\n\t"
0396             "cmpw %0, %3\n\t"
0397             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
0398             "stbcx. %4,%y2\n\t"
0399             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8")
0400             "li %1, 1\n\t"
0401             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0402             : "=&b" (expected), "=&b" (success), "+Z" (storage)
0403             : "b" (expected), "b" (desired)
0404             : "cr0"
0405         );
0406         if (success)
0407             fence_after(success_order);
0408         else
0409             fence_after(failure_order);
0410         return !!success;
0411     }
0412
0413     static BOOST_FORCEINLINE bool compare_exchange_strong(
0414         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0415     {
0416         int success;
0417         fence_before(success_order);
0418         __asm__ __volatile__
0419         (
0420             "li %1, 0\n\t"
0421             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0")
0422             "lbarx %0,%y2\n\t"
0423             "cmpw %0, %3\n\t"
0424             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
0425             "stbcx. %4,%y2\n\t"
0426             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16")
0427             "li %1, 1\n\t"
0428             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0429             : "=&b" (expected), "=&b" (success), "+Z" (storage)
0430             : "b" (expected), "b" (desired)
0431             : "cr0"
0432         );
0433         if (success)
0434             fence_after(success_order);
0435         else
0436             fence_after(failure_order);
0437         return !!success;
0438     }
0439
0440     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0441     {
0442         storage_type original, result;
0443         fence_before(order);
0444         __asm__ __volatile__
0445         (
0446             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0447             "lbarx %0,%y2\n\t"
0448             "add %1,%0,%3\n\t"
0449             "stbcx. %1,%y2\n\t"
0450             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0451             : "=&b" (original), "=&b" (result), "+Z" (storage)
0452             : "b" (v)
0453             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0454         );
0455         fence_after(order);
0456         return original;
0457     }
0458
0459     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0460     {
0461         storage_type original, result;
0462         fence_before(order);
0463         __asm__ __volatile__
0464         (
0465             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0466             "lbarx %0,%y2\n\t"
0467             "sub %1,%0,%3\n\t"
0468             "stbcx. %1,%y2\n\t"
0469             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0470             : "=&b" (original), "=&b" (result), "+Z" (storage)
0471             : "b" (v)
0472             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0473         );
0474         fence_after(order);
0475         return original;
0476     }
0477
0478     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0479     {
0480         storage_type original, result;
0481         fence_before(order);
0482         __asm__ __volatile__
0483         (
0484             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0485             "lbarx %0,%y2\n\t"
0486             "and %1,%0,%3\n\t"
0487             "stbcx. %1,%y2\n\t"
0488             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0489             : "=&b" (original), "=&b" (result), "+Z" (storage)
0490             : "b" (v)
0491             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0492         );
0493         fence_after(order);
0494         return original;
0495     }
0496
0497     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0498     {
0499         storage_type original, result;
0500         fence_before(order);
0501         __asm__ __volatile__
0502         (
0503             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0504             "lbarx %0,%y2\n\t"
0505             "or %1,%0,%3\n\t"
0506             "stbcx. %1,%y2\n\t"
0507             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0508             : "=&b" (original), "=&b" (result), "+Z" (storage)
0509             : "b" (v)
0510             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0511         );
0512         fence_after(order);
0513         return original;
0514     }
0515
0516     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0517     {
0518         storage_type original, result;
0519         fence_before(order);
0520         __asm__ __volatile__
0521         (
0522             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0523             "lbarx %0,%y2\n\t"
0524             "xor %1,%0,%3\n\t"
0525             "stbcx. %1,%y2\n\t"
0526             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0527             : "=&b" (original), "=&b" (result), "+Z" (storage)
0528             : "b" (v)
0529             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0530         );
0531         fence_after(order);
0532         return original;
0533     }
0534
0535     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0536     {
0537         return !!exchange(storage, (storage_type)1, order);
0538     }
0539
0540     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0541     {
0542         store(storage, (storage_type)0, order);
0543     }
0544 };
0545
0546 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
0547
0548 template< bool Interprocess >
0549 struct core_arch_operations< 1u, false, Interprocess > :
0550     public core_arch_operations< 4u, false, Interprocess >
0551 {
0552     typedef core_arch_operations< 4u, false, Interprocess > base_type;
0553     typedef typename base_type::storage_type storage_type;
0554
0555     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0556     {
0557         storage_type original, result;
0558         base_type::fence_before(order);
0559         __asm__ __volatile__
0560         (
0561             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0562             "lwarx %0,%y2\n\t"
0563             "add %1,%0,%3\n\t"
0564             "rlwinm %1, %1, 0, 0xff\n\t"
0565             "stwcx. %1,%y2\n\t"
0566             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0567             : "=&b" (original), "=&b" (result), "+Z" (storage)
0568             : "b" (v)
0569             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0570         );
0571         base_type::fence_after(order);
0572         return original;
0573     }
0574
0575     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0576     {
0577         storage_type original, result;
0578         base_type::fence_before(order);
0579         __asm__ __volatile__
0580         (
0581             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0582             "lwarx %0,%y2\n\t"
0583             "sub %1,%0,%3\n\t"
0584             "rlwinm %1, %1, 0, 0xff\n\t"
0585             "stwcx. %1,%y2\n\t"
0586             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0587             : "=&b" (original), "=&b" (result), "+Z" (storage)
0588             : "b" (v)
0589             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0590         );
0591         base_type::fence_after(order);
0592         return original;
0593     }
0594 };
0595
0596 template< bool Interprocess >
0597 struct core_arch_operations< 1u, true, Interprocess > :
0598     public core_arch_operations< 4u, true, Interprocess >
0599 {
0600     typedef core_arch_operations< 4u, true, Interprocess > base_type;
0601     typedef typename base_type::storage_type storage_type;
0602
0603     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0604     {
0605         storage_type original, result;
0606         base_type::fence_before(order);
0607         __asm__ __volatile__
0608         (
0609             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0610             "lwarx %0,%y2\n\t"
0611             "add %1,%0,%3\n\t"
0612             "extsb %1, %1\n\t"
0613             "stwcx. %1,%y2\n\t"
0614             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0615             : "=&b" (original), "=&b" (result), "+Z" (storage)
0616             : "b" (v)
0617             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0618         );
0619         base_type::fence_after(order);
0620         return original;
0621     }
0622
0623     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0624     {
0625         storage_type original, result;
0626         base_type::fence_before(order);
0627         __asm__ __volatile__
0628         (
0629             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0630             "lwarx %0,%y2\n\t"
0631             "sub %1,%0,%3\n\t"
0632             "extsb %1, %1\n\t"
0633             "stwcx. %1,%y2\n\t"
0634             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0635             : "=&b" (original), "=&b" (result), "+Z" (storage)
0636             : "b" (v)
0637             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0638         );
0639         base_type::fence_after(order);
0640         return original;
0641     }
0642 };
0643
0644 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
0645
0646 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
0647
0648 template< bool Signed, bool Interprocess >
0649 struct core_arch_operations< 2u, Signed, Interprocess > :
0650     public core_arch_operations_gcc_ppc_base
0651 {
0652     typedef typename storage_traits< 2u >::type storage_type;
0653
0654     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
0655     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 2u;
0656     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0657     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0658
0659     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0660     {
0661         fence_before(order);
0662         __asm__ __volatile__
0663         (
0664             "sth %1, %0\n\t"
0665             : "+m" (storage)
0666             : "r" (v)
0667         );
0668     }
0669
0670     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0671     {
0672         storage_type v;
0673         if (order == memory_order_seq_cst)
0674             __asm__ __volatile__ ("sync" ::: "memory");
0675         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0676         {
0677             __asm__ __volatile__
0678             (
0679                 "lhz %0, %1\n\t"
0680                 "cmpw %0, %0\n\t"
0681                 BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4")
0682                 BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0683                 "isync\n\t"
0684                 : "=&r" (v)
0685                 : "m" (storage)
0686                 : "cr0", "memory"
0687             );
0688         }
0689         else
0690         {
0691             __asm__ __volatile__
0692             (
0693                 "lhz %0, %1\n\t"
0694                 : "=&r" (v)
0695                 : "m" (storage)
0696             );
0697         }
0698         return v;
0699     }
0700
0701     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0702     {
0703         storage_type original;
0704         fence_before(order);
0705         __asm__ __volatile__
0706         (
0707             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0708             "lharx %0,%y1\n\t"
0709             "sthcx. %2,%y1\n\t"
0710             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8")
0711             : "=&b" (original), "+Z" (storage)
0712             : "b" (v)
0713             : "cr0"
0714         );
0715         fence_after(order);
0716         return original;
0717     }
0718
0719     static BOOST_FORCEINLINE bool compare_exchange_weak(
0720         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0721     {
0722         int success;
0723         fence_before(success_order);
0724         __asm__ __volatile__
0725         (
0726             "li %1, 0\n\t"
0727             "lharx %0,%y2\n\t"
0728             "cmpw %0, %3\n\t"
0729             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
0730             "sthcx. %4,%y2\n\t"
0731             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8")
0732             "li %1, 1\n\t"
0733             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0734             : "=&b" (expected), "=&b" (success), "+Z" (storage)
0735             : "b" (expected), "b" (desired)
0736             : "cr0"
0737         );
0738         if (success)
0739             fence_after(success_order);
0740         else
0741             fence_after(failure_order);
0742         return !!success;
0743     }
0744
0745     static BOOST_FORCEINLINE bool compare_exchange_strong(
0746         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0747     {
0748         int success;
0749         fence_before(success_order);
0750         __asm__ __volatile__
0751         (
0752             "li %1, 0\n\t"
0753             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0")
0754             "lharx %0,%y2\n\t"
0755             "cmpw %0, %3\n\t"
0756             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
0757             "sthcx. %4,%y2\n\t"
0758             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16")
0759             "li %1, 1\n\t"
0760             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0761             : "=&b" (expected), "=&b" (success), "+Z" (storage)
0762             : "b" (expected), "b" (desired)
0763             : "cr0"
0764         );
0765         if (success)
0766             fence_after(success_order);
0767         else
0768             fence_after(failure_order);
0769         return !!success;
0770     }
0771
0772     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0773     {
0774         storage_type original, result;
0775         fence_before(order);
0776         __asm__ __volatile__
0777         (
0778             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0779             "lharx %0,%y2\n\t"
0780             "add %1,%0,%3\n\t"
0781             "sthcx. %1,%y2\n\t"
0782             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0783             : "=&b" (original), "=&b" (result), "+Z" (storage)
0784             : "b" (v)
0785             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0786         );
0787         fence_after(order);
0788         return original;
0789     }
0790
0791     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0792     {
0793         storage_type original, result;
0794         fence_before(order);
0795         __asm__ __volatile__
0796         (
0797             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0798             "lharx %0,%y2\n\t"
0799             "sub %1,%0,%3\n\t"
0800             "sthcx. %1,%y2\n\t"
0801             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0802             : "=&b" (original), "=&b" (result), "+Z" (storage)
0803             : "b" (v)
0804             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0805         );
0806         fence_after(order);
0807         return original;
0808     }
0809
0810     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0811     {
0812         storage_type original, result;
0813         fence_before(order);
0814         __asm__ __volatile__
0815         (
0816             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0817             "lharx %0,%y2\n\t"
0818             "and %1,%0,%3\n\t"
0819             "sthcx. %1,%y2\n\t"
0820             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0821             : "=&b" (original), "=&b" (result), "+Z" (storage)
0822             : "b" (v)
0823             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0824         );
0825         fence_after(order);
0826         return original;
0827     }
0828
0829     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0830     {
0831         storage_type original, result;
0832         fence_before(order);
0833         __asm__ __volatile__
0834         (
0835             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0836             "lharx %0,%y2\n\t"
0837             "or %1,%0,%3\n\t"
0838             "sthcx. %1,%y2\n\t"
0839             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0840             : "=&b" (original), "=&b" (result), "+Z" (storage)
0841             : "b" (v)
0842             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0843         );
0844         fence_after(order);
0845         return original;
0846     }
0847
0848     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0849     {
0850         storage_type original, result;
0851         fence_before(order);
0852         __asm__ __volatile__
0853         (
0854             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0855             "lharx %0,%y2\n\t"
0856             "xor %1,%0,%3\n\t"
0857             "sthcx. %1,%y2\n\t"
0858             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
0859             : "=&b" (original), "=&b" (result), "+Z" (storage)
0860             : "b" (v)
0861             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0862         );
0863         fence_after(order);
0864         return original;
0865     }
0866
0867     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0868     {
0869         return !!exchange(storage, (storage_type)1, order);
0870     }
0871
0872     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0873     {
0874         store(storage, (storage_type)0, order);
0875     }
0876 };
0877
0878 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
0879
0880 template< bool Interprocess >
0881 struct core_arch_operations< 2u, false, Interprocess > :
0882     public core_arch_operations< 4u, false, Interprocess >
0883 {
0884     typedef core_arch_operations< 4u, false, Interprocess > base_type;
0885     typedef typename base_type::storage_type storage_type;
0886
0887     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0888     {
0889         storage_type original, result;
0890         base_type::fence_before(order);
0891         __asm__ __volatile__
0892         (
0893             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0894             "lwarx %0,%y2\n\t"
0895             "add %1,%0,%3\n\t"
0896             "rlwinm %1, %1, 0, 0xffff\n\t"
0897             "stwcx. %1,%y2\n\t"
0898             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0899             : "=&b" (original), "=&b" (result), "+Z" (storage)
0900             : "b" (v)
0901             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0902         );
0903         base_type::fence_after(order);
0904         return original;
0905     }
0906
0907     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0908     {
0909         storage_type original, result;
0910         base_type::fence_before(order);
0911         __asm__ __volatile__
0912         (
0913             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0914             "lwarx %0,%y2\n\t"
0915             "sub %1,%0,%3\n\t"
0916             "rlwinm %1, %1, 0, 0xffff\n\t"
0917             "stwcx. %1,%y2\n\t"
0918             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0919             : "=&b" (original), "=&b" (result), "+Z" (storage)
0920             : "b" (v)
0921             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0922         );
0923         base_type::fence_after(order);
0924         return original;
0925     }
0926 };
0927
0928 template< bool Interprocess >
0929 struct core_arch_operations< 2u, true, Interprocess > :
0930     public core_arch_operations< 4u, true, Interprocess >
0931 {
0932     typedef core_arch_operations< 4u, true, Interprocess > base_type;
0933     typedef typename base_type::storage_type storage_type;
0934
0935     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0936     {
0937         storage_type original, result;
0938         base_type::fence_before(order);
0939         __asm__ __volatile__
0940         (
0941             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0942             "lwarx %0,%y2\n\t"
0943             "add %1,%0,%3\n\t"
0944             "extsh %1, %1\n\t"
0945             "stwcx. %1,%y2\n\t"
0946             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0947             : "=&b" (original), "=&b" (result), "+Z" (storage)
0948             : "b" (v)
0949             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0950         );
0951         base_type::fence_after(order);
0952         return original;
0953     }
0954
0955     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0956     {
0957         storage_type original, result;
0958         base_type::fence_before(order);
0959         __asm__ __volatile__
0960         (
0961             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
0962             "lwarx %0,%y2\n\t"
0963             "sub %1,%0,%3\n\t"
0964             "extsh %1, %1\n\t"
0965             "stwcx. %1,%y2\n\t"
0966             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16")
0967             : "=&b" (original), "=&b" (result), "+Z" (storage)
0968             : "b" (v)
0969             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0970         );
0971         base_type::fence_after(order);
0972         return original;
0973     }
0974 };
0975
0976 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
0977
0978 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX)
0979
0980 template< bool Signed, bool Interprocess >
0981 struct core_arch_operations< 8u, Signed, Interprocess > :
0982     public core_arch_operations_gcc_ppc_base
0983 {
0984     typedef typename storage_traits< 8u >::type storage_type;
0985
0986     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
0987     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u;
0988     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0989     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0990
0991     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0992     {
0993         fence_before(order);
0994         __asm__ __volatile__
0995         (
0996             "std %1, %0\n\t"
0997             : "+m" (storage)
0998             : "r" (v)
0999         );
1000     }
1001
1002     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
1003     {
1004         storage_type v;
1005         if (order == memory_order_seq_cst)
1006             __asm__ __volatile__ ("sync" ::: "memory");
1007         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
1008         {
1009             __asm__ __volatile__
1010             (
1011                 "ld %0, %1\n\t"
1012                 "cmpd %0, %0\n\t"
1013                 BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4")
1014                 BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1015                 "isync\n\t"
1016                 : "=&b" (v)
1017                 : "m" (storage)
1018                 : "cr0", "memory"
1019             );
1020         }
1021         else
1022         {
1023             __asm__ __volatile__
1024             (
1025                 "ld %0, %1\n\t"
1026                 : "=&b" (v)
1027                 : "m" (storage)
1028             );
1029         }
1030         return v;
1031     }
1032
1033     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1034     {
1035         storage_type original;
1036         fence_before(order);
1037         __asm__ __volatile__
1038         (
1039             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1040             "ldarx %0,%y1\n\t"
1041             "stdcx. %2,%y1\n\t"
1042             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8")
1043             : "=&b" (original), "+Z" (storage)
1044             : "b" (v)
1045             : "cr0"
1046         );
1047         fence_after(order);
1048         return original;
1049     }
1050
1051     static BOOST_FORCEINLINE bool compare_exchange_weak(
1052         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1053     {
1054         int success;
1055         fence_before(success_order);
1056         __asm__ __volatile__
1057         (
1058             "li %1, 0\n\t"
1059             "ldarx %0,%y2\n\t"
1060             "cmpd %0, %3\n\t"
1061             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
1062             "stdcx. %4,%y2\n\t"
1063             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8")
1064             "li %1, 1\n\t"
1065             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1066             : "=&b" (expected), "=&b" (success), "+Z" (storage)
1067             : "b" (expected), "b" (desired)
1068             : "cr0"
1069         );
1070         if (success)
1071             fence_after(success_order);
1072         else
1073             fence_after(failure_order);
1074         return !!success;
1075     }
1076
1077     static BOOST_FORCEINLINE bool compare_exchange_strong(
1078         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1079     {
1080         int success;
1081         fence_before(success_order);
1082         __asm__ __volatile__
1083         (
1084             "li %1, 0\n\t"
1085             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0")
1086             "ldarx %0,%y2\n\t"
1087             "cmpd %0, %3\n\t"
1088             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16")
1089             "stdcx. %4,%y2\n\t"
1090             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16")
1091             "li %1, 1\n\t"
1092             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1093             : "=&b" (expected), "=&b" (success), "+Z" (storage)
1094             : "b" (expected), "b" (desired)
1095             : "cr0"
1096         );
1097         if (success)
1098             fence_after(success_order);
1099         else
1100             fence_after(failure_order);
1101         return !!success;
1102     }
1103
1104     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1105     {
1106         storage_type original, result;
1107         fence_before(order);
1108         __asm__ __volatile__
1109         (
1110             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1111             "ldarx %0,%y2\n\t"
1112             "add %1,%0,%3\n\t"
1113             "stdcx. %1,%y2\n\t"
1114             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
1115             : "=&b" (original), "=&b" (result), "+Z" (storage)
1116             : "b" (v)
1117             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1118         );
1119         fence_after(order);
1120         return original;
1121     }
1122
1123     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1124     {
1125         storage_type original, result;
1126         fence_before(order);
1127         __asm__ __volatile__
1128         (
1129             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1130             "ldarx %0,%y2\n\t"
1131             "sub %1,%0,%3\n\t"
1132             "stdcx. %1,%y2\n\t"
1133             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
1134             : "=&b" (original), "=&b" (result), "+Z" (storage)
1135             : "b" (v)
1136             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1137         );
1138         fence_after(order);
1139         return original;
1140     }
1141
1142     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1143     {
1144         storage_type original, result;
1145         fence_before(order);
1146         __asm__ __volatile__
1147         (
1148             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1149             "ldarx %0,%y2\n\t"
1150             "and %1,%0,%3\n\t"
1151             "stdcx. %1,%y2\n\t"
1152             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
1153             : "=&b" (original), "=&b" (result), "+Z" (storage)
1154             : "b" (v)
1155             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1156         );
1157         fence_after(order);
1158         return original;
1159     }
1160
1161     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1162     {
1163         storage_type original, result;
1164         fence_before(order);
1165         __asm__ __volatile__
1166         (
1167             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1168             "ldarx %0,%y2\n\t"
1169             "or %1,%0,%3\n\t"
1170             "stdcx. %1,%y2\n\t"
1171             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
1172             : "=&b" (original), "=&b" (result), "+Z" (storage)
1173             : "b" (v)
1174             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1175         );
1176         fence_after(order);
1177         return original;
1178     }
1179
1180     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1181     {
1182         storage_type original, result;
1183         fence_before(order);
1184         __asm__ __volatile__
1185         (
1186             BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1")
1187             "ldarx %0,%y2\n\t"
1188             "xor %1,%0,%3\n\t"
1189             "stdcx. %1,%y2\n\t"
1190             BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12")
1191             : "=&b" (original), "=&b" (result), "+Z" (storage)
1192             : "b" (v)
1193             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1194         );
1195         fence_after(order);
1196         return original;
1197     }
1198
1199     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1200     {
1201         return !!exchange(storage, (storage_type)1, order);
1202     }
1203
1204     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1205     {
1206         store(storage, (storage_type)0, order);
1207     }
1208 };
1209
1210 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX)
1211
1212 } // namespace detail
1213 } // namespace atomics
1214 } // namespace boost
1215
1216 #include <boost/atomic/detail/footer.hpp>
1217
1218 #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_