atomic/detail/core_arch_ops_gcc_alpha.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2009 Helge Bahmann
0007  * Copyright (c) 2013 Tim Blechmann
0008  * Copyright (c) 2014 Andrey Semashev
0009  */
0010 /*!
0011  * \file   atomic/detail/core_arch_ops_gcc_alpha.hpp
0012  *
0013  * This header contains implementation of the \c core_arch_operations template.
0014  */
0015
0016 #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ALPHA_HPP_INCLUDED_
0017 #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ALPHA_HPP_INCLUDED_
0018
0019 #include <cstddef>
0020 #include <boost/memory_order.hpp>
0021 #include <boost/atomic/detail/config.hpp>
0022 #include <boost/atomic/detail/storage_traits.hpp>
0023 #include <boost/atomic/detail/core_arch_operations_fwd.hpp>
0024 #include <boost/atomic/detail/header.hpp>
0025
0026 #ifdef BOOST_HAS_PRAGMA_ONCE
0027 #pragma once
0028 #endif
0029
0030 namespace boost {
0031 namespace atomics {
0032 namespace detail {
0033
0034 /*
0035   Refer to http://h71000.www7.hp.com/doc/82final/5601/5601pro_004.html
0036   (HP OpenVMS systems documentation) and the Alpha Architecture Reference Manual.
0037  */
0038
0039 /*
0040     NB: The most natural thing would be to write the increment/decrement
0041     operators along the following lines:
0042
0043     __asm__ __volatile__
0044     (
0045         "1: ldl_l %0,%1 \n"
0046         "addl %0,1,%0 \n"
0047         "stl_c %0,%1 \n"
0048         "beq %0,1b\n"
0049         : "=&b" (tmp)
0050         : "m" (value)
0051         : "cc"
0052     );
0053
0054     However according to the comments on the HP website and matching
0055     comments in the Linux kernel sources this defies branch prediction,
0056     as the cpu assumes that backward branches are always taken; so
0057     instead copy the trick from the Linux kernel, introduce a forward
0058     branch and back again.
0059
0060     I have, however, had a hard time measuring the difference between
0061     the two versions in microbenchmarks -- I am leaving it in nevertheless
0062     as it apparently does not hurt either.
0063 */
0064
0065 struct core_arch_operations_gcc_alpha_base
0066 {
0067     static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = false;
0068     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
0069
0070     static BOOST_FORCEINLINE void fence_before(memory_order order) BOOST_NOEXCEPT
0071     {
0072         if ((static_cast< unsigned int >(order) & static_cast< unsigned int >(memory_order_release)) != 0u)
0073             __asm__ __volatile__ ("mb" ::: "memory");
0074     }
0075
0076     static BOOST_FORCEINLINE void fence_after(memory_order order) BOOST_NOEXCEPT
0077     {
0078         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0079             __asm__ __volatile__ ("mb" ::: "memory");
0080     }
0081
0082     static BOOST_FORCEINLINE void fence_after_store(memory_order order) BOOST_NOEXCEPT
0083     {
0084         if (order == memory_order_seq_cst)
0085             __asm__ __volatile__ ("mb" ::: "memory");
0086     }
0087 };
0088
0089
0090 template< bool Signed, bool Interprocess >
0091 struct core_arch_operations< 4u, Signed, Interprocess > :
0092     public core_arch_operations_gcc_alpha_base
0093 {
0094     typedef typename storage_traits< 4u >::type storage_type;
0095
0096     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
0097     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u;
0098     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0099     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0100
0101     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0102     {
0103         fence_before(order);
0104         storage = v;
0105         fence_after_store(order);
0106     }
0107
0108     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0109     {
0110         storage_type v = storage;
0111         fence_after(order);
0112         return v;
0113     }
0114
0115     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0116     {
0117         storage_type original, tmp;
0118         fence_before(order);
0119         __asm__ __volatile__
0120         (
0121             "1:\n\t"
0122             "mov %3, %1\n\t"
0123             "ldl_l %0, %2\n\t"
0124             "stl_c %1, %2\n\t"
0125             "beq %1, 2f\n\t"
0126
0127             ".subsection 2\n\t"
0128             "2: br 1b\n\t"
0129             ".previous\n\t"
0130
0131             : "=&r" (original),  // %0
0132               "=&r" (tmp)        // %1
0133             : "m" (storage),     // %2
0134               "r" (v)            // %3
0135             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0136         );
0137         fence_after(order);
0138         return original;
0139     }
0140
0141     static BOOST_FORCEINLINE bool compare_exchange_weak(
0142         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0143     {
0144         fence_before(success_order);
0145         int success;
0146         storage_type current;
0147         __asm__ __volatile__
0148         (
0149             "1:\n\t"
0150             "ldl_l %2, %4\n\t"                // current = *(&storage)
0151             "cmpeq %2, %0, %3\n\t"            // success = current == expected
0152             "mov %2, %0\n\t"                  // expected = current
0153             "beq %3, 2f\n\t"                  // if (success == 0) goto end
0154             "stl_c %1, %4\n\t"                // storage = desired; desired = store succeeded
0155             "mov %1, %3\n\t"                  // success = desired
0156             "2:\n\t"
0157             : "+r" (expected),   // %0
0158               "+r" (desired),    // %1
0159               "=&r" (current),   // %2
0160               "=&r" (success)    // %3
0161             : "m" (storage)      // %4
0162             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0163         );
0164         if (success)
0165             fence_after(success_order);
0166         else
0167             fence_after(failure_order);
0168         return !!success;
0169     }
0170
0171     static BOOST_FORCEINLINE bool compare_exchange_strong(
0172         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0173     {
0174         int success;
0175         storage_type current, tmp;
0176         fence_before(success_order);
0177         __asm__ __volatile__
0178         (
0179             "1:\n\t"
0180             "mov %5, %1\n\t"                  // tmp = desired
0181             "ldl_l %2, %4\n\t"                // current = *(&storage)
0182             "cmpeq %2, %0, %3\n\t"            // success = current == expected
0183             "mov %2, %0\n\t"                  // expected = current
0184             "beq %3, 2f\n\t"                  // if (success == 0) goto end
0185             "stl_c %1, %4\n\t"                // storage = tmp; tmp = store succeeded
0186             "beq %1, 3f\n\t"                  // if (tmp == 0) goto retry
0187             "mov %1, %3\n\t"                  // success = tmp
0188             "2:\n\t"
0189
0190             ".subsection 2\n\t"
0191             "3: br 1b\n\t"
0192             ".previous\n\t"
0193
0194             : "+r" (expected),   // %0
0195               "=&r" (tmp),       // %1
0196               "=&r" (current),   // %2
0197               "=&r" (success)    // %3
0198             : "m" (storage),     // %4
0199               "r" (desired)      // %5
0200             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0201         );
0202         if (success)
0203             fence_after(success_order);
0204         else
0205             fence_after(failure_order);
0206         return !!success;
0207     }
0208
0209     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0210     {
0211         storage_type original, modified;
0212         fence_before(order);
0213         __asm__ __volatile__
0214         (
0215             "1:\n\t"
0216             "ldl_l %0, %2\n\t"
0217             "addl %0, %3, %1\n\t"
0218             "stl_c %1, %2\n\t"
0219             "beq %1, 2f\n\t"
0220
0221             ".subsection 2\n\t"
0222             "2: br 1b\n\t"
0223             ".previous\n\t"
0224
0225             : "=&r" (original),  // %0
0226               "=&r" (modified)   // %1
0227             : "m" (storage),     // %2
0228               "r" (v)            // %3
0229             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0230         );
0231         fence_after(order);
0232         return original;
0233     }
0234
0235     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0236     {
0237         storage_type original, modified;
0238         fence_before(order);
0239         __asm__ __volatile__
0240         (
0241             "1:\n\t"
0242             "ldl_l %0, %2\n\t"
0243             "subl %0, %3, %1\n\t"
0244             "stl_c %1, %2\n\t"
0245             "beq %1, 2f\n\t"
0246
0247             ".subsection 2\n\t"
0248             "2: br 1b\n\t"
0249             ".previous\n\t"
0250
0251             : "=&r" (original),  // %0
0252               "=&r" (modified)   // %1
0253             : "m" (storage),     // %2
0254               "r" (v)            // %3
0255             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0256         );
0257         fence_after(order);
0258         return original;
0259     }
0260
0261     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0262     {
0263         storage_type original, modified;
0264         fence_before(order);
0265         __asm__ __volatile__
0266         (
0267             "1:\n\t"
0268             "ldl_l %0, %2\n\t"
0269             "and %0, %3, %1\n\t"
0270             "stl_c %1, %2\n\t"
0271             "beq %1, 2f\n\t"
0272
0273             ".subsection 2\n\t"
0274             "2: br 1b\n\t"
0275             ".previous\n\t"
0276
0277             : "=&r" (original),  // %0
0278               "=&r" (modified)   // %1
0279             : "m" (storage),     // %2
0280               "r" (v)            // %3
0281             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0282         );
0283         fence_after(order);
0284         return original;
0285     }
0286
0287     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0288     {
0289         storage_type original, modified;
0290         fence_before(order);
0291         __asm__ __volatile__
0292         (
0293             "1:\n\t"
0294             "ldl_l %0, %2\n\t"
0295             "bis %0, %3, %1\n\t"
0296             "stl_c %1, %2\n\t"
0297             "beq %1, 2f\n\t"
0298
0299             ".subsection 2\n\t"
0300             "2: br 1b\n\t"
0301             ".previous\n\t"
0302
0303             : "=&r" (original),  // %0
0304               "=&r" (modified)   // %1
0305             : "m" (storage),     // %2
0306               "r" (v)            // %3
0307             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0308         );
0309         fence_after(order);
0310         return original;
0311     }
0312
0313     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0314     {
0315         storage_type original, modified;
0316         fence_before(order);
0317         __asm__ __volatile__
0318         (
0319             "1:\n\t"
0320             "ldl_l %0, %2\n\t"
0321             "xor %0, %3, %1\n\t"
0322             "stl_c %1, %2\n\t"
0323             "beq %1, 2f\n\t"
0324
0325             ".subsection 2\n\t"
0326             "2: br 1b\n\t"
0327             ".previous\n\t"
0328
0329             : "=&r" (original),  // %0
0330               "=&r" (modified)   // %1
0331             : "m" (storage),     // %2
0332               "r" (v)            // %3
0333             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0334         );
0335         fence_after(order);
0336         return original;
0337     }
0338
0339     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0340     {
0341         return !!exchange(storage, (storage_type)1, order);
0342     }
0343
0344     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0345     {
0346         store(storage, 0, order);
0347     }
0348 };
0349
0350
0351 template< bool Interprocess >
0352 struct core_arch_operations< 1u, false, Interprocess > :
0353     public core_arch_operations< 4u, false, Interprocess >
0354 {
0355     typedef core_arch_operations< 4u, false, Interprocess > base_type;
0356     typedef typename base_type::storage_type storage_type;
0357
0358     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0359     {
0360         storage_type original, modified;
0361         base_type::fence_before(order);
0362         __asm__ __volatile__
0363         (
0364             "1:\n\t"
0365             "ldl_l %0, %2\n\t"
0366             "addl %0, %3, %1\n\t"
0367             "zapnot %1, 1, %1\n\t"
0368             "stl_c %1, %2\n\t"
0369             "beq %1, 2f\n\t"
0370
0371             ".subsection 2\n\t"
0372             "2: br 1b\n\t"
0373             ".previous\n\t"
0374
0375             : "=&r" (original),  // %0
0376               "=&r" (modified)   // %1
0377             : "m" (storage),     // %2
0378               "r" (v)            // %3
0379             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0380         );
0381         base_type::fence_after(order);
0382         return original;
0383     }
0384
0385     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0386     {
0387         storage_type original, modified;
0388         base_type::fence_before(order);
0389         __asm__ __volatile__
0390         (
0391             "1:\n\t"
0392             "ldl_l %0, %2\n\t"
0393             "subl %0, %3, %1\n\t"
0394             "zapnot %1, 1, %1\n\t"
0395             "stl_c %1, %2\n\t"
0396             "beq %1, 2f\n\t"
0397
0398             ".subsection 2\n\t"
0399             "2: br 1b\n\t"
0400             ".previous\n\t"
0401
0402             : "=&r" (original),  // %0
0403               "=&r" (modified)   // %1
0404             : "m" (storage),     // %2
0405               "r" (v)            // %3
0406             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0407         );
0408         base_type::fence_after(order);
0409         return original;
0410     }
0411 };
0412
0413 template< bool Interprocess >
0414 struct core_arch_operations< 1u, true, Interprocess > :
0415     public core_arch_operations< 4u, true, Interprocess >
0416 {
0417     typedef core_arch_operations< 4u, true, Interprocess > base_type;
0418     typedef typename base_type::storage_type storage_type;
0419
0420     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0421     {
0422         storage_type original, modified;
0423         base_type::fence_before(order);
0424         __asm__ __volatile__
0425         (
0426             "1:\n\t"
0427             "ldl_l %0, %2\n\t"
0428             "addl %0, %3, %1\n\t"
0429             "sextb %1, %1\n\t"
0430             "stl_c %1, %2\n\t"
0431             "beq %1, 2f\n\t"
0432
0433             ".subsection 2\n\t"
0434             "2: br 1b\n\t"
0435             ".previous\n\t"
0436
0437             : "=&r" (original),  // %0
0438               "=&r" (modified)   // %1
0439             : "m" (storage),     // %2
0440               "r" (v)            // %3
0441             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0442         );
0443         base_type::fence_after(order);
0444         return original;
0445     }
0446
0447     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0448     {
0449         storage_type original, modified;
0450         base_type::fence_before(order);
0451         __asm__ __volatile__
0452         (
0453             "1:\n\t"
0454             "ldl_l %0, %2\n\t"
0455             "subl %0, %3, %1\n\t"
0456             "sextb %1, %1\n\t"
0457             "stl_c %1, %2\n\t"
0458             "beq %1, 2f\n\t"
0459
0460             ".subsection 2\n\t"
0461             "2: br 1b\n\t"
0462             ".previous\n\t"
0463
0464             : "=&r" (original),  // %0
0465               "=&r" (modified)   // %1
0466             : "m" (storage),     // %2
0467               "r" (v)            // %3
0468             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0469         );
0470         base_type::fence_after(order);
0471         return original;
0472     }
0473 };
0474
0475
0476 template< bool Interprocess >
0477 struct core_arch_operations< 2u, false, Interprocess > :
0478     public core_arch_operations< 4u, false, Interprocess >
0479 {
0480     typedef core_arch_operations< 4u, false, Interprocess > base_type;
0481     typedef typename base_type::storage_type storage_type;
0482
0483     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0484     {
0485         storage_type original, modified;
0486         base_type::fence_before(order);
0487         __asm__ __volatile__
0488         (
0489             "1:\n\t"
0490             "ldl_l %0, %2\n\t"
0491             "addl %0, %3, %1\n\t"
0492             "zapnot %1, 3, %1\n\t"
0493             "stl_c %1, %2\n\t"
0494             "beq %1, 2f\n\t"
0495
0496             ".subsection 2\n\t"
0497             "2: br 1b\n\t"
0498             ".previous\n\t"
0499
0500             : "=&r" (original),  // %0
0501               "=&r" (modified)   // %1
0502             : "m" (storage),     // %2
0503               "r" (v)            // %3
0504             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0505         );
0506         base_type::fence_after(order);
0507         return original;
0508     }
0509
0510     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0511     {
0512         storage_type original, modified;
0513         base_type::fence_before(order);
0514         __asm__ __volatile__
0515         (
0516             "1:\n\t"
0517             "ldl_l %0, %2\n\t"
0518             "subl %0, %3, %1\n\t"
0519             "zapnot %1, 3, %1\n\t"
0520             "stl_c %1, %2\n\t"
0521             "beq %1, 2f\n\t"
0522
0523             ".subsection 2\n\t"
0524             "2: br 1b\n\t"
0525             ".previous\n\t"
0526
0527             : "=&r" (original),  // %0
0528               "=&r" (modified)   // %1
0529             : "m" (storage),     // %2
0530               "r" (v)            // %3
0531             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0532         );
0533         base_type::fence_after(order);
0534         return original;
0535     }
0536 };
0537
0538 template< bool Interprocess >
0539 struct core_arch_operations< 2u, true, Interprocess > :
0540     public core_arch_operations< 4u, true, Interprocess >
0541 {
0542     typedef core_arch_operations< 4u, true, Interprocess > base_type;
0543     typedef typename base_type::storage_type storage_type;
0544
0545     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0546     {
0547         storage_type original, modified;
0548         base_type::fence_before(order);
0549         __asm__ __volatile__
0550         (
0551             "1:\n\t"
0552             "ldl_l %0, %2\n\t"
0553             "addl %0, %3, %1\n\t"
0554             "sextw %1, %1\n\t"
0555             "stl_c %1, %2\n\t"
0556             "beq %1, 2f\n\t"
0557
0558             ".subsection 2\n\t"
0559             "2: br 1b\n\t"
0560             ".previous\n\t"
0561
0562             : "=&r" (original),  // %0
0563               "=&r" (modified)   // %1
0564             : "m" (storage),     // %2
0565               "r" (v)            // %3
0566             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0567         );
0568         base_type::fence_after(order);
0569         return original;
0570     }
0571
0572     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0573     {
0574         storage_type original, modified;
0575         base_type::fence_before(order);
0576         __asm__ __volatile__
0577         (
0578             "1:\n\t"
0579             "ldl_l %0, %2\n\t"
0580             "subl %0, %3, %1\n\t"
0581             "sextw %1, %1\n\t"
0582             "stl_c %1, %2\n\t"
0583             "beq %1, 2f\n\t"
0584
0585             ".subsection 2\n\t"
0586             "2: br 1b\n\t"
0587             ".previous\n\t"
0588
0589             : "=&r" (original),  // %0
0590               "=&r" (modified)   // %1
0591             : "m" (storage),     // %2
0592               "r" (v)            // %3
0593             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0594         );
0595         base_type::fence_after(order);
0596         return original;
0597     }
0598 };
0599
0600
0601 template< bool Signed, bool Interprocess >
0602 struct core_arch_operations< 8u, Signed, Interprocess > :
0603     public core_arch_operations_gcc_alpha_base
0604 {
0605     typedef typename storage_traits< 8u >::type storage_type;
0606
0607     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
0608     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u;
0609     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0610     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0611
0612     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0613     {
0614         fence_before(order);
0615         storage = v;
0616         fence_after_store(order);
0617     }
0618
0619     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0620     {
0621         storage_type v = storage;
0622         fence_after(order);
0623         return v;
0624     }
0625
0626     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0627     {
0628         storage_type original, tmp;
0629         fence_before(order);
0630         __asm__ __volatile__
0631         (
0632             "1:\n\t"
0633             "mov %3, %1\n\t"
0634             "ldq_l %0, %2\n\t"
0635             "stq_c %1, %2\n\t"
0636             "beq %1, 2f\n\t"
0637
0638             ".subsection 2\n\t"
0639             "2: br 1b\n\t"
0640             ".previous\n\t"
0641
0642             : "=&r" (original),  // %0
0643               "=&r" (tmp)        // %1
0644             : "m" (storage),     // %2
0645               "r" (v)            // %3
0646             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0647         );
0648         fence_after(order);
0649         return original;
0650     }
0651
0652     static BOOST_FORCEINLINE bool compare_exchange_weak(
0653         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0654     {
0655         fence_before(success_order);
0656         int success;
0657         storage_type current;
0658         __asm__ __volatile__
0659         (
0660             "1:\n\t"
0661             "ldq_l %2, %4\n\t"                // current = *(&storage)
0662             "cmpeq %2, %0, %3\n\t"            // success = current == expected
0663             "mov %2, %0\n\t"                  // expected = current
0664             "beq %3, 2f\n\t"                  // if (success == 0) goto end
0665             "stq_c %1, %4\n\t"                // storage = desired; desired = store succeeded
0666             "mov %1, %3\n\t"                  // success = desired
0667             "2:\n\t"
0668             : "+r" (expected),   // %0
0669               "+r" (desired),    // %1
0670               "=&r" (current),   // %2
0671               "=&r" (success)    // %3
0672             : "m" (storage)      // %4
0673             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0674         );
0675         if (success)
0676             fence_after(success_order);
0677         else
0678             fence_after(failure_order);
0679         return !!success;
0680     }
0681
0682     static BOOST_FORCEINLINE bool compare_exchange_strong(
0683         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0684     {
0685         int success;
0686         storage_type current, tmp;
0687         fence_before(success_order);
0688         __asm__ __volatile__
0689         (
0690             "1:\n\t"
0691             "mov %5, %1\n\t"                  // tmp = desired
0692             "ldq_l %2, %4\n\t"                // current = *(&storage)
0693             "cmpeq %2, %0, %3\n\t"            // success = current == expected
0694             "mov %2, %0\n\t"                  // expected = current
0695             "beq %3, 2f\n\t"                  // if (success == 0) goto end
0696             "stq_c %1, %4\n\t"                // storage = tmp; tmp = store succeeded
0697             "beq %1, 3f\n\t"                  // if (tmp == 0) goto retry
0698             "mov %1, %3\n\t"                  // success = tmp
0699             "2:\n\t"
0700
0701             ".subsection 2\n\t"
0702             "3: br 1b\n\t"
0703             ".previous\n\t"
0704
0705             : "+r" (expected),   // %0
0706               "=&r" (tmp),       // %1
0707               "=&r" (current),   // %2
0708               "=&r" (success)    // %3
0709             : "m" (storage),     // %4
0710               "r" (desired)      // %5
0711             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0712         );
0713         if (success)
0714             fence_after(success_order);
0715         else
0716             fence_after(failure_order);
0717         return !!success;
0718     }
0719
0720     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0721     {
0722         storage_type original, modified;
0723         fence_before(order);
0724         __asm__ __volatile__
0725         (
0726             "1:\n\t"
0727             "ldq_l %0, %2\n\t"
0728             "addq %0, %3, %1\n\t"
0729             "stq_c %1, %2\n\t"
0730             "beq %1, 2f\n\t"
0731
0732             ".subsection 2\n\t"
0733             "2: br 1b\n\t"
0734             ".previous\n\t"
0735
0736             : "=&r" (original),  // %0
0737               "=&r" (modified)   // %1
0738             : "m" (storage),     // %2
0739               "r" (v)            // %3
0740             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0741         );
0742         fence_after(order);
0743         return original;
0744     }
0745
0746     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0747     {
0748         storage_type original, modified;
0749         fence_before(order);
0750         __asm__ __volatile__
0751         (
0752             "1:\n\t"
0753             "ldq_l %0, %2\n\t"
0754             "subq %0, %3, %1\n\t"
0755             "stq_c %1, %2\n\t"
0756             "beq %1, 2f\n\t"
0757
0758             ".subsection 2\n\t"
0759             "2: br 1b\n\t"
0760             ".previous\n\t"
0761
0762             : "=&r" (original),  // %0
0763               "=&r" (modified)   // %1
0764             : "m" (storage),     // %2
0765               "r" (v)            // %3
0766             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0767         );
0768         fence_after(order);
0769         return original;
0770     }
0771
0772     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0773     {
0774         storage_type original, modified;
0775         fence_before(order);
0776         __asm__ __volatile__
0777         (
0778             "1:\n\t"
0779             "ldq_l %0, %2\n\t"
0780             "and %0, %3, %1\n\t"
0781             "stq_c %1, %2\n\t"
0782             "beq %1, 2f\n\t"
0783
0784             ".subsection 2\n\t"
0785             "2: br 1b\n\t"
0786             ".previous\n\t"
0787
0788             : "=&r" (original),  // %0
0789               "=&r" (modified)   // %1
0790             : "m" (storage),     // %2
0791               "r" (v)            // %3
0792             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0793         );
0794         fence_after(order);
0795         return original;
0796     }
0797
0798     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0799     {
0800         storage_type original, modified;
0801         fence_before(order);
0802         __asm__ __volatile__
0803         (
0804             "1:\n\t"
0805             "ldq_l %0, %2\n\t"
0806             "bis %0, %3, %1\n\t"
0807             "stq_c %1, %2\n\t"
0808             "beq %1, 2f\n\t"
0809
0810             ".subsection 2\n\t"
0811             "2: br 1b\n\t"
0812             ".previous\n\t"
0813
0814             : "=&r" (original),  // %0
0815               "=&r" (modified)   // %1
0816             : "m" (storage),     // %2
0817               "r" (v)            // %3
0818             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0819         );
0820         fence_after(order);
0821         return original;
0822     }
0823
0824     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0825     {
0826         storage_type original, modified;
0827         fence_before(order);
0828         __asm__ __volatile__
0829         (
0830             "1:\n\t"
0831             "ldq_l %0, %2\n\t"
0832             "xor %0, %3, %1\n\t"
0833             "stq_c %1, %2\n\t"
0834             "beq %1, 2f\n\t"
0835
0836             ".subsection 2\n\t"
0837             "2: br 1b\n\t"
0838             ".previous\n\t"
0839
0840             : "=&r" (original),  // %0
0841               "=&r" (modified)   // %1
0842             : "m" (storage),     // %2
0843               "r" (v)            // %3
0844             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
0845         );
0846         fence_after(order);
0847         return original;
0848     }
0849
0850     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0851     {
0852         return !!exchange(storage, (storage_type)1, order);
0853     }
0854
0855     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0856     {
0857         store(storage, (storage_type)0, order);
0858     }
0859 };
0860
0861 } // namespace detail
0862 } // namespace atomics
0863 } // namespace boost
0864
0865 #include <boost/atomic/detail/footer.hpp>
0866
0867 #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ALPHA_HPP_INCLUDED_