atomic/detail/core_arch_ops_gcc_x86.hpp

0001 /*
0002  * Distributed under the Boost Software License, Version 1.0.
0003  * (See accompanying file LICENSE_1_0.txt or copy at
0004  * http://www.boost.org/LICENSE_1_0.txt)
0005  *
0006  * Copyright (c) 2009 Helge Bahmann
0007  * Copyright (c) 2012 Tim Blechmann
0008  * Copyright (c) 2014 Andrey Semashev
0009  */
0010 /*!
0011  * \file   atomic/detail/core_arch_ops_gcc_x86.hpp
0012  *
0013  * This header contains implementation of the \c core_arch_operations template.
0014  */
0015
0016 #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_X86_HPP_INCLUDED_
0017 #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_X86_HPP_INCLUDED_
0018
0019 #include <cstddef>
0020 #include <boost/memory_order.hpp>
0021 #include <boost/atomic/detail/config.hpp>
0022 #include <boost/atomic/detail/storage_traits.hpp>
0023 #include <boost/atomic/detail/core_arch_operations_fwd.hpp>
0024 #include <boost/atomic/detail/capabilities.hpp>
0025 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B) || defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
0026 #include <boost/cstdint.hpp>
0027 #include <boost/atomic/detail/intptr.hpp>
0028 #include <boost/atomic/detail/string_ops.hpp>
0029 #include <boost/atomic/detail/core_ops_cas_based.hpp>
0030 #endif
0031 #include <boost/atomic/detail/header.hpp>
0032
0033 #ifdef BOOST_HAS_PRAGMA_ONCE
0034 #pragma once
0035 #endif
0036
0037 namespace boost {
0038 namespace atomics {
0039 namespace detail {
0040
0041 struct core_arch_operations_gcc_x86_base
0042 {
0043     static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = false;
0044     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
0045
0046     static BOOST_FORCEINLINE void fence_before(memory_order order) BOOST_NOEXCEPT
0047     {
0048         if ((static_cast< unsigned int >(order) & static_cast< unsigned int >(memory_order_release)) != 0u)
0049             __asm__ __volatile__ ("" ::: "memory");
0050     }
0051
0052     static BOOST_FORCEINLINE void fence_after(memory_order order) BOOST_NOEXCEPT
0053     {
0054         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
0055             __asm__ __volatile__ ("" ::: "memory");
0056     }
0057 };
0058
0059 template< std::size_t Size, bool Signed, bool Interprocess, typename Derived >
0060 struct core_arch_operations_gcc_x86 :
0061     public core_arch_operations_gcc_x86_base
0062 {
0063     typedef typename storage_traits< Size >::type storage_type;
0064
0065     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = Size;
0066     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = Size;
0067     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0068     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0069
0070     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0071     {
0072         if (order != memory_order_seq_cst)
0073         {
0074             fence_before(order);
0075             storage = v;
0076             fence_after(order);
0077         }
0078         else
0079         {
0080             Derived::exchange(storage, v, order);
0081         }
0082     }
0083
0084     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
0085     {
0086         storage_type v = storage;
0087         fence_after(order);
0088         return v;
0089     }
0090
0091     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0092     {
0093         return Derived::fetch_add(storage, -v, order);
0094     }
0095
0096     static BOOST_FORCEINLINE bool compare_exchange_weak(
0097         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0098     {
0099         return Derived::compare_exchange_strong(storage, expected, desired, success_order, failure_order);
0100     }
0101
0102     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0103     {
0104         return !!Derived::exchange(storage, (storage_type)1, order);
0105     }
0106
0107     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0108     {
0109         store(storage, (storage_type)0, order);
0110     }
0111 };
0112
0113 template< bool Signed, bool Interprocess >
0114 struct core_arch_operations< 1u, Signed, Interprocess > :
0115     public core_arch_operations_gcc_x86< 1u, Signed, Interprocess, core_arch_operations< 1u, Signed, Interprocess > >
0116 {
0117     typedef core_arch_operations_gcc_x86< 1u, Signed, Interprocess, core_arch_operations< 1u, Signed, Interprocess > > base_type;
0118     typedef typename base_type::storage_type storage_type;
0119     typedef typename storage_traits< 4u >::type temp_storage_type;
0120
0121     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0122     {
0123         __asm__ __volatile__
0124         (
0125             "lock; xaddb %0, %1"
0126             : "+q" (v), "+m" (storage)
0127             :
0128             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0129         );
0130         return v;
0131     }
0132
0133     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0134     {
0135         __asm__ __volatile__
0136         (
0137             "xchgb %0, %1"
0138             : "+q" (v), "+m" (storage)
0139             :
0140             : "memory"
0141         );
0142         return v;
0143     }
0144
0145     static BOOST_FORCEINLINE bool compare_exchange_strong(
0146         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
0147     {
0148         storage_type previous = expected;
0149         bool success;
0150 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0151         __asm__ __volatile__
0152         (
0153             "lock; cmpxchgb %3, %1"
0154             : "+a" (previous), "+m" (storage), "=@ccz" (success)
0155             : "q" (desired)
0156             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0157         );
0158 #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0159         __asm__ __volatile__
0160         (
0161             "lock; cmpxchgb %3, %1\n\t"
0162             "sete %2"
0163             : "+a" (previous), "+m" (storage), "=q" (success)
0164             : "q" (desired)
0165             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0166         );
0167 #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0168         expected = previous;
0169         return success;
0170     }
0171
0172 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
0173     temp_storage_type new_val;\
0174     __asm__ __volatile__\
0175     (\
0176         ".align 16\n\t"\
0177         "1: mov %[arg], %2\n\t"\
0178         op " %%al, %b2\n\t"\
0179         "lock; cmpxchgb %b2, %[storage]\n\t"\
0180         "jne 1b"\
0181         : [res] "+a" (result), [storage] "+m" (storage), "=&q" (new_val)\
0182         : [arg] "ir" ((temp_storage_type)argument)\
0183         : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0184     )
0185
0186     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0187     {
0188         storage_type res = storage;
0189         BOOST_ATOMIC_DETAIL_CAS_LOOP("andb", v, res);
0190         return res;
0191     }
0192
0193     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0194     {
0195         storage_type res = storage;
0196         BOOST_ATOMIC_DETAIL_CAS_LOOP("orb", v, res);
0197         return res;
0198     }
0199
0200     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0201     {
0202         storage_type res = storage;
0203         BOOST_ATOMIC_DETAIL_CAS_LOOP("xorb", v, res);
0204         return res;
0205     }
0206
0207 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0208 };
0209
0210 template< bool Signed, bool Interprocess >
0211 struct core_arch_operations< 2u, Signed, Interprocess > :
0212     public core_arch_operations_gcc_x86< 2u, Signed, Interprocess, core_arch_operations< 2u, Signed, Interprocess > >
0213 {
0214     typedef core_arch_operations_gcc_x86< 2u, Signed, Interprocess, core_arch_operations< 2u, Signed, Interprocess > > base_type;
0215     typedef typename base_type::storage_type storage_type;
0216     typedef typename storage_traits< 4u >::type temp_storage_type;
0217
0218     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0219     {
0220         __asm__ __volatile__
0221         (
0222             "lock; xaddw %0, %1"
0223             : "+q" (v), "+m" (storage)
0224             :
0225             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0226         );
0227         return v;
0228     }
0229
0230     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0231     {
0232         __asm__ __volatile__
0233         (
0234             "xchgw %0, %1"
0235             : "+q" (v), "+m" (storage)
0236             :
0237             : "memory"
0238         );
0239         return v;
0240     }
0241
0242     static BOOST_FORCEINLINE bool compare_exchange_strong(
0243         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
0244     {
0245         storage_type previous = expected;
0246         bool success;
0247 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0248         __asm__ __volatile__
0249         (
0250             "lock; cmpxchgw %3, %1"
0251             : "+a" (previous), "+m" (storage), "=@ccz" (success)
0252             : "q" (desired)
0253             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0254         );
0255 #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0256         __asm__ __volatile__
0257         (
0258             "lock; cmpxchgw %3, %1\n\t"
0259             "sete %2"
0260             : "+a" (previous), "+m" (storage), "=q" (success)
0261             : "q" (desired)
0262             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0263         );
0264 #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0265         expected = previous;
0266         return success;
0267     }
0268
0269 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
0270     temp_storage_type new_val;\
0271     __asm__ __volatile__\
0272     (\
0273         ".align 16\n\t"\
0274         "1: mov %[arg], %2\n\t"\
0275         op " %%ax, %w2\n\t"\
0276         "lock; cmpxchgw %w2, %[storage]\n\t"\
0277         "jne 1b"\
0278         : [res] "+a" (result), [storage] "+m" (storage), "=&q" (new_val)\
0279         : [arg] "ir" ((temp_storage_type)argument)\
0280         : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0281     )
0282
0283     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0284     {
0285         storage_type res = storage;
0286         BOOST_ATOMIC_DETAIL_CAS_LOOP("andw", v, res);
0287         return res;
0288     }
0289
0290     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0291     {
0292         storage_type res = storage;
0293         BOOST_ATOMIC_DETAIL_CAS_LOOP("orw", v, res);
0294         return res;
0295     }
0296
0297     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0298     {
0299         storage_type res = storage;
0300         BOOST_ATOMIC_DETAIL_CAS_LOOP("xorw", v, res);
0301         return res;
0302     }
0303
0304 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0305 };
0306
0307 template< bool Signed, bool Interprocess >
0308 struct core_arch_operations< 4u, Signed, Interprocess > :
0309     public core_arch_operations_gcc_x86< 4u, Signed, Interprocess, core_arch_operations< 4u, Signed, Interprocess > >
0310 {
0311     typedef core_arch_operations_gcc_x86< 4u, Signed, Interprocess, core_arch_operations< 4u, Signed, Interprocess > > base_type;
0312     typedef typename base_type::storage_type storage_type;
0313
0314     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0315     {
0316         __asm__ __volatile__
0317         (
0318             "lock; xaddl %0, %1"
0319             : "+r" (v), "+m" (storage)
0320             :
0321             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0322         );
0323         return v;
0324     }
0325
0326     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0327     {
0328         __asm__ __volatile__
0329         (
0330             "xchgl %0, %1"
0331             : "+r" (v), "+m" (storage)
0332             :
0333             : "memory"
0334         );
0335         return v;
0336     }
0337
0338     static BOOST_FORCEINLINE bool compare_exchange_strong(
0339         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
0340     {
0341         storage_type previous = expected;
0342         bool success;
0343 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0344         __asm__ __volatile__
0345         (
0346             "lock; cmpxchgl %3, %1"
0347             : "+a" (previous), "+m" (storage), "=@ccz" (success)
0348             : "r" (desired)
0349             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0350         );
0351 #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0352         __asm__ __volatile__
0353         (
0354             "lock; cmpxchgl %3, %1\n\t"
0355             "sete %2"
0356             : "+a" (previous), "+m" (storage), "=q" (success)
0357             : "r" (desired)
0358             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0359         );
0360 #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0361         expected = previous;
0362         return success;
0363     }
0364
0365 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
0366     storage_type new_val;\
0367     __asm__ __volatile__\
0368     (\
0369         ".align 16\n\t"\
0370         "1: mov %[arg], %[new_val]\n\t"\
0371         op " %%eax, %[new_val]\n\t"\
0372         "lock; cmpxchgl %[new_val], %[storage]\n\t"\
0373         "jne 1b"\
0374         : [res] "+a" (result), [storage] "+m" (storage), [new_val] "=&r" (new_val)\
0375         : [arg] "ir" (argument)\
0376         : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0377     )
0378
0379     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0380     {
0381         storage_type res = storage;
0382         BOOST_ATOMIC_DETAIL_CAS_LOOP("andl", v, res);
0383         return res;
0384     }
0385
0386     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0387     {
0388         storage_type res = storage;
0389         BOOST_ATOMIC_DETAIL_CAS_LOOP("orl", v, res);
0390         return res;
0391     }
0392
0393     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0394     {
0395         storage_type res = storage;
0396         BOOST_ATOMIC_DETAIL_CAS_LOOP("xorl", v, res);
0397         return res;
0398     }
0399
0400 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0401 };
0402
0403 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B)
0404
0405 // Note: In the 32-bit PIC code guarded with BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX below we have to avoid using memory
0406 // operand constraints because the compiler may choose to use ebx as the base register for that operand. At least, clang
0407 // is known to do that. For this reason we have to pre-compute a pointer to storage and pass it in edi. For the same reason
0408 // we cannot save ebx to the stack with a mov instruction, so we use esi as a scratch register and restore it afterwards.
0409 // Alternatively, we could push/pop the register to the stack, but exchanging the registers is faster.
0410 // The need to pass a pointer in edi is a bit wasteful because normally the memory operand would use a base pointer
0411 // with an offset (e.g. `this` + offset). But unfortunately, there seems to be no way around it.
0412
0413 template< bool Signed, bool Interprocess >
0414 struct gcc_dcas_x86
0415 {
0416     typedef typename storage_traits< 8u >::type storage_type;
0417     typedef uint32_t BOOST_ATOMIC_DETAIL_MAY_ALIAS aliasing_uint32_t;
0418 #if defined(__SSE2__)
0419     typedef uint32_t xmm_t __attribute__((__vector_size__(16)));
0420 #elif defined(__SSE__)
0421     typedef float xmm_t __attribute__((__vector_size__(16)));
0422 #endif
0423
0424     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
0425     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u;
0426     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0427     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0428     static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = true;
0429     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
0430
0431     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0432     {
0433         if (BOOST_LIKELY(order != memory_order_seq_cst && (((uintptr_t)&storage) & 7u) == 0u))
0434         {
0435 #if defined(__SSE__)
0436 #if defined(__SSE2__)
0437             xmm_t value = { static_cast< uint32_t >(v), static_cast< uint32_t >(v >> 32u), 0u, 0u };
0438 #else
0439             xmm_t value;
0440             BOOST_ATOMIC_DETAIL_MEMSET(&value, 0, sizeof(value));
0441             BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(v));
0442 #endif
0443             __asm__ __volatile__
0444             (
0445 #if defined(__AVX__)
0446                 "vmovq %[value], %[storage]\n\t"
0447 #elif defined(__SSE2__)
0448                 "movq %[value], %[storage]\n\t"
0449 #else
0450                 "movlps %[value], %[storage]\n\t"
0451 #endif
0452                 : [storage] "=m" (storage)
0453                 : [value] "x" (value)
0454                 : "memory"
0455             );
0456 #else // defined(__SSE__)
0457             __asm__ __volatile__
0458             (
0459                 "fildll %[value]\n\t"
0460                 "fistpll %[storage]\n\t"
0461                 : [storage] "=m" (storage)
0462                 : [value] "m" (v)
0463                 : "memory"
0464             );
0465 #endif // defined(__SSE__)
0466         }
0467         else
0468         {
0469 #if defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0470             __asm__ __volatile__
0471             (
0472                 "xchgl %%ebx, %%esi\n\t"
0473                 "movl %%eax, %%ebx\n\t"
0474                 "movl (%[dest]), %%eax\n\t"
0475                 "movl 4(%[dest]), %%edx\n\t"
0476                 ".align 16\n\t"
0477                 "1: lock; cmpxchg8b (%[dest])\n\t"
0478                 "jne 1b\n\t"
0479                 "xchgl %%ebx, %%esi\n\t"
0480                 :
0481                 : "a" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "D" (&storage)
0482                 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "edx", "memory"
0483             );
0484 #else // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0485             __asm__ __volatile__
0486             (
0487                 "movl %[dest_lo], %%eax\n\t"
0488                 "movl %[dest_hi], %%edx\n\t"
0489                 ".align 16\n\t"
0490                 "1: lock; cmpxchg8b %[dest_lo]\n\t"
0491                 "jne 1b\n\t"
0492                 : [dest_lo] "=m" (storage), [dest_hi] "=m" (reinterpret_cast< volatile aliasing_uint32_t* >(&storage)[1])
0493                 : [value_lo] "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u))
0494                 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "eax", "edx", "memory"
0495             );
0496 #endif // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0497         }
0498     }
0499
0500     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT
0501     {
0502         storage_type value;
0503
0504         if (BOOST_LIKELY((((uintptr_t)&storage) & 7u) == 0u))
0505         {
0506 #if defined(__SSE__)
0507             xmm_t v;
0508             __asm__ __volatile__
0509             (
0510 #if defined(__AVX__)
0511                 "vmovq %[storage], %[value]\n\t"
0512 #elif defined(__SSE2__)
0513                 "movq %[storage], %[value]\n\t"
0514 #else
0515                 "xorps %[value], %[value]\n\t"
0516                 "movlps %[storage], %[value]\n\t"
0517 #endif
0518                 : [value] "=x" (v)
0519                 : [storage] "m" (storage)
0520                 : "memory"
0521             );
0522 #if defined(__SSE2__) && (!defined(BOOST_GCC) || BOOST_GCC >= 40800)
0523             // gcc prior to 4.8 don't support subscript operator for vector types
0524             value = static_cast< storage_type >(v[0]) | (static_cast< storage_type >(v[1]) << 32u);
0525 #else
0526             BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(value));
0527 #endif
0528 #else // defined(__SSE__)
0529             __asm__ __volatile__
0530             (
0531                 "fildll %[storage]\n\t"
0532                 "fistpll %[value]\n\t"
0533                 : [value] "=m" (value)
0534                 : [storage] "m" (storage)
0535                 : "memory"
0536             );
0537 #endif // defined(__SSE__)
0538         }
0539         else
0540         {
0541             // Note that despite const qualification cmpxchg8b below may issue a store to the storage. The storage value
0542             // will not change, but this prevents the storage to reside in read-only memory.
0543
0544 #if defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0545
0546             uint32_t value_bits[2];
0547
0548             // We don't care for comparison result here; the previous value will be stored into value anyway.
0549             // Also we don't care for ebx and ecx values, they just have to be equal to eax and edx before cmpxchg8b.
0550             __asm__ __volatile__
0551             (
0552                 "movl %%ebx, %%eax\n\t"
0553                 "movl %%ecx, %%edx\n\t"
0554                 "lock; cmpxchg8b %[storage]\n\t"
0555                 : "=&a" (value_bits[0]), "=&d" (value_bits[1])
0556                 : [storage] "m" (storage)
0557                 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0558             );
0559             value = static_cast< storage_type >(value_bits[0]) | (static_cast< storage_type >(value_bits[1]) << 32u);
0560
0561 #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0562
0563             // We don't care for comparison result here; the previous value will be stored into value anyway.
0564             // Also we don't care for ebx and ecx values, they just have to be equal to eax and edx before cmpxchg8b.
0565             __asm__ __volatile__
0566             (
0567                 "movl %%ebx, %%eax\n\t"
0568                 "movl %%ecx, %%edx\n\t"
0569                 "lock; cmpxchg8b %[storage]\n\t"
0570                 : "=&A" (value)
0571                 : [storage] "m" (storage)
0572                 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0573             );
0574
0575 #endif // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0576         }
0577
0578         return value;
0579     }
0580
0581     static BOOST_FORCEINLINE bool compare_exchange_strong(
0582         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
0583     {
0584 #if defined(__clang__)
0585
0586         // Clang cannot allocate eax:edx register pairs but it has sync intrinsics
0587         storage_type old_expected = expected;
0588         expected = __sync_val_compare_and_swap(&storage, old_expected, desired);
0589         return expected == old_expected;
0590
0591 #elif defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0592
0593         bool success;
0594
0595 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0596         __asm__ __volatile__
0597         (
0598             "xchgl %%ebx, %%esi\n\t"
0599             "lock; cmpxchg8b (%[dest])\n\t"
0600             "xchgl %%ebx, %%esi\n\t"
0601             : "+A" (expected), [success] "=@ccz" (success)
0602             : "S" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u)), [dest] "D" (&storage)
0603             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0604         );
0605 #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0606         __asm__ __volatile__
0607         (
0608             "xchgl %%ebx, %%esi\n\t"
0609             "lock; cmpxchg8b (%[dest])\n\t"
0610             "xchgl %%ebx, %%esi\n\t"
0611             "sete %[success]\n\t"
0612             : "+A" (expected), [success] "=qm" (success)
0613             : "S" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u)), [dest] "D" (&storage)
0614             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0615         );
0616 #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0617
0618         return success;
0619
0620 #else // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0621
0622         bool success;
0623
0624 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0625         __asm__ __volatile__
0626         (
0627             "lock; cmpxchg8b %[dest]\n\t"
0628             : "+A" (expected), [dest] "+m" (storage), [success] "=@ccz" (success)
0629             : "b" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u))
0630             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0631         );
0632 #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0633         __asm__ __volatile__
0634         (
0635             "lock; cmpxchg8b %[dest]\n\t"
0636             "sete %[success]\n\t"
0637             : "+A" (expected), [dest] "+m" (storage), [success] "=qm" (success)
0638             : "b" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u))
0639             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0640         );
0641 #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0642
0643         return success;
0644
0645 #endif // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0646     }
0647
0648     static BOOST_FORCEINLINE bool compare_exchange_weak(
0649         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
0650     {
0651         return compare_exchange_strong(storage, expected, desired, success_order, failure_order);
0652     }
0653
0654     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0655     {
0656 #if defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0657 #if defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0658
0659         uint32_t old_bits[2];
0660         __asm__ __volatile__
0661         (
0662             "xchgl %%ebx, %%esi\n\t"
0663             "movl (%[dest]), %%eax\n\t"
0664             "movl 4(%[dest]), %%edx\n\t"
0665             ".align 16\n\t"
0666             "1: lock; cmpxchg8b (%[dest])\n\t"
0667             "jne 1b\n\t"
0668             "xchgl %%ebx, %%esi\n\t"
0669             : "=a" (old_bits[0]), "=d" (old_bits[1])
0670             : "S" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "D" (&storage)
0671             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0672         );
0673
0674         storage_type old_value;
0675         BOOST_ATOMIC_DETAIL_MEMCPY(&old_value, old_bits, sizeof(old_value));
0676         return old_value;
0677
0678 #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0679
0680         storage_type old_value;
0681         __asm__ __volatile__
0682         (
0683             "xchgl %%ebx, %%esi\n\t"
0684             "movl (%[dest]), %%eax\n\t"
0685             "movl 4(%[dest]), %%edx\n\t"
0686             ".align 16\n\t"
0687             "1: lock; cmpxchg8b (%[dest])\n\t"
0688             "jne 1b\n\t"
0689             "xchgl %%ebx, %%esi\n\t"
0690             : "=A" (old_value)
0691             : "S" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "D" (&storage)
0692             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0693         );
0694         return old_value;
0695
0696 #endif // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0697 #else // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0698 #if defined(__MINGW32__) && ((__GNUC__+0) * 100 + (__GNUC_MINOR__+0)) < 407
0699
0700         // MinGW gcc up to 4.6 has problems with allocating registers in the asm blocks below
0701         uint32_t old_bits[2];
0702         __asm__ __volatile__
0703         (
0704             "movl (%[dest]), %%eax\n\t"
0705             "movl 4(%[dest]), %%edx\n\t"
0706             ".align 16\n\t"
0707             "1: lock; cmpxchg8b (%[dest])\n\t"
0708             "jne 1b\n\t"
0709             : "=&a" (old_bits[0]), "=&d" (old_bits[1])
0710             : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "DS" (&storage)
0711             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0712         );
0713
0714         storage_type old_value;
0715         BOOST_ATOMIC_DETAIL_MEMCPY(&old_value, old_bits, sizeof(old_value));
0716         return old_value;
0717
0718 #elif defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0719
0720         uint32_t old_bits[2];
0721         __asm__ __volatile__
0722         (
0723             "movl %[dest_lo], %%eax\n\t"
0724             "movl %[dest_hi], %%edx\n\t"
0725             ".align 16\n\t"
0726             "1: lock; cmpxchg8b %[dest_lo]\n\t"
0727             "jne 1b\n\t"
0728             : "=&a" (old_bits[0]), "=&d" (old_bits[1]), [dest_lo] "+m" (storage), [dest_hi] "+m" (reinterpret_cast< volatile aliasing_uint32_t* >(&storage)[1])
0729             : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u))
0730             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0731         );
0732
0733         storage_type old_value;
0734         BOOST_ATOMIC_DETAIL_MEMCPY(&old_value, old_bits, sizeof(old_value));
0735         return old_value;
0736
0737 #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0738
0739         storage_type old_value;
0740         __asm__ __volatile__
0741         (
0742             "movl %[dest_lo], %%eax\n\t"
0743             "movl %[dest_hi], %%edx\n\t"
0744             ".align 16\n\t"
0745             "1: lock; cmpxchg8b %[dest_lo]\n\t"
0746             "jne 1b\n\t"
0747             : "=&A" (old_value), [dest_lo] "+m" (storage), [dest_hi] "+m" (reinterpret_cast< volatile aliasing_uint32_t* >(&storage)[1])
0748             : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u))
0749             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0750         );
0751         return old_value;
0752
0753 #endif // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0754 #endif // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX)
0755     }
0756 };
0757
0758 template< bool Signed, bool Interprocess >
0759 struct core_arch_operations< 8u, Signed, Interprocess > :
0760     public core_operations_cas_based< gcc_dcas_x86< Signed, Interprocess > >
0761 {
0762 };
0763
0764 #elif defined(__x86_64__)
0765
0766 template< bool Signed, bool Interprocess >
0767 struct core_arch_operations< 8u, Signed, Interprocess > :
0768     public core_arch_operations_gcc_x86< 8u, Signed, Interprocess, core_arch_operations< 8u, Signed, Interprocess > >
0769 {
0770     typedef core_arch_operations_gcc_x86< 8u, Signed, Interprocess, core_arch_operations< 8u, Signed, Interprocess > > base_type;
0771     typedef typename base_type::storage_type storage_type;
0772
0773     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0774     {
0775         __asm__ __volatile__
0776         (
0777             "lock; xaddq %0, %1"
0778             : "+r" (v), "+m" (storage)
0779             :
0780             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0781         );
0782         return v;
0783     }
0784
0785     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0786     {
0787         __asm__ __volatile__
0788         (
0789             "xchgq %0, %1"
0790             : "+r" (v), "+m" (storage)
0791             :
0792             : "memory"
0793         );
0794         return v;
0795     }
0796
0797     static BOOST_FORCEINLINE bool compare_exchange_strong(
0798         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
0799     {
0800         storage_type previous = expected;
0801         bool success;
0802 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0803         __asm__ __volatile__
0804         (
0805             "lock; cmpxchgq %3, %1"
0806             : "+a" (previous), "+m" (storage), "=@ccz" (success)
0807             : "r" (desired)
0808             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0809         );
0810 #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0811         __asm__ __volatile__
0812         (
0813             "lock; cmpxchgq %3, %1\n\t"
0814             "sete %2"
0815             : "+a" (previous), "+m" (storage), "=q" (success)
0816             : "r" (desired)
0817             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0818         );
0819 #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0820         expected = previous;
0821         return success;
0822     }
0823
0824 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, result)\
0825     storage_type new_val;\
0826     __asm__ __volatile__\
0827     (\
0828         ".align 16\n\t"\
0829         "1: movq %[arg], %[new_val]\n\t"\
0830         op " %%rax, %[new_val]\n\t"\
0831         "lock; cmpxchgq %[new_val], %[storage]\n\t"\
0832         "jne 1b"\
0833         : [res] "+a" (result), [storage] "+m" (storage), [new_val] "=&r" (new_val)\
0834         : [arg] "r" (argument)\
0835         : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0836     )
0837
0838     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0839     {
0840         storage_type res = storage;
0841         BOOST_ATOMIC_DETAIL_CAS_LOOP("andq", v, res);
0842         return res;
0843     }
0844
0845     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0846     {
0847         storage_type res = storage;
0848         BOOST_ATOMIC_DETAIL_CAS_LOOP("orq", v, res);
0849         return res;
0850     }
0851
0852     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0853     {
0854         storage_type res = storage;
0855         BOOST_ATOMIC_DETAIL_CAS_LOOP("xorq", v, res);
0856         return res;
0857     }
0858
0859 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0860 };
0861
0862 #endif
0863
0864 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
0865
0866 template< bool Signed, bool Interprocess >
0867 struct gcc_dcas_x86_64
0868 {
0869     typedef typename storage_traits< 16u >::type storage_type;
0870     typedef uint64_t BOOST_ATOMIC_DETAIL_MAY_ALIAS aliasing_uint64_t;
0871 #if defined(__AVX__)
0872     typedef uint64_t __attribute__((__vector_size__(16))) xmm_t;
0873 #endif
0874
0875     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 16u;
0876     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 16u;
0877     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
0878     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
0879     static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = true;
0880     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
0881
0882     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
0883     {
0884 #if defined(__AVX__)
0885         if (BOOST_LIKELY(order != memory_order_seq_cst && (((uintptr_t)&storage) & 15u) == 0u))
0886         {
0887             // According to SDM Volume 3, 8.1.1 Guaranteed Atomic Operations, processors supporting AVX guarantee
0888             // aligned vector moves to be atomic.
0889 #if defined(BOOST_HAS_INT128)
0890             xmm_t value = { static_cast< uint64_t >(v), static_cast< uint64_t >(v >> 64u) };
0891 #else
0892             xmm_t value;
0893             BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(v));
0894 #endif
0895             __asm__ __volatile__
0896             (
0897                 "vmovdqa %[value], %[storage]\n\t"
0898                 : [storage] "=m" (storage)
0899                 : [value] "x" (value)
0900                 : "memory"
0901             );
0902
0903             return;
0904         }
0905 #endif // defined(__AVX__)
0906
0907         __asm__ __volatile__
0908         (
0909             "movq %[dest_lo], %%rax\n\t"
0910             "movq %[dest_hi], %%rdx\n\t"
0911             ".align 16\n\t"
0912             "1: lock; cmpxchg16b %[dest_lo]\n\t"
0913             "jne 1b\n\t"
0914             : [dest_lo] "=m" (storage), [dest_hi] "=m" (reinterpret_cast< volatile aliasing_uint64_t* >(&storage)[1])
0915             : "b" (reinterpret_cast< const aliasing_uint64_t* >(&v)[0]), "c" (reinterpret_cast< const aliasing_uint64_t* >(&v)[1])
0916             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "rax", "rdx", "memory"
0917         );
0918     }
0919
0920     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT
0921     {
0922 #if defined(__AVX__)
0923         if (BOOST_LIKELY((((uintptr_t)&storage) & 15u) == 0u))
0924         {
0925             // According to SDM Volume 3, 8.1.1 Guaranteed Atomic Operations, processors supporting AVX guarantee
0926             // aligned vector moves to be atomic.
0927             xmm_t v;
0928             __asm__ __volatile__
0929             (
0930                 "vmovdqa %[storage], %[value]\n\t"
0931                 : [value] "=x" (v)
0932                 : [storage] "m" (storage)
0933                 : "memory"
0934             );
0935
0936 #if defined(BOOST_HAS_INT128) && (!defined(BOOST_GCC) || BOOST_GCC >= 40800)
0937             // gcc prior to 4.8 don't support subscript operator for vector types
0938             storage_type value = static_cast< storage_type >(v[0]) | (static_cast< storage_type >(v[1]) << 64u);
0939 #else
0940             storage_type value;
0941             BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(v));
0942 #endif
0943             return value;
0944         }
0945 #endif // defined(__AVX__)
0946
0947         // Note that despite const qualification cmpxchg16b below may issue a store to the storage. The storage value
0948         // will not change, but this prevents the storage to reside in read-only memory.
0949
0950 #if defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0951
0952         uint64_t value_bits[2];
0953
0954         // We don't care for comparison result here; the previous value will be stored into value anyway.
0955         // Also we don't care for rbx and rcx values, they just have to be equal to rax and rdx before cmpxchg16b.
0956         __asm__ __volatile__
0957         (
0958             "movq %%rbx, %%rax\n\t"
0959             "movq %%rcx, %%rdx\n\t"
0960             "lock; cmpxchg16b %[storage]\n\t"
0961             : "=&a" (value_bits[0]), "=&d" (value_bits[1])
0962             : [storage] "m" (storage)
0963             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0964         );
0965
0966 #if defined(BOOST_HAS_INT128)
0967         storage_type value = static_cast< storage_type >(value_bits[0]) | (static_cast< storage_type >(value_bits[1]) << 64u);
0968 #else
0969         storage_type value;
0970         BOOST_ATOMIC_DETAIL_MEMCPY(&value, value_bits, sizeof(value));
0971 #endif
0972         return value;
0973
0974 #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0975
0976         storage_type value;
0977
0978         // We don't care for comparison result here; the previous value will be stored into value anyway.
0979         // Also we don't care for rbx and rcx values, they just have to be equal to rax and rdx before cmpxchg16b.
0980         __asm__ __volatile__
0981         (
0982             "movq %%rbx, %%rax\n\t"
0983             "movq %%rcx, %%rdx\n\t"
0984             "lock; cmpxchg16b %[storage]\n\t"
0985             : "=&A" (value)
0986             : [storage] "m" (storage)
0987             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0988         );
0989
0990         return value;
0991
0992 #endif // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
0993     }
0994
0995     static BOOST_FORCEINLINE bool compare_exchange_strong(
0996         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
0997     {
0998 #if defined(__clang__)
0999
1000         // Clang cannot allocate rax:rdx register pairs but it has sync intrinsics
1001         storage_type old_expected = expected;
1002         expected = __sync_val_compare_and_swap(&storage, old_expected, desired);
1003         return expected == old_expected;
1004
1005 #elif defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
1006
1007         // Some compilers can't allocate rax:rdx register pair either but also don't support 128-bit __sync_val_compare_and_swap
1008         bool success;
1009         __asm__ __volatile__
1010         (
1011             "lock; cmpxchg16b %[dest]\n\t"
1012             "sete %[success]\n\t"
1013             : [dest] "+m" (storage), "+a" (reinterpret_cast< aliasing_uint64_t* >(&expected)[0]), "+d" (reinterpret_cast< aliasing_uint64_t* >(&expected)[1]), [success] "=q" (success)
1014             : "b" (reinterpret_cast< const aliasing_uint64_t* >(&desired)[0]), "c" (reinterpret_cast< const aliasing_uint64_t* >(&desired)[1])
1015             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1016         );
1017
1018         return success;
1019
1020 #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
1021
1022         bool success;
1023
1024 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1025         __asm__ __volatile__
1026         (
1027             "lock; cmpxchg16b %[dest]\n\t"
1028             : "+A" (expected), [dest] "+m" (storage), "=@ccz" (success)
1029             : "b" (reinterpret_cast< const aliasing_uint64_t* >(&desired)[0]), "c" (reinterpret_cast< const aliasing_uint64_t* >(&desired)[1])
1030             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1031         );
1032 #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1033         __asm__ __volatile__
1034         (
1035             "lock; cmpxchg16b %[dest]\n\t"
1036             "sete %[success]\n\t"
1037             : "+A" (expected), [dest] "+m" (storage), [success] "=qm" (success)
1038             : "b" (reinterpret_cast< const aliasing_uint64_t* >(&desired)[0]), "c" (reinterpret_cast< const aliasing_uint64_t* >(&desired)[1])
1039             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1040         );
1041 #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1042
1043         return success;
1044
1045 #endif // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
1046     }
1047
1048     static BOOST_FORCEINLINE bool compare_exchange_weak(
1049         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1050     {
1051         return compare_exchange_strong(storage, expected, desired, success_order, failure_order);
1052     }
1053
1054     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1055     {
1056 #if defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
1057         uint64_t old_bits[2];
1058         __asm__ __volatile__
1059         (
1060             "movq %[dest_lo], %%rax\n\t"
1061             "movq %[dest_hi], %%rdx\n\t"
1062             ".align 16\n\t"
1063             "1: lock; cmpxchg16b %[dest_lo]\n\t"
1064             "jne 1b\n\t"
1065             : [dest_lo] "+m" (storage), [dest_hi] "+m" (reinterpret_cast< volatile aliasing_uint64_t* >(&storage)[1]), "=&a" (old_bits[0]), "=&d" (old_bits[1])
1066             : "b" (reinterpret_cast< const aliasing_uint64_t* >(&v)[0]), "c" (reinterpret_cast< const aliasing_uint64_t* >(&v)[1])
1067             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1068         );
1069
1070 #if defined(BOOST_HAS_INT128)
1071         storage_type old_value = static_cast< storage_type >(old_bits[0]) | (static_cast< storage_type >(old_bits[1]) << 64u);
1072 #else
1073         storage_type old_value;
1074         BOOST_ATOMIC_DETAIL_MEMCPY(&old_value, old_bits, sizeof(old_value));
1075 #endif
1076         return old_value;
1077 #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
1078         storage_type old_value;
1079         __asm__ __volatile__
1080         (
1081             "movq %[dest_lo], %%rax\n\t"
1082             "movq %[dest_hi], %%rdx\n\t"
1083             ".align 16\n\t"
1084             "1: lock; cmpxchg16b %[dest_lo]\n\t"
1085             "jne 1b\n\t"
1086             : "=&A" (old_value), [dest_lo] "+m" (storage), [dest_hi] "+m" (reinterpret_cast< volatile aliasing_uint64_t* >(&storage)[1])
1087             : "b" (reinterpret_cast< const aliasing_uint64_t* >(&v)[0]), "c" (reinterpret_cast< const aliasing_uint64_t* >(&v)[1])
1088             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1089         );
1090
1091         return old_value;
1092 #endif // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS)
1093     }
1094 };
1095
1096 template< bool Signed, bool Interprocess >
1097 struct core_arch_operations< 16u, Signed, Interprocess > :
1098     public core_operations_cas_based< gcc_dcas_x86_64< Signed, Interprocess > >
1099 {
1100 };
1101
1102 #endif // defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
1103
1104 } // namespace detail
1105 } // namespace atomics
1106 } // namespace boost
1107
1108 #include <boost/atomic/detail/footer.hpp>
1109
1110 #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_X86_HPP_INCLUDED_