File indexing completed on 2025-01-30 09:33:56
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014 #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_X86_HPP_INCLUDED_
0015 #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_X86_HPP_INCLUDED_
0016
0017 #include <cstddef>
0018 #include <boost/cstdint.hpp>
0019 #include <boost/memory_order.hpp>
0020 #include <boost/atomic/detail/config.hpp>
0021 #include <boost/atomic/detail/storage_traits.hpp>
0022 #include <boost/atomic/detail/extra_operations_fwd.hpp>
0023 #include <boost/atomic/detail/extra_ops_generic.hpp>
0024 #include <boost/atomic/detail/header.hpp>
0025
0026 #ifdef BOOST_HAS_PRAGMA_ONCE
0027 #pragma once
0028 #endif
0029
0030 namespace boost {
0031 namespace atomics {
0032 namespace detail {
0033
0034 template< typename Base, bool Signed >
0035 struct extra_operations< Base, 1u, Signed, true > :
0036 public extra_operations_generic< Base, 1u, Signed >
0037 {
0038 typedef extra_operations_generic< Base, 1u, Signed > base_type;
0039 typedef typename base_type::storage_type storage_type;
0040 typedef typename storage_traits< 4u >::type temp_storage_type;
0041
0042 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\
0043 __asm__ __volatile__\
0044 (\
0045 ".align 16\n\t"\
0046 "1: movzbl %[orig], %2\n\t"\
0047 op " %b2\n\t"\
0048 "lock; cmpxchgb %b2, %[storage]\n\t"\
0049 "jne 1b"\
0050 : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\
0051 : \
0052 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0053 )
0054
0055 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0056 {
0057 storage_type original = storage;
0058 temp_storage_type result;
0059 BOOST_ATOMIC_DETAIL_CAS_LOOP("negb", original, result);
0060 return original;
0061 }
0062
0063 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0064 {
0065 storage_type original = storage;
0066 temp_storage_type result;
0067 BOOST_ATOMIC_DETAIL_CAS_LOOP("notb", original, result);
0068 return original;
0069 }
0070
0071 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0072 {
0073 storage_type original = storage;
0074 temp_storage_type result;
0075 BOOST_ATOMIC_DETAIL_CAS_LOOP("negb", original, result);
0076 return static_cast< storage_type >(result);
0077 }
0078
0079 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0080 {
0081 storage_type original = storage;
0082 temp_storage_type result;
0083 BOOST_ATOMIC_DETAIL_CAS_LOOP("notb", original, result);
0084 return static_cast< storage_type >(result);
0085 }
0086
0087 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0088
0089 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\
0090 __asm__ __volatile__\
0091 (\
0092 ".align 16\n\t"\
0093 "1: mov %[arg], %2\n\t"\
0094 op " %%al, %b2\n\t"\
0095 "lock; cmpxchgb %b2, %[storage]\n\t"\
0096 "jne 1b"\
0097 : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\
0098 : [arg] "ir" ((temp_storage_type)argument)\
0099 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0100 )
0101
0102 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0103 {
0104 storage_type original = storage;
0105 temp_storage_type result;
0106 BOOST_ATOMIC_DETAIL_CAS_LOOP("andb", v, original, result);
0107 return static_cast< storage_type >(result);
0108 }
0109
0110 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0111 {
0112 storage_type original = storage;
0113 temp_storage_type result;
0114 BOOST_ATOMIC_DETAIL_CAS_LOOP("orb", v, original, result);
0115 return static_cast< storage_type >(result);
0116 }
0117
0118 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0119 {
0120 storage_type original = storage;
0121 temp_storage_type result;
0122 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorb", v, original, result);
0123 return static_cast< storage_type >(result);
0124 }
0125
0126 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0127
0128 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0129 {
0130 return !!negate(storage, order);
0131 }
0132
0133 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0134 {
0135 return !!bitwise_complement(storage, order);
0136 }
0137
0138 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0139 {
0140 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0141 {
0142 __asm__ __volatile__
0143 (
0144 "lock; incb %[storage]\n\t"
0145 : [storage] "+m" (storage)
0146 :
0147 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0148 );
0149 }
0150 else
0151 {
0152 __asm__ __volatile__
0153 (
0154 "lock; addb %[argument], %[storage]\n\t"
0155 : [storage] "+m" (storage)
0156 : [argument] "iq" (v)
0157 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0158 );
0159 }
0160 }
0161
0162 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0163 {
0164 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0165 {
0166 __asm__ __volatile__
0167 (
0168 "lock; decb %[storage]\n\t"
0169 : [storage] "+m" (storage)
0170 :
0171 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0172 );
0173 }
0174 else
0175 {
0176 __asm__ __volatile__
0177 (
0178 "lock; subb %[argument], %[storage]\n\t"
0179 : [storage] "+m" (storage)
0180 : [argument] "iq" (v)
0181 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0182 );
0183 }
0184 }
0185
0186 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0187 {
0188 __asm__ __volatile__
0189 (
0190 "lock; negb %[storage]\n\t"
0191 : [storage] "+m" (storage)
0192 :
0193 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0194 );
0195 }
0196
0197 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0198 {
0199 __asm__ __volatile__
0200 (
0201 "lock; andb %[argument], %[storage]\n\t"
0202 : [storage] "+m" (storage)
0203 : [argument] "iq" (v)
0204 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0205 );
0206 }
0207
0208 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0209 {
0210 __asm__ __volatile__
0211 (
0212 "lock; orb %[argument], %[storage]\n\t"
0213 : [storage] "+m" (storage)
0214 : [argument] "iq" (v)
0215 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0216 );
0217 }
0218
0219 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0220 {
0221 __asm__ __volatile__
0222 (
0223 "lock; xorb %[argument], %[storage]\n\t"
0224 : [storage] "+m" (storage)
0225 : [argument] "iq" (v)
0226 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0227 );
0228 }
0229
0230 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0231 {
0232 __asm__ __volatile__
0233 (
0234 "lock; notb %[storage]\n\t"
0235 : [storage] "+m" (storage)
0236 :
0237 : "memory"
0238 );
0239 }
0240
0241 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0242 {
0243 bool res;
0244 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0245 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0246 {
0247 __asm__ __volatile__
0248 (
0249 "lock; incb %[storage]\n\t"
0250 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0251 :
0252 : "memory"
0253 );
0254 }
0255 else
0256 {
0257 __asm__ __volatile__
0258 (
0259 "lock; addb %[argument], %[storage]\n\t"
0260 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0261 : [argument] "iq" (v)
0262 : "memory"
0263 );
0264 }
0265 #else
0266 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0267 {
0268 __asm__ __volatile__
0269 (
0270 "lock; incb %[storage]\n\t"
0271 "setnz %[result]\n\t"
0272 : [storage] "+m" (storage), [result] "=q" (res)
0273 :
0274 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0275 );
0276 }
0277 else
0278 {
0279 __asm__ __volatile__
0280 (
0281 "lock; addb %[argument], %[storage]\n\t"
0282 "setnz %[result]\n\t"
0283 : [storage] "+m" (storage), [result] "=q" (res)
0284 : [argument] "iq" (v)
0285 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0286 );
0287 }
0288 #endif
0289 return res;
0290 }
0291
0292 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0293 {
0294 bool res;
0295 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0296 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0297 {
0298 __asm__ __volatile__
0299 (
0300 "lock; decb %[storage]\n\t"
0301 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0302 :
0303 : "memory"
0304 );
0305 }
0306 else
0307 {
0308 __asm__ __volatile__
0309 (
0310 "lock; subb %[argument], %[storage]\n\t"
0311 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0312 : [argument] "iq" (v)
0313 : "memory"
0314 );
0315 }
0316 #else
0317 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0318 {
0319 __asm__ __volatile__
0320 (
0321 "lock; decb %[storage]\n\t"
0322 "setnz %[result]\n\t"
0323 : [storage] "+m" (storage), [result] "=q" (res)
0324 :
0325 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0326 );
0327 }
0328 else
0329 {
0330 __asm__ __volatile__
0331 (
0332 "lock; subb %[argument], %[storage]\n\t"
0333 "setnz %[result]\n\t"
0334 : [storage] "+m" (storage), [result] "=q" (res)
0335 : [argument] "iq" (v)
0336 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0337 );
0338 }
0339 #endif
0340 return res;
0341 }
0342
0343 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0344 {
0345 bool res;
0346 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0347 __asm__ __volatile__
0348 (
0349 "lock; andb %[argument], %[storage]\n\t"
0350 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0351 : [argument] "iq" (v)
0352 : "memory"
0353 );
0354 #else
0355 __asm__ __volatile__
0356 (
0357 "lock; andb %[argument], %[storage]\n\t"
0358 "setnz %[result]\n\t"
0359 : [storage] "+m" (storage), [result] "=q" (res)
0360 : [argument] "iq" (v)
0361 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0362 );
0363 #endif
0364 return res;
0365 }
0366
0367 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0368 {
0369 bool res;
0370 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0371 __asm__ __volatile__
0372 (
0373 "lock; orb %[argument], %[storage]\n\t"
0374 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0375 : [argument] "iq" (v)
0376 : "memory"
0377 );
0378 #else
0379 __asm__ __volatile__
0380 (
0381 "lock; orb %[argument], %[storage]\n\t"
0382 "setnz %[result]\n\t"
0383 : [storage] "+m" (storage), [result] "=q" (res)
0384 : [argument] "iq" (v)
0385 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0386 );
0387 #endif
0388 return res;
0389 }
0390
0391 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0392 {
0393 bool res;
0394 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0395 __asm__ __volatile__
0396 (
0397 "lock; xorb %[argument], %[storage]\n\t"
0398 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0399 : [argument] "iq" (v)
0400 : "memory"
0401 );
0402 #else
0403 __asm__ __volatile__
0404 (
0405 "lock; xorb %[argument], %[storage]\n\t"
0406 "setnz %[result]\n\t"
0407 : [storage] "+m" (storage), [result] "=q" (res)
0408 : [argument] "iq" (v)
0409 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0410 );
0411 #endif
0412 return res;
0413 }
0414 };
0415
0416 template< typename Base, bool Signed >
0417 struct extra_operations< Base, 2u, Signed, true > :
0418 public extra_operations_generic< Base, 2u, Signed >
0419 {
0420 typedef extra_operations_generic< Base, 2u, Signed > base_type;
0421 typedef typename base_type::storage_type storage_type;
0422 typedef typename storage_traits< 4u >::type temp_storage_type;
0423
0424 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\
0425 __asm__ __volatile__\
0426 (\
0427 ".align 16\n\t"\
0428 "1: movzwl %[orig], %2\n\t"\
0429 op " %w2\n\t"\
0430 "lock; cmpxchgw %w2, %[storage]\n\t"\
0431 "jne 1b"\
0432 : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\
0433 : \
0434 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0435 )
0436
0437 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0438 {
0439 storage_type original = storage;
0440 temp_storage_type result;
0441 BOOST_ATOMIC_DETAIL_CAS_LOOP("negw", original, result);
0442 return original;
0443 }
0444
0445 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0446 {
0447 storage_type original = storage;
0448 temp_storage_type result;
0449 BOOST_ATOMIC_DETAIL_CAS_LOOP("notw", original, result);
0450 return original;
0451 }
0452
0453 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0454 {
0455 storage_type original = storage;
0456 temp_storage_type result;
0457 BOOST_ATOMIC_DETAIL_CAS_LOOP("negw", original, result);
0458 return static_cast< storage_type >(result);
0459 }
0460
0461 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0462 {
0463 storage_type original = storage;
0464 temp_storage_type result;
0465 BOOST_ATOMIC_DETAIL_CAS_LOOP("notw", original, result);
0466 return static_cast< storage_type >(result);
0467 }
0468
0469 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0470
0471 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\
0472 __asm__ __volatile__\
0473 (\
0474 ".align 16\n\t"\
0475 "1: mov %[arg], %2\n\t"\
0476 op " %%ax, %w2\n\t"\
0477 "lock; cmpxchgw %w2, %[storage]\n\t"\
0478 "jne 1b"\
0479 : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\
0480 : [arg] "ir" ((temp_storage_type)argument)\
0481 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0482 )
0483
0484 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0485 {
0486 storage_type original = storage;
0487 temp_storage_type result;
0488 BOOST_ATOMIC_DETAIL_CAS_LOOP("andw", v, original, result);
0489 return static_cast< storage_type >(result);
0490 }
0491
0492 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0493 {
0494 storage_type original = storage;
0495 temp_storage_type result;
0496 BOOST_ATOMIC_DETAIL_CAS_LOOP("orw", v, original, result);
0497 return static_cast< storage_type >(result);
0498 }
0499
0500 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0501 {
0502 storage_type original = storage;
0503 temp_storage_type result;
0504 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorw", v, original, result);
0505 return static_cast< storage_type >(result);
0506 }
0507
0508 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0509
0510 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0511 {
0512 return !!negate(storage, order);
0513 }
0514
0515 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0516 {
0517 return !!bitwise_complement(storage, order);
0518 }
0519
0520 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0521 {
0522 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0523 {
0524 __asm__ __volatile__
0525 (
0526 "lock; incw %[storage]\n\t"
0527 : [storage] "+m" (storage)
0528 :
0529 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0530 );
0531 }
0532 else
0533 {
0534 __asm__ __volatile__
0535 (
0536 "lock; addw %[argument], %[storage]\n\t"
0537 : [storage] "+m" (storage)
0538 : [argument] "iq" (v)
0539 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0540 );
0541 }
0542 }
0543
0544 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0545 {
0546 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0547 {
0548 __asm__ __volatile__
0549 (
0550 "lock; decw %[storage]\n\t"
0551 : [storage] "+m" (storage)
0552 :
0553 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0554 );
0555 }
0556 else
0557 {
0558 __asm__ __volatile__
0559 (
0560 "lock; subw %[argument], %[storage]\n\t"
0561 : [storage] "+m" (storage)
0562 : [argument] "iq" (v)
0563 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0564 );
0565 }
0566 }
0567
0568 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0569 {
0570 __asm__ __volatile__
0571 (
0572 "lock; negw %[storage]\n\t"
0573 : [storage] "+m" (storage)
0574 :
0575 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0576 );
0577 }
0578
0579 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0580 {
0581 __asm__ __volatile__
0582 (
0583 "lock; andw %[argument], %[storage]\n\t"
0584 : [storage] "+m" (storage)
0585 : [argument] "iq" (v)
0586 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0587 );
0588 }
0589
0590 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0591 {
0592 __asm__ __volatile__
0593 (
0594 "lock; orw %[argument], %[storage]\n\t"
0595 : [storage] "+m" (storage)
0596 : [argument] "iq" (v)
0597 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0598 );
0599 }
0600
0601 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0602 {
0603 __asm__ __volatile__
0604 (
0605 "lock; xorw %[argument], %[storage]\n\t"
0606 : [storage] "+m" (storage)
0607 : [argument] "iq" (v)
0608 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0609 );
0610 }
0611
0612 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0613 {
0614 __asm__ __volatile__
0615 (
0616 "lock; notw %[storage]\n\t"
0617 : [storage] "+m" (storage)
0618 :
0619 : "memory"
0620 );
0621 }
0622
0623 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0624 {
0625 bool res;
0626 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0627 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0628 {
0629 __asm__ __volatile__
0630 (
0631 "lock; incw %[storage]\n\t"
0632 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0633 :
0634 : "memory"
0635 );
0636 }
0637 else
0638 {
0639 __asm__ __volatile__
0640 (
0641 "lock; addw %[argument], %[storage]\n\t"
0642 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0643 : [argument] "iq" (v)
0644 : "memory"
0645 );
0646 }
0647 #else
0648 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0649 {
0650 __asm__ __volatile__
0651 (
0652 "lock; incw %[storage]\n\t"
0653 "setnz %[result]\n\t"
0654 : [storage] "+m" (storage), [result] "=q" (res)
0655 :
0656 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0657 );
0658 }
0659 else
0660 {
0661 __asm__ __volatile__
0662 (
0663 "lock; addw %[argument], %[storage]\n\t"
0664 "setnz %[result]\n\t"
0665 : [storage] "+m" (storage), [result] "=q" (res)
0666 : [argument] "iq" (v)
0667 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0668 );
0669 }
0670 #endif
0671 return res;
0672 }
0673
0674 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0675 {
0676 bool res;
0677 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0678 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0679 {
0680 __asm__ __volatile__
0681 (
0682 "lock; decw %[storage]\n\t"
0683 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0684 :
0685 : "memory"
0686 );
0687 }
0688 else
0689 {
0690 __asm__ __volatile__
0691 (
0692 "lock; subw %[argument], %[storage]\n\t"
0693 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0694 : [argument] "iq" (v)
0695 : "memory"
0696 );
0697 }
0698 #else
0699 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0700 {
0701 __asm__ __volatile__
0702 (
0703 "lock; decw %[storage]\n\t"
0704 "setnz %[result]\n\t"
0705 : [storage] "+m" (storage), [result] "=q" (res)
0706 :
0707 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0708 );
0709 }
0710 else
0711 {
0712 __asm__ __volatile__
0713 (
0714 "lock; subw %[argument], %[storage]\n\t"
0715 "setnz %[result]\n\t"
0716 : [storage] "+m" (storage), [result] "=q" (res)
0717 : [argument] "iq" (v)
0718 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0719 );
0720 }
0721 #endif
0722 return res;
0723 }
0724
0725 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0726 {
0727 bool res;
0728 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0729 __asm__ __volatile__
0730 (
0731 "lock; andw %[argument], %[storage]\n\t"
0732 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0733 : [argument] "iq" (v)
0734 : "memory"
0735 );
0736 #else
0737 __asm__ __volatile__
0738 (
0739 "lock; andw %[argument], %[storage]\n\t"
0740 "setnz %[result]\n\t"
0741 : [storage] "+m" (storage), [result] "=q" (res)
0742 : [argument] "iq" (v)
0743 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0744 );
0745 #endif
0746 return res;
0747 }
0748
0749 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0750 {
0751 bool res;
0752 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0753 __asm__ __volatile__
0754 (
0755 "lock; orw %[argument], %[storage]\n\t"
0756 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0757 : [argument] "iq" (v)
0758 : "memory"
0759 );
0760 #else
0761 __asm__ __volatile__
0762 (
0763 "lock; orw %[argument], %[storage]\n\t"
0764 "setnz %[result]\n\t"
0765 : [storage] "+m" (storage), [result] "=q" (res)
0766 : [argument] "iq" (v)
0767 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0768 );
0769 #endif
0770 return res;
0771 }
0772
0773 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0774 {
0775 bool res;
0776 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0777 __asm__ __volatile__
0778 (
0779 "lock; xorw %[argument], %[storage]\n\t"
0780 : [storage] "+m" (storage), [result] "=@ccnz" (res)
0781 : [argument] "iq" (v)
0782 : "memory"
0783 );
0784 #else
0785 __asm__ __volatile__
0786 (
0787 "lock; xorw %[argument], %[storage]\n\t"
0788 "setnz %[result]\n\t"
0789 : [storage] "+m" (storage), [result] "=q" (res)
0790 : [argument] "iq" (v)
0791 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0792 );
0793 #endif
0794 return res;
0795 }
0796
0797 static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
0798 {
0799 bool res;
0800 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0801 __asm__ __volatile__
0802 (
0803 "lock; btsw %[bit_number], %[storage]\n\t"
0804 : [storage] "+m" (storage), [result] "=@ccc" (res)
0805 : [bit_number] "Kq" ((uint16_t)bit_number)
0806 : "memory"
0807 );
0808 #else
0809 __asm__ __volatile__
0810 (
0811 "lock; btsw %[bit_number], %[storage]\n\t"
0812 "setc %[result]\n\t"
0813 : [storage] "+m" (storage), [result] "=q" (res)
0814 : [bit_number] "Kq" ((uint16_t)bit_number)
0815 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0816 );
0817 #endif
0818 return res;
0819 }
0820
0821 static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
0822 {
0823 bool res;
0824 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0825 __asm__ __volatile__
0826 (
0827 "lock; btrw %[bit_number], %[storage]\n\t"
0828 : [storage] "+m" (storage), [result] "=@ccc" (res)
0829 : [bit_number] "Kq" ((uint16_t)bit_number)
0830 : "memory"
0831 );
0832 #else
0833 __asm__ __volatile__
0834 (
0835 "lock; btrw %[bit_number], %[storage]\n\t"
0836 "setc %[result]\n\t"
0837 : [storage] "+m" (storage), [result] "=q" (res)
0838 : [bit_number] "Kq" ((uint16_t)bit_number)
0839 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0840 );
0841 #endif
0842 return res;
0843 }
0844
0845 static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
0846 {
0847 bool res;
0848 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
0849 __asm__ __volatile__
0850 (
0851 "lock; btcw %[bit_number], %[storage]\n\t"
0852 : [storage] "+m" (storage), [result] "=@ccc" (res)
0853 : [bit_number] "Kq" ((uint16_t)bit_number)
0854 : "memory"
0855 );
0856 #else
0857 __asm__ __volatile__
0858 (
0859 "lock; btcw %[bit_number], %[storage]\n\t"
0860 "setc %[result]\n\t"
0861 : [storage] "+m" (storage), [result] "=q" (res)
0862 : [bit_number] "Kq" ((uint16_t)bit_number)
0863 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0864 );
0865 #endif
0866 return res;
0867 }
0868 };
0869
0870 template< typename Base, bool Signed >
0871 struct extra_operations< Base, 4u, Signed, true > :
0872 public extra_operations_generic< Base, 4u, Signed >
0873 {
0874 typedef extra_operations_generic< Base, 4u, Signed > base_type;
0875 typedef typename base_type::storage_type storage_type;
0876
0877 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\
0878 __asm__ __volatile__\
0879 (\
0880 ".align 16\n\t"\
0881 "1: mov %[orig], %[res]\n\t"\
0882 op " %[res]\n\t"\
0883 "lock; cmpxchgl %[res], %[storage]\n\t"\
0884 "jne 1b"\
0885 : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\
0886 : \
0887 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0888 )
0889
0890 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0891 {
0892 storage_type original = storage;
0893 storage_type result;
0894 BOOST_ATOMIC_DETAIL_CAS_LOOP("negl", original, result);
0895 return original;
0896 }
0897
0898 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0899 {
0900 storage_type original = storage;
0901 storage_type result;
0902 BOOST_ATOMIC_DETAIL_CAS_LOOP("notl", original, result);
0903 return original;
0904 }
0905
0906 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0907 {
0908 storage_type original = storage;
0909 storage_type result;
0910 BOOST_ATOMIC_DETAIL_CAS_LOOP("negl", original, result);
0911 return result;
0912 }
0913
0914 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
0915 {
0916 storage_type original = storage;
0917 storage_type result;
0918 BOOST_ATOMIC_DETAIL_CAS_LOOP("notl", original, result);
0919 return result;
0920 }
0921
0922 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0923
0924 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\
0925 __asm__ __volatile__\
0926 (\
0927 ".align 16\n\t"\
0928 "1: mov %[arg], %[res]\n\t"\
0929 op " %%eax, %[res]\n\t"\
0930 "lock; cmpxchgl %[res], %[storage]\n\t"\
0931 "jne 1b"\
0932 : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\
0933 : [arg] "ir" (argument)\
0934 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
0935 )
0936
0937 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0938 {
0939 storage_type original = storage;
0940 storage_type result;
0941 BOOST_ATOMIC_DETAIL_CAS_LOOP("andl", v, original, result);
0942 return static_cast< storage_type >(result);
0943 }
0944
0945 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0946 {
0947 storage_type original = storage;
0948 storage_type result;
0949 BOOST_ATOMIC_DETAIL_CAS_LOOP("orl", v, original, result);
0950 return static_cast< storage_type >(result);
0951 }
0952
0953 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0954 {
0955 storage_type original = storage;
0956 storage_type result;
0957 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorl", v, original, result);
0958 return static_cast< storage_type >(result);
0959 }
0960
0961 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
0962
0963 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0964 {
0965 return !!negate(storage, order);
0966 }
0967
0968 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
0969 {
0970 return !!bitwise_complement(storage, order);
0971 }
0972
0973 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0974 {
0975 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
0976 {
0977 __asm__ __volatile__
0978 (
0979 "lock; incl %[storage]\n\t"
0980 : [storage] "+m" (storage)
0981 :
0982 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0983 );
0984 }
0985 else
0986 {
0987 __asm__ __volatile__
0988 (
0989 "lock; addl %[argument], %[storage]\n\t"
0990 : [storage] "+m" (storage)
0991 : [argument] "ir" (v)
0992 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
0993 );
0994 }
0995 }
0996
0997 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
0998 {
0999 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1000 {
1001 __asm__ __volatile__
1002 (
1003 "lock; decl %[storage]\n\t"
1004 : [storage] "+m" (storage)
1005 :
1006 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1007 );
1008 }
1009 else
1010 {
1011 __asm__ __volatile__
1012 (
1013 "lock; subl %[argument], %[storage]\n\t"
1014 : [storage] "+m" (storage)
1015 : [argument] "ir" (v)
1016 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1017 );
1018 }
1019 }
1020
1021 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1022 {
1023 __asm__ __volatile__
1024 (
1025 "lock; negl %[storage]\n\t"
1026 : [storage] "+m" (storage)
1027 :
1028 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1029 );
1030 }
1031
1032 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1033 {
1034 __asm__ __volatile__
1035 (
1036 "lock; andl %[argument], %[storage]\n\t"
1037 : [storage] "+m" (storage)
1038 : [argument] "ir" (v)
1039 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1040 );
1041 }
1042
1043 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1044 {
1045 __asm__ __volatile__
1046 (
1047 "lock; orl %[argument], %[storage]\n\t"
1048 : [storage] "+m" (storage)
1049 : [argument] "ir" (v)
1050 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1051 );
1052 }
1053
1054 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1055 {
1056 __asm__ __volatile__
1057 (
1058 "lock; xorl %[argument], %[storage]\n\t"
1059 : [storage] "+m" (storage)
1060 : [argument] "ir" (v)
1061 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1062 );
1063 }
1064
1065 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1066 {
1067 __asm__ __volatile__
1068 (
1069 "lock; notl %[storage]\n\t"
1070 : [storage] "+m" (storage)
1071 :
1072 : "memory"
1073 );
1074 }
1075
1076 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1077 {
1078 bool res;
1079 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1080 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1081 {
1082 __asm__ __volatile__
1083 (
1084 "lock; incl %[storage]\n\t"
1085 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1086 :
1087 : "memory"
1088 );
1089 }
1090 else
1091 {
1092 __asm__ __volatile__
1093 (
1094 "lock; addl %[argument], %[storage]\n\t"
1095 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1096 : [argument] "ir" (v)
1097 : "memory"
1098 );
1099 }
1100 #else
1101 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1102 {
1103 __asm__ __volatile__
1104 (
1105 "lock; incl %[storage]\n\t"
1106 "setnz %[result]\n\t"
1107 : [storage] "+m" (storage), [result] "=q" (res)
1108 :
1109 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1110 );
1111 }
1112 else
1113 {
1114 __asm__ __volatile__
1115 (
1116 "lock; addl %[argument], %[storage]\n\t"
1117 "setnz %[result]\n\t"
1118 : [storage] "+m" (storage), [result] "=q" (res)
1119 : [argument] "ir" (v)
1120 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1121 );
1122 }
1123 #endif
1124 return res;
1125 }
1126
1127 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1128 {
1129 bool res;
1130 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1131 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1132 {
1133 __asm__ __volatile__
1134 (
1135 "lock; decl %[storage]\n\t"
1136 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1137 :
1138 : "memory"
1139 );
1140 }
1141 else
1142 {
1143 __asm__ __volatile__
1144 (
1145 "lock; subl %[argument], %[storage]\n\t"
1146 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1147 : [argument] "ir" (v)
1148 : "memory"
1149 );
1150 }
1151 #else
1152 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1153 {
1154 __asm__ __volatile__
1155 (
1156 "lock; decl %[storage]\n\t"
1157 "setnz %[result]\n\t"
1158 : [storage] "+m" (storage), [result] "=q" (res)
1159 :
1160 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1161 );
1162 }
1163 else
1164 {
1165 __asm__ __volatile__
1166 (
1167 "lock; subl %[argument], %[storage]\n\t"
1168 "setnz %[result]\n\t"
1169 : [storage] "+m" (storage), [result] "=q" (res)
1170 : [argument] "ir" (v)
1171 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1172 );
1173 }
1174 #endif
1175 return res;
1176 }
1177
1178 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1179 {
1180 bool res;
1181 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1182 __asm__ __volatile__
1183 (
1184 "lock; andl %[argument], %[storage]\n\t"
1185 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1186 : [argument] "ir" (v)
1187 : "memory"
1188 );
1189 #else
1190 __asm__ __volatile__
1191 (
1192 "lock; andl %[argument], %[storage]\n\t"
1193 "setnz %[result]\n\t"
1194 : [storage] "+m" (storage), [result] "=q" (res)
1195 : [argument] "ir" (v)
1196 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1197 );
1198 #endif
1199 return res;
1200 }
1201
1202 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1203 {
1204 bool res;
1205 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1206 __asm__ __volatile__
1207 (
1208 "lock; orl %[argument], %[storage]\n\t"
1209 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1210 : [argument] "ir" (v)
1211 : "memory"
1212 );
1213 #else
1214 __asm__ __volatile__
1215 (
1216 "lock; orl %[argument], %[storage]\n\t"
1217 "setnz %[result]\n\t"
1218 : [storage] "+m" (storage), [result] "=q" (res)
1219 : [argument] "ir" (v)
1220 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1221 );
1222 #endif
1223 return res;
1224 }
1225
1226 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1227 {
1228 bool res;
1229 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1230 __asm__ __volatile__
1231 (
1232 "lock; xorl %[argument], %[storage]\n\t"
1233 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1234 : [argument] "ir" (v)
1235 : "memory"
1236 );
1237 #else
1238 __asm__ __volatile__
1239 (
1240 "lock; xorl %[argument], %[storage]\n\t"
1241 "setnz %[result]\n\t"
1242 : [storage] "+m" (storage), [result] "=q" (res)
1243 : [argument] "ir" (v)
1244 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1245 );
1246 #endif
1247 return res;
1248 }
1249
1250 static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1251 {
1252 bool res;
1253 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1254 __asm__ __volatile__
1255 (
1256 "lock; btsl %[bit_number], %[storage]\n\t"
1257 : [storage] "+m" (storage), [result] "=@ccc" (res)
1258 : [bit_number] "Kr" ((uint32_t)bit_number)
1259 : "memory"
1260 );
1261 #else
1262 __asm__ __volatile__
1263 (
1264 "lock; btsl %[bit_number], %[storage]\n\t"
1265 "setc %[result]\n\t"
1266 : [storage] "+m" (storage), [result] "=q" (res)
1267 : [bit_number] "Kr" ((uint32_t)bit_number)
1268 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1269 );
1270 #endif
1271 return res;
1272 }
1273
1274 static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1275 {
1276 bool res;
1277 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1278 __asm__ __volatile__
1279 (
1280 "lock; btrl %[bit_number], %[storage]\n\t"
1281 : [storage] "+m" (storage), [result] "=@ccc" (res)
1282 : [bit_number] "Kr" ((uint32_t)bit_number)
1283 : "memory"
1284 );
1285 #else
1286 __asm__ __volatile__
1287 (
1288 "lock; btrl %[bit_number], %[storage]\n\t"
1289 "setc %[result]\n\t"
1290 : [storage] "+m" (storage), [result] "=q" (res)
1291 : [bit_number] "Kr" ((uint32_t)bit_number)
1292 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1293 );
1294 #endif
1295 return res;
1296 }
1297
1298 static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1299 {
1300 bool res;
1301 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1302 __asm__ __volatile__
1303 (
1304 "lock; btcl %[bit_number], %[storage]\n\t"
1305 : [storage] "+m" (storage), [result] "=@ccc" (res)
1306 : [bit_number] "Kr" ((uint32_t)bit_number)
1307 : "memory"
1308 );
1309 #else
1310 __asm__ __volatile__
1311 (
1312 "lock; btcl %[bit_number], %[storage]\n\t"
1313 "setc %[result]\n\t"
1314 : [storage] "+m" (storage), [result] "=q" (res)
1315 : [bit_number] "Kr" ((uint32_t)bit_number)
1316 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1317 );
1318 #endif
1319 return res;
1320 }
1321 };
1322
1323 #if defined(__x86_64__)
1324
1325 template< typename Base, bool Signed >
1326 struct extra_operations< Base, 8u, Signed, true > :
1327 public extra_operations_generic< Base, 8u, Signed >
1328 {
1329 typedef extra_operations_generic< Base, 8u, Signed > base_type;
1330 typedef typename base_type::storage_type storage_type;
1331
1332 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\
1333 __asm__ __volatile__\
1334 (\
1335 ".align 16\n\t"\
1336 "1: mov %[orig], %[res]\n\t"\
1337 op " %[res]\n\t"\
1338 "lock; cmpxchgq %[res], %[storage]\n\t"\
1339 "jne 1b"\
1340 : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\
1341 : \
1342 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1343 )
1344
1345 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1346 {
1347 storage_type original = storage;
1348 storage_type result;
1349 BOOST_ATOMIC_DETAIL_CAS_LOOP("negq", original, result);
1350 return original;
1351 }
1352
1353 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1354 {
1355 storage_type original = storage;
1356 storage_type result;
1357 BOOST_ATOMIC_DETAIL_CAS_LOOP("notq", original, result);
1358 return original;
1359 }
1360
1361 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1362 {
1363 storage_type original = storage;
1364 storage_type result;
1365 BOOST_ATOMIC_DETAIL_CAS_LOOP("negq", original, result);
1366 return result;
1367 }
1368
1369 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1370 {
1371 storage_type original = storage;
1372 storage_type result;
1373 BOOST_ATOMIC_DETAIL_CAS_LOOP("notq", original, result);
1374 return result;
1375 }
1376
1377 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
1378
1379 #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\
1380 __asm__ __volatile__\
1381 (\
1382 ".align 16\n\t"\
1383 "1: mov %[arg], %[res]\n\t"\
1384 op " %%rax, %[res]\n\t"\
1385 "lock; cmpxchgq %[res], %[storage]\n\t"\
1386 "jne 1b"\
1387 : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\
1388 : [arg] "r" (argument)\
1389 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\
1390 )
1391
1392 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1393 {
1394 storage_type original = storage;
1395 storage_type result;
1396 BOOST_ATOMIC_DETAIL_CAS_LOOP("andq", v, original, result);
1397 return static_cast< storage_type >(result);
1398 }
1399
1400 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1401 {
1402 storage_type original = storage;
1403 storage_type result;
1404 BOOST_ATOMIC_DETAIL_CAS_LOOP("orq", v, original, result);
1405 return static_cast< storage_type >(result);
1406 }
1407
1408 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1409 {
1410 storage_type original = storage;
1411 storage_type result;
1412 BOOST_ATOMIC_DETAIL_CAS_LOOP("xorq", v, original, result);
1413 return static_cast< storage_type >(result);
1414 }
1415
1416 #undef BOOST_ATOMIC_DETAIL_CAS_LOOP
1417
1418 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1419 {
1420 return !!negate(storage, order);
1421 }
1422
1423 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1424 {
1425 return !!bitwise_complement(storage, order);
1426 }
1427
1428 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1429 {
1430 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1431 {
1432 __asm__ __volatile__
1433 (
1434 "lock; incq %[storage]\n\t"
1435 : [storage] "+m" (storage)
1436 :
1437 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1438 );
1439 }
1440 else
1441 {
1442 __asm__ __volatile__
1443 (
1444 "lock; addq %[argument], %[storage]\n\t"
1445 : [storage] "+m" (storage)
1446 : [argument] "er" (v)
1447 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1448 );
1449 }
1450 }
1451
1452 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1453 {
1454 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1455 {
1456 __asm__ __volatile__
1457 (
1458 "lock; decq %[storage]\n\t"
1459 : [storage] "+m" (storage)
1460 :
1461 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1462 );
1463 }
1464 else
1465 {
1466 __asm__ __volatile__
1467 (
1468 "lock; subq %[argument], %[storage]\n\t"
1469 : [storage] "+m" (storage)
1470 : [argument] "er" (v)
1471 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1472 );
1473 }
1474 }
1475
1476 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1477 {
1478 __asm__ __volatile__
1479 (
1480 "lock; negq %[storage]\n\t"
1481 : [storage] "+m" (storage)
1482 :
1483 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1484 );
1485 }
1486
1487 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1488 {
1489 __asm__ __volatile__
1490 (
1491 "lock; andq %[argument], %[storage]\n\t"
1492 : [storage] "+m" (storage)
1493 : [argument] "er" (v)
1494 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1495 );
1496 }
1497
1498 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1499 {
1500 __asm__ __volatile__
1501 (
1502 "lock; orq %[argument], %[storage]\n\t"
1503 : [storage] "+m" (storage)
1504 : [argument] "er" (v)
1505 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1506 );
1507 }
1508
1509 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1510 {
1511 __asm__ __volatile__
1512 (
1513 "lock; xorq %[argument], %[storage]\n\t"
1514 : [storage] "+m" (storage)
1515 : [argument] "er" (v)
1516 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1517 );
1518 }
1519
1520 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT
1521 {
1522 __asm__ __volatile__
1523 (
1524 "lock; notq %[storage]\n\t"
1525 : [storage] "+m" (storage)
1526 :
1527 : "memory"
1528 );
1529 }
1530
1531 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1532 {
1533 bool res;
1534 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1535 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1536 {
1537 __asm__ __volatile__
1538 (
1539 "lock; incq %[storage]\n\t"
1540 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1541 :
1542 : "memory"
1543 );
1544 }
1545 else
1546 {
1547 __asm__ __volatile__
1548 (
1549 "lock; addq %[argument], %[storage]\n\t"
1550 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1551 : [argument] "er" (v)
1552 : "memory"
1553 );
1554 }
1555 #else
1556 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1557 {
1558 __asm__ __volatile__
1559 (
1560 "lock; incq %[storage]\n\t"
1561 "setnz %[result]\n\t"
1562 : [storage] "+m" (storage), [result] "=q" (res)
1563 :
1564 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1565 );
1566 }
1567 else
1568 {
1569 __asm__ __volatile__
1570 (
1571 "lock; addq %[argument], %[storage]\n\t"
1572 "setnz %[result]\n\t"
1573 : [storage] "+m" (storage), [result] "=q" (res)
1574 : [argument] "er" (v)
1575 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1576 );
1577 }
1578 #endif
1579 return res;
1580 }
1581
1582 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1583 {
1584 bool res;
1585 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1586 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1587 {
1588 __asm__ __volatile__
1589 (
1590 "lock; decq %[storage]\n\t"
1591 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1592 :
1593 : "memory"
1594 );
1595 }
1596 else
1597 {
1598 __asm__ __volatile__
1599 (
1600 "lock; subq %[argument], %[storage]\n\t"
1601 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1602 : [argument] "er" (v)
1603 : "memory"
1604 );
1605 }
1606 #else
1607 if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1)
1608 {
1609 __asm__ __volatile__
1610 (
1611 "lock; decq %[storage]\n\t"
1612 "setnz %[result]\n\t"
1613 : [storage] "+m" (storage), [result] "=q" (res)
1614 :
1615 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1616 );
1617 }
1618 else
1619 {
1620 __asm__ __volatile__
1621 (
1622 "lock; subq %[argument], %[storage]\n\t"
1623 "setnz %[result]\n\t"
1624 : [storage] "+m" (storage), [result] "=q" (res)
1625 : [argument] "er" (v)
1626 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1627 );
1628 }
1629 #endif
1630 return res;
1631 }
1632
1633 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1634 {
1635 bool res;
1636 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1637 __asm__ __volatile__
1638 (
1639 "lock; andq %[argument], %[storage]\n\t"
1640 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1641 : [argument] "er" (v)
1642 : "memory"
1643 );
1644 #else
1645 __asm__ __volatile__
1646 (
1647 "lock; andq %[argument], %[storage]\n\t"
1648 "setnz %[result]\n\t"
1649 : [storage] "+m" (storage), [result] "=q" (res)
1650 : [argument] "er" (v)
1651 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1652 );
1653 #endif
1654 return res;
1655 }
1656
1657 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1658 {
1659 bool res;
1660 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1661 __asm__ __volatile__
1662 (
1663 "lock; orq %[argument], %[storage]\n\t"
1664 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1665 : [argument] "er" (v)
1666 : "memory"
1667 );
1668 #else
1669 __asm__ __volatile__
1670 (
1671 "lock; orq %[argument], %[storage]\n\t"
1672 "setnz %[result]\n\t"
1673 : [storage] "+m" (storage), [result] "=q" (res)
1674 : [argument] "er" (v)
1675 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1676 );
1677 #endif
1678 return res;
1679 }
1680
1681 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
1682 {
1683 bool res;
1684 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1685 __asm__ __volatile__
1686 (
1687 "lock; xorq %[argument], %[storage]\n\t"
1688 : [storage] "+m" (storage), [result] "=@ccnz" (res)
1689 : [argument] "er" (v)
1690 : "memory"
1691 );
1692 #else
1693 __asm__ __volatile__
1694 (
1695 "lock; xorq %[argument], %[storage]\n\t"
1696 "setnz %[result]\n\t"
1697 : [storage] "+m" (storage), [result] "=q" (res)
1698 : [argument] "er" (v)
1699 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1700 );
1701 #endif
1702 return res;
1703 }
1704
1705 static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1706 {
1707 bool res;
1708 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1709 __asm__ __volatile__
1710 (
1711 "lock; btsq %[bit_number], %[storage]\n\t"
1712 : [storage] "+m" (storage), [result] "=@ccc" (res)
1713 : [bit_number] "Kr" ((uint64_t)bit_number)
1714 : "memory"
1715 );
1716 #else
1717 __asm__ __volatile__
1718 (
1719 "lock; btsq %[bit_number], %[storage]\n\t"
1720 "setc %[result]\n\t"
1721 : [storage] "+m" (storage), [result] "=q" (res)
1722 : [bit_number] "Kr" ((uint64_t)bit_number)
1723 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1724 );
1725 #endif
1726 return res;
1727 }
1728
1729 static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1730 {
1731 bool res;
1732 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1733 __asm__ __volatile__
1734 (
1735 "lock; btrq %[bit_number], %[storage]\n\t"
1736 : [storage] "+m" (storage), [result] "=@ccc" (res)
1737 : [bit_number] "Kr" ((uint64_t)bit_number)
1738 : "memory"
1739 );
1740 #else
1741 __asm__ __volatile__
1742 (
1743 "lock; btrq %[bit_number], %[storage]\n\t"
1744 "setc %[result]\n\t"
1745 : [storage] "+m" (storage), [result] "=q" (res)
1746 : [bit_number] "Kr" ((uint64_t)bit_number)
1747 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1748 );
1749 #endif
1750 return res;
1751 }
1752
1753 static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1754 {
1755 bool res;
1756 #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS)
1757 __asm__ __volatile__
1758 (
1759 "lock; btcq %[bit_number], %[storage]\n\t"
1760 : [storage] "+m" (storage), [result] "=@ccc" (res)
1761 : [bit_number] "Kr" ((uint64_t)bit_number)
1762 : "memory"
1763 );
1764 #else
1765 __asm__ __volatile__
1766 (
1767 "lock; btcq %[bit_number], %[storage]\n\t"
1768 "setc %[result]\n\t"
1769 : [storage] "+m" (storage), [result] "=q" (res)
1770 : [bit_number] "Kr" ((uint64_t)bit_number)
1771 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
1772 );
1773 #endif
1774 return res;
1775 }
1776 };
1777
1778 #endif
1779
1780 }
1781 }
1782 }
1783
1784 #include <boost/atomic/detail/footer.hpp>
1785
1786 #endif
