File indexing completed on 2025-12-16 10:33:09
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025
0026
0027
0028 #ifndef VC_COMMON_SIMDARRAYHELPER_H_
0029 #define VC_COMMON_SIMDARRAYHELPER_H_
0030
0031 #include "macros.h"
0032
0033 namespace Vc_VERSIONED_NAMESPACE
0034 {
0035
0036 namespace
0037 {
0038 static constexpr struct private_init_t {} private_init = {};
0039 }
0040
0041
0042 namespace Common
0043 {
0044
0045
0046
0047
0048 namespace Operations
0049 {
0050 struct tag {};
0051 #define Vc_DEFINE_OPERATION(name_) \
0052 struct name_ : public tag { \
0053 template <typename V, typename... Args> \
0054 Vc_INTRINSIC void operator()(V &v, Args &&... args) \
0055 { \
0056 v.name_(std::forward<Args>(args)...); \
0057 } \
0058 }
0059 Vc_DEFINE_OPERATION(gather);
0060 Vc_DEFINE_OPERATION(scatter);
0061 Vc_DEFINE_OPERATION(load);
0062 Vc_DEFINE_OPERATION(store);
0063 Vc_DEFINE_OPERATION(setZero);
0064 Vc_DEFINE_OPERATION(setZeroInverted);
0065 Vc_DEFINE_OPERATION(assign);
0066 #undef Vc_DEFINE_OPERATION
0067 #define Vc_DEFINE_OPERATION(name_, code_) \
0068 struct name_ : public tag { \
0069 template <typename V> Vc_INTRINSIC void operator()(V &v) { code_; } \
0070 }
0071 Vc_DEFINE_OPERATION(increment, ++(v));
0072 Vc_DEFINE_OPERATION(decrement, --(v));
0073 Vc_DEFINE_OPERATION(random, v = V::Random());
0074 #undef Vc_DEFINE_OPERATION
0075 #define Vc_DEFINE_OPERATION_FORWARD(name_) \
0076 struct Forward_##name_ : public tag \
0077 { \
0078 template <typename... Args, typename = decltype(name_(std::declval<Args>()...))> \
0079 Vc_INTRINSIC void operator()(decltype(name_(std::declval<Args>()...)) &v, \
0080 Args &&... args) \
0081 { \
0082 v = name_(std::forward<Args>(args)...); \
0083 } \
0084 template <typename... Args, typename = decltype(name_(std::declval<Args>()...))> \
0085 Vc_INTRINSIC void operator()(std::nullptr_t, Args && ... args) \
0086 { \
0087 name_(std::forward<Args>(args)...); \
0088 } \
0089 }
0090 Vc_DEFINE_OPERATION_FORWARD(abs);
0091 Vc_DEFINE_OPERATION_FORWARD(asin);
0092 Vc_DEFINE_OPERATION_FORWARD(atan);
0093 Vc_DEFINE_OPERATION_FORWARD(atan2);
0094 Vc_DEFINE_OPERATION_FORWARD(cos);
0095 Vc_DEFINE_OPERATION_FORWARD(ceil);
0096 Vc_DEFINE_OPERATION_FORWARD(copysign);
0097 Vc_DEFINE_OPERATION_FORWARD(exp);
0098 Vc_DEFINE_OPERATION_FORWARD(exponent);
0099 Vc_DEFINE_OPERATION_FORWARD(fma);
0100 Vc_DEFINE_OPERATION_FORWARD(floor);
0101 Vc_DEFINE_OPERATION_FORWARD(frexp);
0102 Vc_DEFINE_OPERATION_FORWARD(isfinite);
0103 Vc_DEFINE_OPERATION_FORWARD(isinf);
0104 Vc_DEFINE_OPERATION_FORWARD(isnan);
0105 Vc_DEFINE_OPERATION_FORWARD(isnegative);
0106 Vc_DEFINE_OPERATION_FORWARD(ldexp);
0107 Vc_DEFINE_OPERATION_FORWARD(log);
0108 Vc_DEFINE_OPERATION_FORWARD(log10);
0109 Vc_DEFINE_OPERATION_FORWARD(log2);
0110 Vc_DEFINE_OPERATION_FORWARD(reciprocal);
0111 Vc_DEFINE_OPERATION_FORWARD(round);
0112 Vc_DEFINE_OPERATION_FORWARD(rsqrt);
0113 Vc_DEFINE_OPERATION_FORWARD(sin);
0114 Vc_DEFINE_OPERATION_FORWARD(sincos);
0115 Vc_DEFINE_OPERATION_FORWARD(sqrt);
0116 Vc_DEFINE_OPERATION_FORWARD(trunc);
0117 Vc_DEFINE_OPERATION_FORWARD(min);
0118 Vc_DEFINE_OPERATION_FORWARD(max);
0119 #undef Vc_DEFINE_OPERATION_FORWARD
0120 template<typename T> using is_operation = std::is_base_of<tag, T>;
0121 }
0122
0123
0124
0125
0126
0127
0128
0129
0130 template <typename T_, std::size_t Pieces_, std::size_t Index_> struct Segment
0131 {
0132 static_assert(Index_ < Pieces_, "You found a bug in Vc. Please report.");
0133
0134 using type = T_;
0135 using type_decayed = typename std::decay<type>::type;
0136 static constexpr std::size_t Pieces = Pieces_;
0137 static constexpr std::size_t Index = Index_;
0138 using fixed_size_type =
0139 fixed_size_simd<conditional_t<Traits::is_simd_vector<type_decayed>::value,
0140 typename type_decayed::EntryType, float>,
0141 type_decayed::Size / Pieces>;
0142
0143 type data;
0144
0145 static constexpr std::size_t EntryOffset = Index * type_decayed::Size / Pieces;
0146
0147
0148 decltype(std::declval<const type &>()[0]) operator[](size_t i) const { return data[i + EntryOffset]; }
0149
0150 fixed_size_type to_fixed_size() const
0151 {
0152 return simd_cast<fixed_size_type, Index>(data);
0153 }
0154 };
0155
0156
0157 template <typename T_, std::size_t Pieces_, std::size_t Index_>
0158 struct Segment<T_ *, Pieces_, Index_> {
0159 static_assert(Index_ < Pieces_, "You found a bug in Vc. Please report.");
0160
0161 using type = T_ *;
0162 using type_decayed = typename std::decay<T_>::type;
0163 static constexpr size_t Pieces = Pieces_;
0164 static constexpr size_t Index = Index_;
0165 using fixed_size_type = fixed_size_simd<
0166 typename std::conditional<Traits::is_simd_vector<type_decayed>::value,
0167 typename type_decayed::VectorEntryType, float>::type,
0168 type_decayed::Size / Pieces> *;
0169
0170 type data;
0171
0172 static constexpr std::size_t EntryOffset = Index * type_decayed::size() / Pieces;
0173
0174 fixed_size_type to_fixed_size() const
0175 {
0176 return reinterpret_cast<
0177 #ifdef Vc_GCC
0178
0179
0180 typename std::remove_pointer<fixed_size_type>::type
0181 #else
0182 MayAlias<typename std::remove_pointer<fixed_size_type>::type>
0183 #endif
0184 *>(data) +
0185 Index;
0186 }
0187
0188
0189
0190 };
0191
0192
0193
0194
0195
0196
0197
0198
0199
0200
0201 template <typename T, std::size_t Offset> struct AddOffset
0202 {
0203 constexpr AddOffset() = default;
0204 };
0205
0206
0207
0208
0209
0210
0211
0212
0213
0214
0215 template <std::size_t secondOffset> class Split
0216 {
0217
0218 template <typename U, std::size_t N, typename V, std::size_t M,
0219 typename = enable_if<N != M>>
0220 static Vc_INTRINSIC auto loImpl(const SimdArray<U, N, V, M> &x)
0221 -> decltype(internal_data0(x))
0222 {
0223 return internal_data0(x);
0224 }
0225 template <typename U, std::size_t N, typename V, std::size_t M,
0226 typename = enable_if<N != M>>
0227 static Vc_INTRINSIC auto hiImpl(const SimdArray<U, N, V, M> &x)
0228 -> decltype(internal_data1(x))
0229 {
0230 return internal_data1(x);
0231 }
0232 template <typename U, std::size_t N, typename V, std::size_t M,
0233 typename = enable_if<N != M>>
0234 static Vc_INTRINSIC auto loImpl(SimdArray<U, N, V, M> *x)
0235 -> decltype(&internal_data0(*x))
0236 {
0237 return &internal_data0(*x);
0238 }
0239 template <typename U, std::size_t N, typename V, std::size_t M,
0240 typename = enable_if<N != M>>
0241 static Vc_INTRINSIC auto hiImpl(SimdArray<U, N, V, M> *x)
0242 -> decltype(&internal_data1(*x))
0243 {
0244 return &internal_data1(*x);
0245 }
0246
0247
0248 template <typename U, std::size_t N, typename V>
0249 static Vc_INTRINSIC Segment<V, 2, 0> loImpl(const SimdArray<U, N, V, N> &x)
0250 {
0251 return {internal_data(x)};
0252 }
0253 template <typename U, std::size_t N, typename V>
0254 static Vc_INTRINSIC Segment<V, 2, 1> hiImpl(const SimdArray<U, N, V, N> &x)
0255 {
0256 return {internal_data(x)};
0257 }
0258 template <typename U, std::size_t N, typename V>
0259 static Vc_INTRINSIC Segment<V *, 2, 0> loImpl(SimdArray<U, N, V, N> *x)
0260 {
0261 return {&internal_data(*x)};
0262 }
0263 template <typename U, std::size_t N, typename V>
0264 static Vc_INTRINSIC Segment<V *, 2, 1> hiImpl(SimdArray<U, N, V, N> *x)
0265 {
0266 return {&internal_data(*x)};
0267 }
0268
0269
0270 template <typename U, std::size_t N, typename V, std::size_t M>
0271 static Vc_INTRINSIC auto loImpl(const SimdMaskArray<U, N, V, M> &x) -> decltype(internal_data0(x))
0272 {
0273 return internal_data0(x);
0274 }
0275 template <typename U, std::size_t N, typename V, std::size_t M>
0276 static Vc_INTRINSIC auto hiImpl(const SimdMaskArray<U, N, V, M> &x) -> decltype(internal_data1(x))
0277 {
0278 return internal_data1(x);
0279 }
0280
0281 template <typename U, std::size_t N, typename V>
0282 static Vc_INTRINSIC Segment<typename SimdMaskArray<U, N, V, N>::mask_type, 2, 0> loImpl(
0283 const SimdMaskArray<U, N, V, N> &x)
0284 {
0285 return {internal_data(x)};
0286 }
0287 template <typename U, std::size_t N, typename V>
0288 static Vc_INTRINSIC Segment<typename SimdMaskArray<U, N, V, N>::mask_type, 2, 1> hiImpl(
0289 const SimdMaskArray<U, N, V, N> &x)
0290 {
0291 return {internal_data(x)};
0292 }
0293
0294
0295 #ifdef Vc_IMPL_AVX
0296 template <class T>
0297 static Vc_INTRINSIC SSE::Vector<T> loImpl(Vector<T, VectorAbi::Avx> &&x)
0298 {
0299 return simd_cast<SSE::Vector<T>, 0>(x);
0300 }
0301 template <class T>
0302 static Vc_INTRINSIC SSE::Vector<T> hiImpl(Vector<T, VectorAbi::Avx> &&x)
0303 {
0304 return simd_cast<SSE::Vector<T>, 1>(x);
0305 }
0306 template <class T>
0307 static Vc_INTRINSIC SSE::Mask<T> loImpl(Mask<T, VectorAbi::Avx> &&x)
0308 {
0309 return simd_cast<SSE::Mask<T>, 0>(x);
0310 }
0311 template <class T>
0312 static Vc_INTRINSIC SSE::Mask<T> hiImpl(Mask<T, VectorAbi::Avx> &&x)
0313 {
0314 return simd_cast<SSE::Mask<T>, 1>(x);
0315 }
0316 #endif
0317 template <typename T>
0318 static constexpr bool is_vector_or_mask(){
0319 return (Traits::is_simd_vector<T>::value && !Traits::isSimdArray<T>::value) ||
0320 (Traits::is_simd_mask<T>::value && !Traits::isSimdMaskArray<T>::value);
0321 }
0322 template <typename V>
0323 static Vc_INTRINSIC Segment<V, 2, 0> loImpl(V &&x, enable_if<is_vector_or_mask<V>()> = nullarg)
0324 {
0325 return {std::forward<V>(x)};
0326 }
0327 template <typename V>
0328 static Vc_INTRINSIC Segment<V, 2, 1> hiImpl(V &&x, enable_if<is_vector_or_mask<V>()> = nullarg)
0329 {
0330 return {std::forward<V>(x)};
0331 }
0332
0333
0334 template <class T, class A>
0335 static Vc_INTRINSIC const T *loImpl(const std::vector<T, A> &x)
0336 {
0337 return x.data();
0338 }
0339 template <class T, class A>
0340 static Vc_INTRINSIC const T *hiImpl(const std::vector<T, A> &x)
0341 {
0342 return x.data() + secondOffset;
0343 }
0344
0345
0346 template <typename V, std::size_t Pieces, std::size_t Index>
0347 static Vc_INTRINSIC Segment<V, 2 * Pieces, 2 * Index> loImpl(
0348 const Segment<V, Pieces, Index> &x)
0349 {
0350 return {x.data};
0351 }
0352 template <typename V, std::size_t Pieces, std::size_t Index>
0353 static Vc_INTRINSIC Segment<V, 2 * Pieces, 2 * Index + 1> hiImpl(
0354 const Segment<V, Pieces, Index> &x)
0355 {
0356 return {x.data};
0357 }
0358
0359
0360
0361
0362
0363 template <typename T, typename = decltype(loImpl(std::declval<T>()))>
0364 static std::true_type have_lo_impl(int);
0365 template <typename T> static std::false_type have_lo_impl(float);
0366 template <typename T> static constexpr bool have_lo_impl()
0367 {
0368 return decltype(have_lo_impl<T>(1))::value;
0369 }
0370
0371 template <typename T, typename = decltype(hiImpl(std::declval<T>()))>
0372 static std::true_type have_hi_impl(int);
0373 template <typename T> static std::false_type have_hi_impl(float);
0374 template <typename T> static constexpr bool have_hi_impl()
0375 {
0376 return decltype(have_hi_impl<T>(1))::value;
0377 }
0378
0379
0380 public:
0381
0382
0383
0384
0385
0386
0387
0388 template <typename U>
0389 static Vc_INTRINSIC const U *lo(Operations::gather, const U *ptr)
0390 {
0391 return ptr;
0392 }
0393 template <typename U>
0394 static Vc_INTRINSIC const U *hi(Operations::gather, const U *ptr)
0395 {
0396 return ptr + secondOffset;
0397 }
0398 template <typename U, typename = enable_if<!std::is_pointer<U>::value>>
0399 static Vc_ALWAYS_INLINE decltype(loImpl(std::declval<U>()))
0400 lo(Operations::gather, U &&x)
0401 {
0402 return loImpl(std::forward<U>(x));
0403 }
0404 template <typename U, typename = enable_if<!std::is_pointer<U>::value>>
0405 static Vc_ALWAYS_INLINE decltype(hiImpl(std::declval<U>()))
0406 hi(Operations::gather, U &&x)
0407 {
0408 return hiImpl(std::forward<U>(x));
0409 }
0410 template <typename U>
0411 static Vc_INTRINSIC const U *lo(Operations::scatter, const U *ptr)
0412 {
0413 return ptr;
0414 }
0415 template <typename U>
0416 static Vc_INTRINSIC const U *hi(Operations::scatter, const U *ptr)
0417 {
0418 return ptr + secondOffset;
0419 }
0420
0421
0422
0423
0424
0425
0426
0427
0428
0429
0430
0431
0432
0433 template <typename U>
0434 static Vc_ALWAYS_INLINE decltype(loImpl(std::declval<U>())) lo(U &&x)
0435 {
0436 return loImpl(std::forward<U>(x));
0437 }
0438 template <typename U>
0439 static Vc_ALWAYS_INLINE decltype(hiImpl(std::declval<U>())) hi(U &&x)
0440 {
0441 return hiImpl(std::forward<U>(x));
0442 }
0443
0444 template <typename U>
0445 static Vc_ALWAYS_INLINE enable_if<!have_lo_impl<U>(), U> lo(U &&x)
0446 {
0447 return std::forward<U>(x);
0448 }
0449 template <typename U>
0450 static Vc_ALWAYS_INLINE enable_if<!have_hi_impl<U>(), U> hi(U &&x)
0451 {
0452 return std::forward<U>(x);
0453 }
0454
0455 };
0456
0457
0458 template <typename Op, typename U, std::size_t M, typename V>
0459 static Vc_INTRINSIC const V &actual_value(Op, const SimdArray<U, M, V, M> &x)
0460 {
0461 return internal_data(x);
0462 }
0463 template <typename Op, typename U, std::size_t M, typename V>
0464 static Vc_INTRINSIC V *actual_value(Op, SimdArray<U, M, V, M> *x)
0465 {
0466 return &internal_data(*x);
0467 }
0468 template <typename Op, typename T, size_t Pieces, size_t Index>
0469 static Vc_INTRINSIC typename Segment<T, Pieces, Index>::fixed_size_type actual_value(
0470 Op, Segment<T, Pieces, Index> &&seg)
0471 {
0472 return seg.to_fixed_size();
0473 }
0474
0475 template <typename Op, typename U, std::size_t M, typename V>
0476 static Vc_INTRINSIC const typename V::Mask &actual_value(Op, const SimdMaskArray<U, M, V, M> &x)
0477 {
0478 return internal_data(x);
0479 }
0480 template <typename Op, typename U, std::size_t M, typename V>
0481 static Vc_INTRINSIC typename V::Mask *actual_value(Op, SimdMaskArray<U, M, V, M> *x)
0482 {
0483 return &internal_data(*x);
0484 }
0485
0486
0487
0488
0489
0490
0491
0492
0493
0494
0495
0496
0497
0498
0499
0500
0501
0502
0503
0504
0505 template <typename Op, typename Arg>
0506 Vc_INTRINSIC decltype(actual_value(std::declval<Op &>(), std::declval<Arg>()))
0507 conditionalUnpack(std::true_type, Op op, Arg &&arg)
0508 {
0509 return actual_value(op, std::forward<Arg>(arg));
0510 }
0511
0512 template <typename Op, typename Arg>
0513 Vc_INTRINSIC Arg conditionalUnpack(std::false_type, Op, Arg &&arg)
0514 {
0515 return std::forward<Arg>(arg);
0516 }
0517
0518
0519 template <size_t A, size_t B>
0520 struct selectorType : public std::integral_constant<bool, !((A & (size_t(1) << B)) != 0)> {
0521 };
0522
0523
0524 template <size_t I, typename Op, typename R, typename... Args, size_t... Indexes>
0525 Vc_INTRINSIC decltype(std::declval<Op &>()(std::declval<R &>(),
0526 conditionalUnpack(selectorType<I, Indexes>(),
0527 std::declval<Op &>(),
0528 std::declval<Args>())...))
0529 unpackArgumentsAutoImpl(int, index_sequence<Indexes...>, Op op, R &&r, Args &&... args)
0530 {
0531 op(std::forward<R>(r),
0532 conditionalUnpack(selectorType<I, Indexes>(), op, std::forward<Args>(args))...);
0533 }
0534
0535
0536 template <size_t I, typename Op, typename R, typename... Args, size_t... Indexes>
0537 Vc_INTRINSIC enable_if<(I <= (size_t(1) << sizeof...(Args))), void> unpackArgumentsAutoImpl(
0538 float, index_sequence<Indexes...> is, Op op, R &&r, Args &&... args)
0539 {
0540
0541
0542
0543
0544 static_assert(
0545 I < (1 << sizeof...(Args)) - (std::is_same<R, std::nullptr_t>::value ? 1 : 0),
0546 "Vc or compiler bug. Please report. Failed to find a combination of "
0547 "actual_value(arg) transformations that allows calling Op.");
0548 unpackArgumentsAutoImpl<I + 1, Op, R, Args...>(int(), is, op, std::forward<R>(r),
0549 std::forward<Args>(args)...);
0550 }
0551
0552 #ifdef Vc_ICC
0553 template <size_t, typename... Ts> struct IccWorkaround {
0554 using type = void;
0555 };
0556 template <typename... Ts> struct IccWorkaround<2, Ts...> {
0557 using type = typename std::remove_pointer<typename std::decay<
0558 typename std::tuple_element<1, std::tuple<Ts...>>::type>::type>::type;
0559 };
0560 #endif
0561
0562
0563 template <typename Op, typename R, typename... Args>
0564 Vc_INTRINSIC void unpackArgumentsAuto(Op op, R &&r, Args &&... args)
0565 {
0566 #ifdef Vc_ICC
0567
0568
0569
0570 const int recursionStart =
0571 Traits::isSimdArray<
0572 typename IccWorkaround<sizeof...(Args), Args...>::type>::value &&
0573 (std::is_same<Op, Common::Operations::Forward_frexp>::value ||
0574 std::is_same<Op, Common::Operations::Forward_ldexp>::value)
0575 ? 2
0576 : 0;
0577 #else
0578 const int recursionStart = 0;
0579 #endif
0580 unpackArgumentsAutoImpl<recursionStart>(
0581 int(), make_index_sequence<sizeof...(Args)>(), op, std::forward<R>(r),
0582 std::forward<Args>(args)...);
0583 }
0584
0585
0586
0587
0588 }
0589 }
0590
0591 #endif
0592
0593