xsimd/arch/xsimd_avx_128.hpp

0001 /***************************************************************************
0002  * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
0003  * Martin Renou                                                             *
0004  * Copyright (c) QuantStack                                                 *
0005  * Copyright (c) Serge Guelton                                              *
0006  * Copyright (c) Marco Barbone                                              *
0007  *                                                                          *
0008  * Distributed under the terms of the BSD 3-Clause License.                 *
0009  *                                                                          *
0010  * The full license is in the file LICENSE, distributed with this software. *
0011  ****************************************************************************/
0012
0013 #ifndef XSIMD_AVX_128_HPP
0014 #define XSIMD_AVX_128_HPP
0015
0016 #include "../types/xsimd_avx_register.hpp"
0017 #include "../types/xsimd_batch_constant.hpp"
0018
0019 #include <type_traits>
0020
0021 namespace xsimd
0022 {
0023     namespace kernel
0024     {
0025         using namespace types;
0026
0027         // broadcast
0028         template <class A, class T, class = std::enable_if_t<std::is_same<T, float>::value>>
0029         XSIMD_INLINE batch<T, A> broadcast(T val, requires_arch<avx_128>) noexcept
0030         {
0031             return _mm_broadcast_ss(&val);
0032         }
0033
0034         // eq
0035         template <class A>
0036         XSIMD_INLINE batch_bool<float, A> eq(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx_128>) noexcept
0037         {
0038             return _mm_cmp_ps(self, other, _CMP_EQ_OQ);
0039         }
0040         template <class A>
0041         XSIMD_INLINE batch_bool<double, A> eq(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx_128>) noexcept
0042         {
0043             return _mm_cmp_pd(self, other, _CMP_EQ_OQ);
0044         }
0045
0046         // gt
0047         template <class A>
0048         XSIMD_INLINE batch_bool<float, A> gt(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx_128>) noexcept
0049         {
0050             return _mm_cmp_ps(self, other, _CMP_GT_OQ);
0051         }
0052         template <class A>
0053         XSIMD_INLINE batch_bool<double, A> gt(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx_128>) noexcept
0054         {
0055             return _mm_cmp_pd(self, other, _CMP_GT_OQ);
0056         }
0057
0058         // ge
0059         template <class A>
0060         XSIMD_INLINE batch_bool<float, A> ge(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx_128>) noexcept
0061         {
0062             return _mm_cmp_ps(self, other, _CMP_GE_OQ);
0063         }
0064         template <class A>
0065         XSIMD_INLINE batch_bool<double, A> ge(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx_128>) noexcept
0066         {
0067             return _mm_cmp_pd(self, other, _CMP_GE_OQ);
0068         }
0069
0070         // lt
0071         template <class A>
0072         XSIMD_INLINE batch_bool<float, A> lt(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx_128>) noexcept
0073         {
0074             return _mm_cmp_ps(self, other, _CMP_LT_OQ);
0075         }
0076         template <class A>
0077         XSIMD_INLINE batch_bool<double, A> lt(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx_128>) noexcept
0078         {
0079             return _mm_cmp_pd(self, other, _CMP_LT_OQ);
0080         }
0081
0082         // le
0083         template <class A>
0084         XSIMD_INLINE batch_bool<float, A> le(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx_128>) noexcept
0085         {
0086             return _mm_cmp_ps(self, other, _CMP_LE_OQ);
0087         }
0088         template <class A>
0089         XSIMD_INLINE batch_bool<double, A> le(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx_128>) noexcept
0090         {
0091             return _mm_cmp_pd(self, other, _CMP_LE_OQ);
0092         }
0093
0094         // neq
0095         template <class A>
0096         XSIMD_INLINE batch_bool<float, A> neq(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx_128>) noexcept
0097         {
0098             return _mm_cmp_ps(self, other, _CMP_NEQ_UQ);
0099         }
0100         template <class A>
0101         XSIMD_INLINE batch_bool<double, A> neq(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx_128>) noexcept
0102         {
0103             return _mm_cmp_pd(self, other, _CMP_NEQ_UQ);
0104         }
0105
0106         // Masks that lower to plain moves go to sse2; the rest gain nothing on a
0107         // single register, so take the runtime path.
0108         template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_floating_point<T>::value>>
0109         XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, convert<T>, Mode, requires_arch<avx_128>) noexcept
0110         {
0111             XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
0112             {
0113                 return load_masked(mem, mask, convert<T> {}, Mode {}, sse2 {});
0114             }
0115             else
0116             {
0117                 return load_masked(mem, mask.as_batch_bool(), convert<T> {}, Mode {}, avx_128 {});
0118             }
0119         }
0120
0121         // Runtime-mask load (float/double).
0122         template <class A, class Mode>
0123         XSIMD_INLINE batch<float, A>
0124         load_masked(float const* mem, batch_bool<float, A> mask, convert<float>, Mode, requires_arch<avx_128>) noexcept
0125         {
0126             return _mm_maskload_ps(mem, _mm_castps_si128(mask));
0127         }
0128         template <class A, class Mode>
0129         XSIMD_INLINE batch<double, A>
0130         load_masked(double const* mem, batch_bool<double, A> mask, convert<double>, Mode, requires_arch<avx_128>) noexcept
0131         {
0132             return _mm_maskload_pd(mem, _mm_castpd_si128(mask));
0133         }
0134
0135         // 4/8-byte ints: bitcast to same-width float, reuse the vmaskmov path.
0136         template <class A, class T, class Mode>
0137         XSIMD_INLINE std::enable_if_t<std::is_integral<T>::value && (sizeof(T) == 4 || sizeof(T) == 8), batch<T, A>>
0138         load_masked(T const* mem, batch_bool<T, A> mask, convert<T>, Mode, requires_arch<avx_128>) noexcept
0139         {
0140             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0141             {
0142                 return bitwise_cast<T>(batch<float, A>(_mm_maskload_ps(reinterpret_cast<float const*>(mem), __m128i(mask))));
0143             }
0144             else
0145             {
0146                 return bitwise_cast<T>(batch<double, A>(_mm_maskload_pd(reinterpret_cast<double const*>(mem), __m128i(mask))));
0147             }
0148         }
0149
0150         template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_floating_point<T>::value>>
0151         XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool_constant<T, A, Values...> mask, Mode, requires_arch<avx_128>) noexcept
0152         {
0153             XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
0154             {
0155                 store_masked(mem, src, mask, Mode {}, sse2 {});
0156             }
0157             else
0158             {
0159                 store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx_128 {});
0160             }
0161         }
0162
0163         // Runtime-mask store (float/double).
0164         template <class A, class Mode>
0165         XSIMD_INLINE void
0166         store_masked(float* mem, batch<float, A> const& src, batch_bool<float, A> mask, Mode, requires_arch<avx_128>) noexcept
0167         {
0168             _mm_maskstore_ps(mem, _mm_castps_si128(mask), src);
0169         }
0170         template <class A, class Mode>
0171         XSIMD_INLINE void
0172         store_masked(double* mem, batch<double, A> const& src, batch_bool<double, A> mask, Mode, requires_arch<avx_128>) noexcept
0173         {
0174             _mm_maskstore_pd(mem, _mm_castpd_si128(mask), src);
0175         }
0176
0177         // 4/8-byte ints: bitcast to same-width float, reuse the vmaskmov path.
0178         template <class A, class T, class Mode>
0179         XSIMD_INLINE std::enable_if_t<std::is_integral<T>::value && (sizeof(T) == 4 || sizeof(T) == 8), void>
0180         store_masked(T* mem, batch<T, A> const& src, batch_bool<T, A> mask, Mode, requires_arch<avx_128>) noexcept
0181         {
0182             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0183             {
0184                 _mm_maskstore_ps(reinterpret_cast<float*>(mem), __m128i(mask), bitwise_cast<float>(src));
0185             }
0186             else
0187             {
0188                 _mm_maskstore_pd(reinterpret_cast<double*>(mem), __m128i(mask), bitwise_cast<double>(src));
0189             }
0190         }
0191
0192         // swizzle (dynamic mask)
0193         template <class A, class ITy>
0194         XSIMD_INLINE batch<float, A> swizzle(batch<float, A> const& self, batch<ITy, A> mask, requires_arch<avx_128>) noexcept
0195         {
0196             static_assert(sizeof(float) == sizeof(ITy), "index type must match value width");
0197             return _mm_permutevar_ps(self, mask);
0198         }
0199         template <class A, class ITy>
0200         XSIMD_INLINE batch<double, A> swizzle(batch<double, A> const& self, batch<ITy, A> mask, requires_arch<avx_128>) noexcept
0201         {
0202             static_assert(sizeof(double) == sizeof(ITy), "index type must match value width");
0203             // VPERMILPD's variable control reads bit 1 of each 64-bit selector
0204             // (bit 0 is ignored), so a {0,1} index needs to become {0,2}.
0205             // Negation is a cheap alternative to a left shift by 1.
0206             return _mm_permutevar_pd(self, -mask);
0207         }
0208
0209         // swizzle (constant mask)
0210         template <class A, uint32_t V0, uint32_t V1, uint32_t V2, uint32_t V3>
0211         XSIMD_INLINE batch<float, A> swizzle(batch<float, A> const& self, batch_constant<uint32_t, A, V0, V1, V2, V3>, requires_arch<avx_128>) noexcept
0212         {
0213             return _mm_permute_ps(self, detail::mod_shuffle(V0, V1, V2, V3));
0214         }
0215
0216         template <class A, uint32_t V0, uint32_t V1>
0217         XSIMD_INLINE batch<double, A> swizzle(batch<double, A> const& self, batch_constant<uint64_t, A, V0, V1>, requires_arch<avx_128>) noexcept
0218         {
0219             return _mm_permute_pd(self, detail::mod_shuffle(V0, V1));
0220         }
0221
0222     }
0223 }
0224
0225 #endif