Back to home page

EIC code displayed by LXR
 
 

    

File indexing completed on 2026-11-07 10:37:05 

0001 /***************************************************************************
0002  * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
0003  * Martin Renou                                                             *
0004  * Copyright (c) QuantStack                                                 *
0005  * Copyright (c) Serge Guelton                                              *
0006  * Copyright (c) Marco Barbone                                              *
0007  *                                                                          *
0008  * Distributed under the terms of the BSD 3-Clause License.                 *
0009  *                                                                          *
0010  * The full license is in the file LICENSE, distributed with this software. *
0011  ****************************************************************************/
0012 
0013 #ifndef XSIMD_AVX512VL_256_HPP
0014 #define XSIMD_AVX512VL_256_HPP
0015 
0016 #include "../types/xsimd_avx512vl_register.hpp"
0017 #include "../types/xsimd_batch_constant.hpp"
0018 
0019 #include <type_traits>
0020 
0021 namespace xsimd
0022 {
0023     namespace kernel
0024     {
0025         using namespace types;
0026 
0027         namespace detail
0028         {
0029             // Defined in xsimd_avx512f.hpp. This header is included before it so
0030             // that avx512f.hpp's masked load/store forwarder can resolve the
0031             // avx512vl_256 overloads below by ordinary lookup; forward-declare
0032             // the two helpers it borrows from there.
0033             XSIMD_INLINE uint32_t morton(uint16_t x, uint16_t y) noexcept;
0034             template <size_t N>
0035             XSIMD_INLINE unsigned char tobitset(unsigned char unpacked[N]);
0036 
0037             template <class A, class T, int Cmp>
0038             XSIMD_INLINE batch_bool<T, A> compare_int_avx512vl_256(batch<T, A> const& self, batch<T, A> const& other) noexcept
0039             {
0040                 using register_type = typename batch_bool<T, A>::register_type;
0041                 if (std::is_signed<T>::value)
0042                 {
0043                     XSIMD_IF_CONSTEXPR(sizeof(T) == 1)
0044                     {
0045                         // shifting to take sign into account
0046                         uint64_t mask_low0 = _mm256_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x000000FF)) << 24,
0047                                                                    (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x000000FF)) << 24,
0048                                                                    Cmp);
0049                         uint64_t mask_low1 = _mm256_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x0000FF00)) << 16,
0050                                                                    (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x0000FF00)) << 16,
0051                                                                    Cmp);
0052                         uint64_t mask_high0 = _mm256_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x00FF0000)) << 8,
0053                                                                     (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x00FF0000)) << 8,
0054                                                                     Cmp);
0055                         uint64_t mask_high1 = _mm256_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0xFF000000)),
0056                                                                     (batch<int32_t, A>(other.data) & batch<int32_t, A>(0xFF000000)),
0057                                                                     Cmp);
0058                         uint64_t mask = 0;
0059                         for (unsigned i = 0; i < 8; ++i)
0060                         {
0061                             mask |= (mask_low0 & (uint64_t(1) << i)) << (3 * i + 0);
0062                             mask |= (mask_low1 & (uint64_t(1) << i)) << (3 * i + 1);
0063                             mask |= (mask_high0 & (uint64_t(1) << i)) << (3 * i + 2);
0064                             mask |= (mask_high1 & (uint64_t(1) << i)) << (3 * i + 3);
0065                         }
0066                         return (register_type)mask;
0067                     }
0068                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
0069                     {
0070                         // shifting to take sign into account
0071                         uint16_t mask_low = _mm256_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x0000FFFF)) << 16,
0072                                                                   (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x0000FFFF)) << 16,
0073                                                                   Cmp);
0074                         uint16_t mask_high = _mm256_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0xFFFF0000)),
0075                                                                    (batch<int32_t, A>(other.data) & batch<int32_t, A>(0xFFFF0000)),
0076                                                                    Cmp);
0077                         return static_cast<register_type>(morton(mask_low, mask_high));
0078                     }
0079                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0080                     {
0081                         return (register_type)_mm256_cmp_epi32_mask(self, other, Cmp);
0082                     }
0083                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0084                     {
0085                         return (register_type)_mm256_cmp_epi64_mask(self, other, Cmp);
0086                     }
0087                 }
0088                 else
0089                 {
0090                     XSIMD_IF_CONSTEXPR(sizeof(T) == 1)
0091                     {
0092                         uint64_t mask_low0 = _mm256_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x000000FF)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x000000FF)), Cmp);
0093                         uint64_t mask_low1 = _mm256_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x0000FF00)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x0000FF00)), Cmp);
0094                         uint64_t mask_high0 = _mm256_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x00FF0000)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x00FF0000)), Cmp);
0095                         uint64_t mask_high1 = _mm256_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0xFF000000)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0xFF000000)), Cmp);
0096                         uint64_t mask = 0;
0097                         for (unsigned i = 0; i < 8; ++i)
0098                         {
0099                             mask |= (mask_low0 & (uint64_t(1) << i)) << (3 * i + 0);
0100                             mask |= (mask_low1 & (uint64_t(1) << i)) << (3 * i + 1);
0101                             mask |= (mask_high0 & (uint64_t(1) << i)) << (3 * i + 2);
0102                             mask |= (mask_high1 & (uint64_t(1) << i)) << (3 * i + 3);
0103                         }
0104                         return (register_type)mask;
0105                     }
0106                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
0107                     {
0108                         uint16_t mask_low = _mm256_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x0000FFFF)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x0000FFFF)), Cmp);
0109                         uint16_t mask_high = _mm256_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0xFFFF0000)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0xFFFF0000)), Cmp);
0110                         return static_cast<register_type>(morton(mask_low, mask_high));
0111                     }
0112                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0113                     {
0114                         return (register_type)_mm256_cmp_epu32_mask(self, other, Cmp);
0115                     }
0116                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0117                     {
0118                         return (register_type)_mm256_cmp_epu64_mask(self, other, Cmp);
0119                     }
0120                 }
0121             }
0122         }
0123 
0124         // load mask
0125         template <class A, class T>
0126         XSIMD_INLINE batch_bool<T, A> load_unaligned(bool const* mem, batch_bool<T, A>, requires_arch<avx512vl_256>) noexcept
0127         {
0128             using register_type = typename batch_bool<T, A>::register_type;
0129             constexpr auto size = batch_bool<T, A>::size;
0130             constexpr auto chunk_size = size >= 8 ? 8 : 4;
0131             constexpr auto iter = size / chunk_size;
0132             static_assert((size % chunk_size) == 0, "incorrect size of bool batch");
0133             register_type mask = 0;
0134             for (std::size_t i = 0; i < iter; ++i)
0135             {
0136                 unsigned char block = detail::tobitset<chunk_size>((unsigned char*)mem + i * chunk_size);
0137                 mask |= (register_type(block) << (i * chunk_size));
0138             }
0139             return mask;
0140         }
0141 
0142         // from bool
0143         template <class A, class T>
0144         XSIMD_INLINE batch<T, A> from_bool(batch_bool<T, A> const& self, requires_arch<avx512vl_256>) noexcept
0145         {
0146             return select(self, batch<T, A>(1), batch<T, A>(0));
0147         }
0148 
0149         // from_mask
0150         template <class T, class A>
0151         XSIMD_INLINE batch_bool<T, A> from_mask(batch_bool<T, A> const&, uint64_t mask, requires_arch<avx512vl_256>) noexcept
0152         {
0153             assert(mask == (mask & ((uint64_t(1) << batch_bool<T, A>::size) - 1)) && "inbound mask");
0154             return static_cast<typename batch_bool<T, A>::register_type>(mask & ((uint64_t(1) << batch_bool<T, A>::size) - 1));
0155         }
0156 
0157         // mask
0158         template <class A, class T>
0159         XSIMD_INLINE uint64_t mask(batch_bool<T, A> const& self, requires_arch<avx512vl_256>) noexcept
0160         {
0161             return self.data & ((uint64_t(1) << batch_bool<T, A>::size) - 1);
0162         }
0163 
0164         // batch_bool_cast
0165         template <class A, class T_out, class T_in>
0166         XSIMD_INLINE batch_bool<T_out, A> batch_bool_cast(batch_bool<T_in, A> const& self, batch_bool<T_out, A> const&, requires_arch<avx512vl_256>) noexcept
0167         {
0168             return self.data;
0169         }
0170 
0171         // set
0172         template <class A, class T, class... Values>
0173         XSIMD_INLINE batch_bool<T, A> set(batch_bool<T, A> const&, requires_arch<avx512vl_256>, Values... values) noexcept
0174         {
0175             static_assert(sizeof...(Values) == batch_bool<T, A>::size, "consistent init");
0176             using register_type = typename batch_bool<T, A>::register_type;
0177             register_type r = 0;
0178             unsigned shift = 0;
0179             (void)std::initializer_list<register_type> { (r |= register_type(values ? 1 : 0) << (shift++))... };
0180             return r;
0181         }
0182 
0183         // store
0184         template <class T, class A>
0185         XSIMD_INLINE void store(batch_bool<T, A> const& self, bool* mem, requires_arch<avx512vl_256>) noexcept
0186         {
0187             constexpr auto size = batch_bool<T, A>::size;
0188             for (std::size_t i = 0; i < size; ++i)
0189                 mem[i] = (self.data >> i) & 0x1;
0190         }
0191 
0192         // abs
0193         template <class A>
0194         XSIMD_INLINE batch<int64_t, A> abs(batch<int64_t, A> const& self, requires_arch<avx512vl_256>) noexcept
0195         {
0196             return _mm256_abs_epi64(self);
0197         }
0198 
0199         // Masked load/store: native 256-bit EVEX predication shared by the
0200         // constant (batch_bool_constant) and runtime (batch_bool) overloads.
0201         // Partial ordering picks the avx512vl_256 tag over the avx2 bridges this
0202         // arch inherits — crucial because the k-register mask cannot feed the
0203         // avx2 vpmaskmov path. 8/16-bit elements fall back to the common scalar
0204         // path. Unsigned element types reinterpret to the signed EVEX intrinsic.
0205         namespace detail
0206         {
0207             // One core per native register type; signed and unsigned integrals
0208             // share an overload (the EVEX intrinsic is sign-agnostic). Mode
0209             // selects aligned vs unaligned.
0210             template <class T, class Mode, enable_sized_integral_t<T, 4> = 0>
0211             XSIMD_INLINE __m256i maskload256(T const* mem, uint64_t m, Mode) noexcept
0212             {
0213                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0214                 {
0215                     return _mm256_maskz_load_epi32((__mmask8)m, mem);
0216                 }
0217                 else
0218                 {
0219                     return _mm256_maskz_loadu_epi32((__mmask8)m, mem);
0220                 }
0221             }
0222             template <class T, class Mode, enable_sized_integral_t<T, 8> = 0>
0223             XSIMD_INLINE __m256i maskload256(T const* mem, uint64_t m, Mode) noexcept
0224             {
0225                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0226                 {
0227                     return _mm256_maskz_load_epi64((__mmask8)m, mem);
0228                 }
0229                 else
0230                 {
0231                     return _mm256_maskz_loadu_epi64((__mmask8)m, mem);
0232                 }
0233             }
0234             template <class Mode>
0235             XSIMD_INLINE __m256 maskload256(float const* mem, uint64_t m, Mode) noexcept
0236             {
0237                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0238                 {
0239                     return _mm256_maskz_load_ps((__mmask8)m, mem);
0240                 }
0241                 else
0242                 {
0243                     return _mm256_maskz_loadu_ps((__mmask8)m, mem);
0244                 }
0245             }
0246             template <class Mode>
0247             XSIMD_INLINE __m256d maskload256(double const* mem, uint64_t m, Mode) noexcept
0248             {
0249                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0250                 {
0251                     return _mm256_maskz_load_pd((__mmask8)m, mem);
0252                 }
0253                 else
0254                 {
0255                     return _mm256_maskz_loadu_pd((__mmask8)m, mem);
0256                 }
0257             }
0258 
0259             template <class T, class Mode, enable_sized_integral_t<T, 4> = 0>
0260             XSIMD_INLINE void maskstore256(T* mem, __m256i src, uint64_t m, Mode) noexcept
0261             {
0262                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0263                 {
0264                     _mm256_mask_store_epi32(mem, (__mmask8)m, src);
0265                 }
0266                 else
0267                 {
0268                     _mm256_mask_storeu_epi32(mem, (__mmask8)m, src);
0269                 }
0270             }
0271             template <class T, class Mode, enable_sized_integral_t<T, 8> = 0>
0272             XSIMD_INLINE void maskstore256(T* mem, __m256i src, uint64_t m, Mode) noexcept
0273             {
0274                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0275                 {
0276                     _mm256_mask_store_epi64(mem, (__mmask8)m, src);
0277                 }
0278                 else
0279                 {
0280                     _mm256_mask_storeu_epi64(mem, (__mmask8)m, src);
0281                 }
0282             }
0283             template <class Mode>
0284             XSIMD_INLINE void maskstore256(float* mem, __m256 src, uint64_t m, Mode) noexcept
0285             {
0286                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0287                 {
0288                     _mm256_mask_store_ps(mem, (__mmask8)m, src);
0289                 }
0290                 else
0291                 {
0292                     _mm256_mask_storeu_ps(mem, (__mmask8)m, src);
0293                 }
0294             }
0295             template <class Mode>
0296             XSIMD_INLINE void maskstore256(double* mem, __m256d src, uint64_t m, Mode) noexcept
0297             {
0298                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0299                 {
0300                     _mm256_mask_store_pd(mem, (__mmask8)m, src);
0301                 }
0302                 else
0303                 {
0304                     _mm256_mask_storeu_pd(mem, (__mmask8)m, src);
0305                 }
0306             }
0307         }
0308 
0309         // Constant masks: prefix/suffix shapes lower to plain moves; interior
0310         // masks keep the EVEX path.
0311         template <class A, class T, bool... V, class Mode,
0312                   typename = std::enable_if_t<std::is_arithmetic<T>::value && (sizeof(T) == 4 || sizeof(T) == 8)>>
0313         XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, V...> mask, convert<T>, Mode, requires_arch<avx512vl_256>) noexcept
0314         {
0315             // all() reaches here only via the avx512f half-split cascade.
0316             XSIMD_IF_CONSTEXPR(mask.all())
0317             {
0318                 return batch<T, A>::load(mem, Mode {});
0319             }
0320             else XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
0321             {
0322                 return detail::plain_move_load<avx>(mem, mask, convert<T> {}, Mode {});
0323             }
0324             else
0325             {
0326                 return detail::maskload256(mem, mask.mask(), Mode {});
0327             }
0328         }
0329 
0330         template <class A, class T, class Mode,
0331                   typename = std::enable_if_t<std::is_arithmetic<T>::value && (sizeof(T) == 4 || sizeof(T) == 8)>>
0332         XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool<T, A> mask, convert<T>, Mode, requires_arch<avx512vl_256>) noexcept
0333         {
0334             return detail::maskload256(mem, mask.mask(), Mode {});
0335         }
0336 
0337         template <class A, class T, bool... V, class Mode,
0338                   typename = std::enable_if_t<std::is_arithmetic<T>::value && (sizeof(T) == 4 || sizeof(T) == 8)>>
0339         XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool_constant<T, A, V...> mask, Mode, requires_arch<avx512vl_256>) noexcept
0340         {
0341             XSIMD_IF_CONSTEXPR(mask.all())
0342             {
0343                 src.store(mem, Mode {});
0344             }
0345             else XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
0346             {
0347                 detail::plain_move_store<avx2>(mem, src, mask, Mode {});
0348             }
0349             else
0350             {
0351                 detail::maskstore256(mem, src, mask.mask(), Mode {});
0352             }
0353         }
0354 
0355         template <class A, class T, class Mode,
0356                   typename = std::enable_if_t<std::is_arithmetic<T>::value && (sizeof(T) == 4 || sizeof(T) == 8)>>
0357         XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool<T, A> mask, Mode, requires_arch<avx512vl_256>) noexcept
0358         {
0359             detail::maskstore256(mem, src, mask.mask(), Mode {});
0360         }
0361 
0362         // gather
0363         template <class T, class A, class U, detail::enable_sized_integral_t<T, 4> = 0, detail::enable_sized_integral_t<U, 4> = 0>
0364         XSIMD_INLINE batch<T, A> gather(batch<T, A> const&, T const* src, batch<U, A> const& index,
0365                                         kernel::requires_arch<avx512vl_256>) noexcept
0366         {
0367             return _mm256_mmask_i32gather_epi32(_mm256_setzero_si256(), (__mmask8)0xFF, index, static_cast<const void*>(src), sizeof(T));
0368         }
0369 
0370         template <class T, class A, class U, detail::enable_sized_integral_t<T, 8> = 0, detail::enable_sized_integral_t<U, 8> = 0>
0371         XSIMD_INLINE batch<T, A> gather(batch<T, A> const&, T const* src, batch<U, A> const& index,
0372                                         kernel::requires_arch<avx512vl_256>) noexcept
0373         {
0374             return _mm256_mmask_i64gather_epi64(_mm256_setzero_si256(), (__mmask8)0xF, index, static_cast<const void*>(src), sizeof(T));
0375         }
0376 
0377         template <class A, class U, detail::enable_sized_integral_t<U, 4> = 0>
0378         XSIMD_INLINE batch<float, A> gather(batch<float, A> const&, float const* src,
0379                                             batch<U, A> const& index,
0380                                             kernel::requires_arch<avx512vl_256>) noexcept
0381         {
0382             return _mm256_mmask_i32gather_ps(_mm256_setzero_ps(), (__mmask8)0xFF, index, src, sizeof(float));
0383         }
0384 
0385         template <class A, class U, detail::enable_sized_integral_t<U, 8> = 0>
0386         XSIMD_INLINE batch<double, A> gather(batch<double, A> const&, double const* src,
0387                                              batch<U, A> const& index,
0388                                              requires_arch<avx512vl_256>) noexcept
0389         {
0390             return _mm256_mmask_i64gather_pd(_mm256_setzero_pd(), (__mmask8)0xF, index, src, sizeof(double));
0391         }
0392 
0393         // scatter
0394         template <class A, class T,
0395                   class = std::enable_if_t<std::is_same<uint32_t, T>::value || std::is_same<int32_t, T>::value>>
0396         XSIMD_INLINE void scatter(batch<T, A> const& src, T* dst,
0397                                   batch<int32_t, A> const& index,
0398                                   kernel::requires_arch<avx512vl_256>) noexcept
0399         {
0400             _mm256_i32scatter_epi32(dst, index, src, sizeof(T));
0401         }
0402 
0403         template <class A, class T,
0404                   class = std::enable_if_t<std::is_same<uint64_t, T>::value || std::is_same<int64_t, T>::value>>
0405         XSIMD_INLINE void scatter(batch<T, A> const& src, T* dst,
0406                                   batch<int64_t, A> const& index,
0407                                   kernel::requires_arch<avx512vl_256>) noexcept
0408         {
0409             _mm256_i64scatter_epi64(dst, index, src, sizeof(T));
0410         }
0411 
0412         template <class A>
0413         XSIMD_INLINE void scatter(batch<float, A> const& src, float* dst,
0414                                   batch<int32_t, A> const& index,
0415                                   kernel::requires_arch<avx512vl_256>) noexcept
0416         {
0417             _mm256_i32scatter_ps(dst, index, src, sizeof(float));
0418         }
0419 
0420         template <class A>
0421         XSIMD_INLINE void scatter(batch<double, A> const& src, double* dst,
0422                                   batch<int64_t, A> const& index,
0423                                   kernel::requires_arch<avx512vl_256>) noexcept
0424         {
0425             _mm256_i64scatter_pd(dst, index, src, sizeof(double));
0426         }
0427 
0428         // max
0429         template <class A>
0430         XSIMD_INLINE batch<int64_t, A> max(batch<int64_t, A> const& self, batch<int64_t, A> const& other, requires_arch<avx512vl_256>) noexcept
0431         {
0432             return _mm256_max_epi64(self, other);
0433         }
0434         template <class A>
0435         XSIMD_INLINE batch<uint64_t, A> max(batch<uint64_t, A> const& self, batch<uint64_t, A> const& other, requires_arch<avx512vl_256>) noexcept
0436         {
0437             return _mm256_max_epu64(self, other);
0438         }
0439 
0440         // min
0441         template <class A>
0442         XSIMD_INLINE batch<int64_t, A> min(batch<int64_t, A> const& self, batch<int64_t, A> const& other, requires_arch<avx512vl_256>) noexcept
0443         {
0444             return _mm256_min_epi64(self, other);
0445         }
0446         template <class A>
0447         XSIMD_INLINE batch<uint64_t, A> min(batch<uint64_t, A> const& self, batch<uint64_t, A> const& other, requires_arch<avx512vl_256>) noexcept
0448         {
0449             return _mm256_min_epu64(self, other);
0450         }
0451 
0452         // swizzle (dynamic version)
0453         template <class A>
0454         XSIMD_INLINE batch<float, A> swizzle(batch<float, A> const& self, batch<uint32_t, A> mask, requires_arch<avx512vl_256>) noexcept
0455         {
0456             return _mm256_permutexvar_ps(mask, self);
0457         }
0458 
0459         template <class A>
0460         XSIMD_INLINE batch<double, A> swizzle(batch<double, A> const& self, batch<uint64_t, A> mask, requires_arch<avx512vl_256>) noexcept
0461         {
0462             return _mm256_permutexvar_pd(mask, self);
0463         }
0464 
0465         template <class A>
0466         XSIMD_INLINE batch<uint64_t, A> swizzle(batch<uint64_t, A> const& self, batch<uint64_t, A> mask, requires_arch<avx512vl_256>) noexcept
0467         {
0468             return _mm256_permutexvar_epi64(mask, self);
0469         }
0470 
0471         template <class A>
0472         XSIMD_INLINE batch<int64_t, A> swizzle(batch<int64_t, A> const& self, batch<uint64_t, A> mask, requires_arch<avx512vl_256>) noexcept
0473         {
0474             return bitwise_cast<int64_t>(swizzle(bitwise_cast<uint64_t>(self), mask, avx512vl_256 {}));
0475         }
0476 
0477         template <class A>
0478         XSIMD_INLINE batch<uint32_t, A> swizzle(batch<uint32_t, A> const& self, batch<uint32_t, A> mask, requires_arch<avx512vl_256>) noexcept
0479         {
0480             return _mm256_permutexvar_epi32(mask, self);
0481         }
0482 
0483         template <class A>
0484         XSIMD_INLINE batch<int32_t, A> swizzle(batch<int32_t, A> const& self, batch<uint32_t, A> mask, requires_arch<avx512vl_256>) noexcept
0485         {
0486             return bitwise_cast<int32_t>(swizzle(bitwise_cast<uint32_t>(self), mask, avx512vl_256 {}));
0487         }
0488         template <class A>
0489         XSIMD_INLINE batch<uint8_t, A> swizzle(batch<uint8_t, A> const& self, batch<uint8_t, A> mask, requires_arch<avx512vl_256>) noexcept
0490         {
0491             return swizzle(batch<uint8_t, avx2> { self.data }, batch<uint8_t, avx2> { mask.data }, avx2 {}).data;
0492         }
0493         // No EVEX byte/word permute without AVX512_VBMI, so 16-bit dynamic
0494         // swizzle forwards to avx2 — mirrors the uint8_t base above. Without
0495         // this base the sized-2 bridge below re-selects itself for uint16_t
0496         // and recurses forever (always_inline build failure).
0497         template <class A>
0498         XSIMD_INLINE batch<uint16_t, A> swizzle(batch<uint16_t, A> const& self, batch<uint16_t, A> mask, requires_arch<avx512vl_256>) noexcept
0499         {
0500             return swizzle(batch<uint16_t, avx2> { self.data }, batch<uint16_t, avx2> { mask.data }, avx2 {}).data;
0501         }
0502         template <class A, typename T, detail::enable_sized_t<T, 1> = 0>
0503         XSIMD_INLINE batch<T, A> swizzle(batch<T, A> const& self, batch<uint8_t, A> const& mask, requires_arch<avx512vl_256> req) noexcept
0504         {
0505             return bitwise_cast<T>(swizzle(bitwise_cast<uint8_t>(self), mask, req));
0506         }
0507         template <class A, typename T, detail::enable_sized_t<T, 2> = 0>
0508         XSIMD_INLINE batch<T, A> swizzle(batch<T, A> const& self, batch<uint16_t, A> const& mask, requires_arch<avx512vl_256> req) noexcept
0509         {
0510             return bitwise_cast<T>(swizzle(bitwise_cast<uint16_t>(self), mask, req));
0511         }
0512 
0513         // swizzle
0514         template <class A, uint8_t... Vals, typename T, detail::enable_sized_t<T, 1> = 0>
0515         XSIMD_INLINE batch<T, A> swizzle(batch<T, A> const& self, batch_constant<uint8_t, A, Vals...> const& mask, requires_arch<avx512vl_256>) noexcept
0516         {
0517             return swizzle(self, mask, fma3<avx2> {});
0518         }
0519         template <class A, uint16_t... Vals, typename T, detail::enable_sized_t<T, 2> = 0>
0520         XSIMD_INLINE batch<T, A> swizzle(batch<T, A> const& self, batch_constant<uint16_t, A, Vals...> const& mask, requires_arch<avx512vl_256>) noexcept
0521         {
0522             return swizzle(self, mask, fma3<avx2> {});
0523         }
0524         template <class A, uint32_t... Vals, typename T, detail::enable_sized_t<T, 4> = 0>
0525         XSIMD_INLINE batch<T, A> swizzle(batch<T, A> const& self, batch_constant<uint32_t, A, Vals...> const& mask, requires_arch<avx512vl_256>) noexcept
0526         {
0527             return swizzle(self, mask, fma3<avx2> {});
0528         }
0529 
0530         template <class A, uint64_t V0, uint64_t V1, uint64_t V2, uint64_t V3>
0531         XSIMD_INLINE batch<double, A> swizzle(batch<double, A> const& self, batch_constant<uint64_t, A, V0, V1, V2, V3>, requires_arch<avx512vl_256>) noexcept
0532         {
0533             constexpr auto mask = detail::mod_shuffle(V0, V1, V2, V3);
0534             return _mm256_permutex_pd(self, mask);
0535         }
0536         template <class A, typename T, uint64_t V0, uint64_t V1, uint64_t V2, uint64_t V3, detail::enable_sized_t<T, 8> = 0>
0537         XSIMD_INLINE batch<T, A> swizzle(batch<T, A> const& self, batch_constant<uint64_t, A, V0, V1, V2, V3>, requires_arch<avx512vl_256>) noexcept
0538         {
0539             constexpr auto mask = detail::mod_shuffle(V0, V1, V2, V3);
0540             return _mm256_permutex_epi64(self, mask);
0541         }
0542 
0543         // insert
0544         template <class A, size_t I>
0545         XSIMD_INLINE batch<float, A> insert(batch<float, A> const& self, float val, index<I>, requires_arch<avx512vl_256>) noexcept
0546         {
0547 
0548             int32_t tmp = bit_cast<int32_t>(val);
0549             return _mm256_castsi256_ps(_mm256_mask_set1_epi32(_mm256_castps_si256(self), __mmask8(1 << (I & 7)), tmp));
0550         }
0551 
0552         template <class A, size_t I>
0553         XSIMD_INLINE batch<double, A> insert(batch<double, A> const& self, double val, index<I>, requires_arch<avx512vl_256>) noexcept
0554         {
0555             int64_t tmp = bit_cast<int64_t>(val);
0556             return _mm256_castsi256_pd(_mm256_mask_set1_epi64(_mm256_castpd_si256(self), __mmask8(1 << (I & 3)), tmp));
0557         }
0558 
0559         template <class A, class T, size_t I, class = std::enable_if_t<std::is_integral<T>::value>>
0560         XSIMD_INLINE batch<T, A> insert(batch<T, A> const& self, T val, index<I> pos, requires_arch<avx512vl_256>) noexcept
0561         {
0562             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0563             {
0564                 return _mm256_mask_set1_epi32(self, __mmask8(1 << (I & 7)), val);
0565             }
0566             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0567             {
0568                 return _mm256_mask_set1_epi64(self, __mmask8(1 << (I & 3)), val);
0569             }
0570             else
0571             {
0572                 return insert(self, val, pos, common {});
0573             }
0574         }
0575 
0576         // isnan
0577         template <class A>
0578         XSIMD_INLINE batch_bool<float, A> isnan(batch<float, A> const& self, requires_arch<avx512vl_256>) noexcept
0579         {
0580             return (typename batch_bool<float, A>::register_type)_mm256_cmp_ps_mask(self, self, _CMP_UNORD_Q);
0581         }
0582         template <class A>
0583         XSIMD_INLINE batch_bool<double, A> isnan(batch<double, A> const& self, requires_arch<avx512vl_256>) noexcept
0584         {
0585             return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, self, _CMP_UNORD_Q);
0586         }
0587 
0588         // bitwise_rshift — signed int64 uses the native EVEX arithmetic shift
0589         // (VPSRAQ / VPSRAVQ, lat 1 / CPI 0.5). Every other width/sign keeps the
0590         // inherited avx2 codegen (srai/srav for 32-bit, srli for unsigned 64).
0591         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0592         XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_256>) noexcept
0593         {
0594             XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
0595             {
0596                 return _mm256_srai_epi64(self, other);
0597             }
0598             else
0599             {
0600                 return bitwise_rshift(self, other, avx2 {});
0601             }
0602         }
0603         template <size_t shift, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0604         XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, requires_arch<avx512vl_256>) noexcept
0605         {
0606             constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
0607             static_assert(shift < bits, "Shift amount must be less than the number of bits in T");
0608             XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
0609             {
0610                 return _mm256_srai_epi64(self, shift);
0611             }
0612             else
0613             {
0614                 return bitwise_rshift<shift>(self, avx2 {});
0615             }
0616         }
0617         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0618         XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0619         {
0620             XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
0621             {
0622                 return _mm256_srav_epi64(self, other);
0623             }
0624             else
0625             {
0626                 return bitwise_rshift(self, other, avx2 {});
0627             }
0628         }
0629 
0630         // rotl
0631         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0632         XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0633         {
0634             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0635             {
0636                 return _mm256_rolv_epi32(self, other);
0637             }
0638             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0639             {
0640                 return _mm256_rolv_epi64(self, other);
0641             }
0642             else
0643             {
0644                 return rotl(self, other, avx2 {});
0645             }
0646         }
0647         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0648         XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_256>) noexcept
0649         {
0650             return rotl(self, batch<T, A>(other), A {});
0651         }
0652         template <size_t count, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0653         XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, requires_arch<avx512vl_256>) noexcept
0654         {
0655             constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
0656             static_assert(count < bits, "Count must be less than the number of bits in T");
0657             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0658             {
0659                 return _mm256_rol_epi32(self, count);
0660             }
0661             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0662             {
0663                 return _mm256_rol_epi64(self, count);
0664             }
0665             else
0666             {
0667                 return rotl<count>(self, avx2 {});
0668             }
0669         }
0670 
0671         // rotr
0672         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0673         XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0674         {
0675             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0676             {
0677                 return _mm256_rorv_epi32(self, other);
0678             }
0679             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0680             {
0681                 return _mm256_rorv_epi64(self, other);
0682             }
0683             else
0684             {
0685                 return rotr(self, other, avx2 {});
0686             }
0687         }
0688         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0689         XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_256>) noexcept
0690         {
0691             return rotr(self, batch<T, A>(other), A {});
0692         }
0693 
0694         template <size_t count, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0695         XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, requires_arch<avx512vl_256>) noexcept
0696         {
0697             constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
0698             static_assert(count < bits, "Count must be less than the number of bits in T");
0699             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0700             {
0701                 return _mm256_ror_epi32(self, count);
0702             }
0703             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0704             {
0705                 return _mm256_ror_epi64(self, count);
0706             }
0707             else
0708             {
0709                 return rotr<count>(self, avx2 {});
0710             }
0711         }
0712 
0713         // compress — native EVEX VPCOMPRESS{PS,PD,Q,D} for the widths with VL
0714         // forms. 8/16-bit need AVX512_VBMI2, so they fall through to common{}.
0715         template <class A>
0716         XSIMD_INLINE batch<float, A> compress(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_256>) noexcept
0717         {
0718             return _mm256_maskz_compress_ps(mask.mask(), self);
0719         }
0720         template <class A>
0721         XSIMD_INLINE batch<double, A> compress(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_256>) noexcept
0722         {
0723             return _mm256_maskz_compress_pd(mask.mask(), self);
0724         }
0725         template <class A>
0726         XSIMD_INLINE batch<int32_t, A> compress(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0727         {
0728             return _mm256_maskz_compress_epi32(mask.mask(), self);
0729         }
0730         template <class A>
0731         XSIMD_INLINE batch<uint32_t, A> compress(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0732         {
0733             return _mm256_maskz_compress_epi32(mask.mask(), self);
0734         }
0735         template <class A>
0736         XSIMD_INLINE batch<int64_t, A> compress(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0737         {
0738             return _mm256_maskz_compress_epi64(mask.mask(), self);
0739         }
0740         template <class A>
0741         XSIMD_INLINE batch<uint64_t, A> compress(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0742         {
0743             return _mm256_maskz_compress_epi64(mask.mask(), self);
0744         }
0745 
0746         // expand
0747         template <class A>
0748         XSIMD_INLINE batch<float, A> expand(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_256>) noexcept
0749         {
0750             return _mm256_maskz_expand_ps(mask.mask(), self);
0751         }
0752         template <class A>
0753         XSIMD_INLINE batch<double, A> expand(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_256>) noexcept
0754         {
0755             return _mm256_maskz_expand_pd(mask.mask(), self);
0756         }
0757         template <class A>
0758         XSIMD_INLINE batch<int32_t, A> expand(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0759         {
0760             return _mm256_maskz_expand_epi32(mask.mask(), self);
0761         }
0762         template <class A>
0763         XSIMD_INLINE batch<uint32_t, A> expand(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0764         {
0765             return _mm256_maskz_expand_epi32(mask.mask(), self);
0766         }
0767         template <class A>
0768         XSIMD_INLINE batch<int64_t, A> expand(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0769         {
0770             return _mm256_maskz_expand_epi64(mask.mask(), self);
0771         }
0772         template <class A>
0773         XSIMD_INLINE batch<uint64_t, A> expand(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
0774         {
0775             return _mm256_maskz_expand_epi64(mask.mask(), self);
0776         }
0777 
0778         // all
0779         template <class A, class T>
0780         XSIMD_INLINE bool all(batch_bool<T, A> const& self, requires_arch<avx512vl_256>) noexcept
0781         {
0782             using register_type = typename batch_bool<T, A>::register_type;
0783             constexpr register_type bitmask = (register_type(1) << batch_bool<T, A>::size) - 1;
0784             return (self.data & bitmask) == bitmask;
0785         }
0786 
0787         // any
0788         template <class A, class T>
0789         XSIMD_INLINE bool any(batch_bool<T, A> const& self, requires_arch<avx512vl_256>) noexcept
0790         {
0791             using register_type = typename batch_bool<T, A>::register_type;
0792             constexpr register_type bitmask = (register_type(1) << batch_bool<T, A>::size) - 1;
0793             return (self.data & bitmask) != 0;
0794         }
0795 
0796         // eq
0797         template <class A>
0798         XSIMD_INLINE batch_bool<float, A> eq(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_256>) noexcept
0799         {
0800             return (typename batch_bool<float, A>::register_type)_mm256_cmp_ps_mask(self, other, _CMP_EQ_OQ);
0801         }
0802         template <class A>
0803         XSIMD_INLINE batch_bool<double, A> eq(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_256>) noexcept
0804         {
0805             return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, other, _CMP_EQ_OQ);
0806         }
0807 
0808         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0809         XSIMD_INLINE batch_bool<T, A> eq(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0810         {
0811             return detail::compare_int_avx512vl_256<A, T, _MM_CMPINT_EQ>(self, other);
0812         }
0813         template <class A, class T>
0814         XSIMD_INLINE batch_bool<T, A> eq(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0815         {
0816             using register_type = typename batch_bool<T, A>::register_type;
0817             return register_type(~self.data ^ other.data);
0818         }
0819 
0820         // neq
0821         template <class A>
0822         XSIMD_INLINE batch_bool<float, A> neq(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_256>) noexcept
0823         {
0824             return (typename batch_bool<float, A>::register_type)_mm256_cmp_ps_mask(self, other, _CMP_NEQ_UQ);
0825         }
0826         template <class A>
0827         XSIMD_INLINE batch_bool<double, A> neq(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_256>) noexcept
0828         {
0829             return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, other, _CMP_NEQ_UQ);
0830         }
0831 
0832         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0833         XSIMD_INLINE batch_bool<T, A> neq(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0834         {
0835             return (~(self == other));
0836         }
0837         template <class A, class T>
0838         XSIMD_INLINE batch_bool<T, A> neq(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0839         {
0840             using register_type = typename batch_bool<T, A>::register_type;
0841             return register_type(self.data ^ other.data);
0842         }
0843 
0844         // gt
0845         template <class A>
0846         XSIMD_INLINE batch_bool<float, A> gt(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_256>) noexcept
0847         {
0848             return (typename batch_bool<float, A>::register_type)_mm256_cmp_ps_mask(self, other, _CMP_GT_OQ);
0849         }
0850         template <class A>
0851         XSIMD_INLINE batch_bool<double, A> gt(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_256>) noexcept
0852         {
0853             return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, other, _CMP_GT_OQ);
0854         }
0855         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0856         XSIMD_INLINE batch_bool<T, A> gt(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0857         {
0858             return detail::compare_int_avx512vl_256<A, T, _MM_CMPINT_GT>(self, other);
0859         }
0860 
0861         // ge
0862         template <class A>
0863         XSIMD_INLINE batch_bool<float, A> ge(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_256>) noexcept
0864         {
0865             return (typename batch_bool<float, A>::register_type)_mm256_cmp_ps_mask(self, other, _CMP_GE_OQ);
0866         }
0867         template <class A>
0868         XSIMD_INLINE batch_bool<double, A> ge(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_256>) noexcept
0869         {
0870             return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, other, _CMP_GE_OQ);
0871         }
0872 
0873         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0874         XSIMD_INLINE batch_bool<T, A> ge(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0875         {
0876             return detail::compare_int_avx512vl_256<A, T, _MM_CMPINT_GE>(self, other);
0877         }
0878 
0879         // lt
0880         template <class A>
0881         XSIMD_INLINE batch_bool<float, A> lt(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_256>) noexcept
0882         {
0883             return (typename batch_bool<float, A>::register_type)_mm256_cmp_ps_mask(self, other, _CMP_LT_OQ);
0884         }
0885         template <class A>
0886         XSIMD_INLINE batch_bool<double, A> lt(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_256>) noexcept
0887         {
0888             return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, other, _CMP_LT_OQ);
0889         }
0890 
0891         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0892         XSIMD_INLINE batch_bool<T, A> lt(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0893         {
0894             return detail::compare_int_avx512vl_256<A, T, _MM_CMPINT_LT>(self, other);
0895         }
0896 
0897         // le
0898         template <class A>
0899         XSIMD_INLINE batch_bool<float, A> le(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_256>) noexcept
0900         {
0901             return (typename batch_bool<float, A>::register_type)_mm256_cmp_ps_mask(self, other, _CMP_LE_OQ);
0902         }
0903         template <class A>
0904         XSIMD_INLINE batch_bool<double, A> le(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_256>) noexcept
0905         {
0906             return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, other, _CMP_LE_OQ);
0907         }
0908 
0909         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0910         XSIMD_INLINE batch_bool<T, A> le(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0911         {
0912             return detail::compare_int_avx512vl_256<A, T, _MM_CMPINT_LE>(self, other);
0913         }
0914 
0915         // select
0916         template <class A>
0917         XSIMD_INLINE batch<float, A> select(batch_bool<float, A> const& cond, batch<float, A> const& true_br, batch<float, A> const& false_br, requires_arch<avx512vl_256>) noexcept
0918         {
0919             return _mm256_mask_blend_ps(cond, false_br, true_br);
0920         }
0921         template <class A>
0922         XSIMD_INLINE batch<double, A> select(batch_bool<double, A> const& cond, batch<double, A> const& true_br, batch<double, A> const& false_br, requires_arch<avx512vl_256>) noexcept
0923         {
0924             return _mm256_mask_blend_pd(cond, false_br, true_br);
0925         }
0926         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0927         XSIMD_INLINE batch<T, A> select(batch_bool<T, A> const& cond, batch<T, A> const& true_br, batch<T, A> const& false_br, requires_arch<avx512vl_256>) noexcept
0928         {
0929             XSIMD_IF_CONSTEXPR(sizeof(T) == 1)
0930             {
0931                 batch_bool<T, avx2> batch_cond = batch_bool<T, avx2>::from_mask(cond.mask());
0932                 return _mm256_blendv_epi8(false_br, true_br, batch_cond);
0933             }
0934             else XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
0935             {
0936                 batch_bool<T, avx2> batch_cond = batch_bool<T, avx2>::from_mask(cond.mask());
0937                 return _mm256_blendv_epi8(false_br, true_br, batch_cond);
0938             }
0939             else XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0940             {
0941                 return _mm256_mask_blend_epi32(cond, false_br, true_br);
0942             }
0943             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0944             {
0945                 return _mm256_mask_blend_epi64(cond, false_br, true_br);
0946             }
0947         }
0948         template <class A, class T, bool... Values>
0949         XSIMD_INLINE batch<T, A> select(batch_bool_constant<T, A, Values...> const&, batch<T, A> const& true_br, batch<T, A> const& false_br, requires_arch<avx512vl_256>) noexcept
0950         {
0951             return select(batch_bool<T, A> { Values... }, true_br, false_br, avx512vl_256 {});
0952         }
0953 
0954         // decr_if / incr_if — the inherited avx kernels compute
0955         // `self ± batch<T>(mask.data)`, which assumes a vector batch_bool whose
0956         // true lanes are all-ones. Here batch_bool::data is a k-mask bitfield,
0957         // so that broadcast yields garbage. Delegate to the select-based common
0958         // implementation instead.
0959         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0960         XSIMD_INLINE batch<T, A> decr_if(batch<T, A> const& self, batch_bool<T, A> const& mask, requires_arch<avx512vl_256>) noexcept
0961         {
0962             return decr_if(self, mask, common {});
0963         }
0964         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0965         XSIMD_INLINE batch<T, A> incr_if(batch<T, A> const& self, batch_bool<T, A> const& mask, requires_arch<avx512vl_256>) noexcept
0966         {
0967             return incr_if(self, mask, common {});
0968         }
0969 
0970         // reciprocal
0971         template <class A>
0972         XSIMD_INLINE batch<float, A>
0973         reciprocal(batch<float, A> const& self,
0974                    kernel::requires_arch<avx512vl_256>) noexcept
0975         {
0976             return _mm256_rcp14_ps(self);
0977         }
0978 
0979         template <class A>
0980         XSIMD_INLINE batch<double, A>
0981         reciprocal(batch<double, A> const& self,
0982                    kernel::requires_arch<avx512vl_256>) noexcept
0983         {
0984             return _mm256_rcp14_pd(self);
0985         }
0986 
0987         // bitwise_and
0988         template <class A, class T>
0989         XSIMD_INLINE batch_bool<T, A> bitwise_and(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0990         {
0991             using register_type = typename batch_bool<T, A>::register_type;
0992             return register_type(self.data & other.data);
0993         }
0994 
0995         // bitwise_andnot
0996         template <class A, class T>
0997         XSIMD_INLINE batch_bool<T, A> bitwise_andnot(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_256>) noexcept
0998         {
0999             using register_type = typename batch_bool<T, A>::register_type;
1000             return register_type(self.data & ~other.data);
1001         }
1002 
1003         // bitwise_not
1004         template <class A, class T>
1005         XSIMD_INLINE batch_bool<T, A> bitwise_not(batch_bool<T, A> const& self, requires_arch<avx512vl_256>) noexcept
1006         {
1007             using register_type = typename batch_bool<T, A>::register_type;
1008             return register_type(~self.data);
1009         }
1010 
1011         // bitwise_or
1012         template <class A, class T>
1013         XSIMD_INLINE batch_bool<T, A> bitwise_or(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_256>) noexcept
1014         {
1015             using register_type = typename batch_bool<T, A>::register_type;
1016             return register_type(self.data | other.data);
1017         }
1018 
1019         // bitwise_xor
1020         template <class A, class T>
1021         XSIMD_INLINE batch_bool<T, A> bitwise_xor(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_256>) noexcept
1022         {
1023             using register_type = typename batch_bool<T, A>::register_type;
1024             return register_type(self.data ^ other.data);
1025         }
1026 
1027         // sadd
1028         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
1029         XSIMD_INLINE batch<T, A> sadd(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
1030         {
1031             if (std::is_signed<T>::value)
1032             {
1033                 auto mask = other < 0;
1034                 auto self_pos_branch = min(std::numeric_limits<T>::max() - other, self);
1035                 auto self_neg_branch = max(std::numeric_limits<T>::min() - other, self);
1036                 return other + select(mask, self_neg_branch, self_pos_branch);
1037             }
1038             else
1039             {
1040                 const auto diffmax = std::numeric_limits<T>::max() - self;
1041                 const auto mindiff = min(diffmax, other);
1042                 return self + mindiff;
1043             }
1044         }
1045 
1046     }
1047 }
1048 
1049 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.3.7 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!