xsimd/arch/xsimd_avx512vl_128.hpp

0001 /***************************************************************************
0002  * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
0003  * Martin Renou                                                             *
0004  * Copyright (c) QuantStack                                                 *
0005  * Copyright (c) Serge Guelton                                              *
0006  * Copyright (c) Marco Barbone                                              *
0007  *                                                                          *
0008  * Distributed under the terms of the BSD 3-Clause License.                 *
0009  *                                                                          *
0010  * The full license is in the file LICENSE, distributed with this software. *
0011  ****************************************************************************/
0012
0013 #ifndef XSIMD_AVX512VL_128_HPP
0014 #define XSIMD_AVX512VL_128_HPP
0015
0016 #include "../types/xsimd_avx512vl_register.hpp"
0017 #include "../types/xsimd_batch_constant.hpp"
0018
0019 #include <type_traits>
0020
0021 namespace xsimd
0022 {
0023     namespace kernel
0024     {
0025         using namespace types;
0026
0027         namespace detail
0028         {
0029             // Defined in xsimd_avx512f.hpp. This header is included before it so
0030             // that avx512f.hpp's masked load/store forwarder can resolve the
0031             // avx512vl_128 overloads below by ordinary lookup; forward-declare
0032             // the two helpers it borrows from there.
0033             XSIMD_INLINE uint32_t morton(uint16_t x, uint16_t y) noexcept;
0034             template <size_t N>
0035             XSIMD_INLINE unsigned char tobitset(unsigned char unpacked[N]);
0036
0037             template <class A, class T, int Cmp>
0038             XSIMD_INLINE batch_bool<T, A> compare_int_avx512vl_128(batch<T, A> const& self, batch<T, A> const& other) noexcept
0039             {
0040                 using register_type = typename batch_bool<T, A>::register_type;
0041                 if (std::is_signed<T>::value)
0042                 {
0043                     XSIMD_IF_CONSTEXPR(sizeof(T) == 1)
0044                     {
0045                         // shifting to take sign into account
0046                         uint64_t mask_low0 = _mm_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x000000FF)) << 24,
0047                                                                 (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x000000FF)) << 24,
0048                                                                 Cmp);
0049                         uint64_t mask_low1 = _mm_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x0000FF00)) << 16,
0050                                                                 (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x0000FF00)) << 16,
0051                                                                 Cmp);
0052                         uint64_t mask_high0 = _mm_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x00FF0000)) << 8,
0053                                                                  (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x00FF0000)) << 8,
0054                                                                  Cmp);
0055                         uint64_t mask_high1 = _mm_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0xFF000000)),
0056                                                                  (batch<int32_t, A>(other.data) & batch<int32_t, A>(0xFF000000)),
0057                                                                  Cmp);
0058                         uint64_t mask = 0;
0059                         for (unsigned i = 0; i < 8; ++i)
0060                         {
0061                             mask |= (mask_low0 & (uint64_t(1) << i)) << (3 * i + 0);
0062                             mask |= (mask_low1 & (uint64_t(1) << i)) << (3 * i + 1);
0063                             mask |= (mask_high0 & (uint64_t(1) << i)) << (3 * i + 2);
0064                             mask |= (mask_high1 & (uint64_t(1) << i)) << (3 * i + 3);
0065                         }
0066                         return (register_type)mask;
0067                     }
0068                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
0069                     {
0070                         // shifting to take sign into account
0071                         uint16_t mask_low = _mm_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0x0000FFFF)) << 16,
0072                                                                (batch<int32_t, A>(other.data) & batch<int32_t, A>(0x0000FFFF)) << 16,
0073                                                                Cmp);
0074                         uint16_t mask_high = _mm_cmp_epi32_mask((batch<int32_t, A>(self.data) & batch<int32_t, A>(0xFFFF0000)),
0075                                                                 (batch<int32_t, A>(other.data) & batch<int32_t, A>(0xFFFF0000)),
0076                                                                 Cmp);
0077                         return static_cast<register_type>(morton(mask_low, mask_high));
0078                     }
0079                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0080                     {
0081                         return (register_type)_mm_cmp_epi32_mask(self, other, Cmp);
0082                     }
0083                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0084                     {
0085                         return (register_type)_mm_cmp_epi64_mask(self, other, Cmp);
0086                     }
0087                 }
0088                 else
0089                 {
0090                     XSIMD_IF_CONSTEXPR(sizeof(T) == 1)
0091                     {
0092                         uint64_t mask_low0 = _mm_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x000000FF)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x000000FF)), Cmp);
0093                         uint64_t mask_low1 = _mm_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x0000FF00)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x0000FF00)), Cmp);
0094                         uint64_t mask_high0 = _mm_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x00FF0000)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x00FF0000)), Cmp);
0095                         uint64_t mask_high1 = _mm_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0xFF000000)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0xFF000000)), Cmp);
0096                         uint64_t mask = 0;
0097                         for (unsigned i = 0; i < 8; ++i)
0098                         {
0099                             mask |= (mask_low0 & (uint64_t(1) << i)) << (3 * i + 0);
0100                             mask |= (mask_low1 & (uint64_t(1) << i)) << (3 * i + 1);
0101                             mask |= (mask_high0 & (uint64_t(1) << i)) << (3 * i + 2);
0102                             mask |= (mask_high1 & (uint64_t(1) << i)) << (3 * i + 3);
0103                         }
0104                         return (register_type)mask;
0105                     }
0106                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
0107                     {
0108                         uint16_t mask_low = _mm_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0x0000FFFF)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0x0000FFFF)), Cmp);
0109                         uint16_t mask_high = _mm_cmp_epu32_mask((batch<uint32_t, A>(self.data) & batch<uint32_t, A>(0xFFFF0000)), (batch<uint32_t, A>(other.data) & batch<uint32_t, A>(0xFFFF0000)), Cmp);
0110                         return static_cast<register_type>(morton(mask_low, mask_high));
0111                     }
0112                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0113                     {
0114                         return (register_type)_mm_cmp_epu32_mask(self, other, Cmp);
0115                     }
0116                     else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0117                     {
0118                         return (register_type)_mm_cmp_epu64_mask(self, other, Cmp);
0119                     }
0120                 }
0121             }
0122         }
0123
0124         // load mask
0125         template <class A, class T>
0126         XSIMD_INLINE batch_bool<T, A> load_unaligned(bool const* mem, batch_bool<T, A>, requires_arch<avx512vl_128>) noexcept
0127         {
0128             using register_type = typename batch_bool<T, A>::register_type;
0129             constexpr auto size = batch_bool<T, A>::size;
0130             constexpr auto chunk_size = size >= 8 ? 8 : (size >= 4 ? 4 : 2);
0131             constexpr auto iter = size / chunk_size;
0132             static_assert((size % chunk_size) == 0, "incorrect size of bool batch");
0133             register_type mask = 0;
0134             for (std::size_t i = 0; i < iter; ++i)
0135             {
0136                 unsigned char block = detail::tobitset<chunk_size>((unsigned char*)mem + i * chunk_size);
0137                 mask |= (register_type(block) << (i * chunk_size));
0138             }
0139             return mask;
0140         }
0141
0142         // from bool
0143         template <class A, class T>
0144         XSIMD_INLINE batch<T, A> from_bool(batch_bool<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0145         {
0146             return select(self, batch<T, A>(1), batch<T, A>(0));
0147         }
0148
0149         // from_mask
0150         template <class T, class A>
0151         XSIMD_INLINE batch_bool<T, A> from_mask(batch_bool<T, A> const&, uint64_t mask, requires_arch<avx512vl_128>) noexcept
0152         {
0153             assert(mask == (mask & ((uint64_t(1) << batch_bool<T, A>::size) - 1)) && "inbound mask");
0154             return static_cast<typename batch_bool<T, A>::register_type>(mask & ((uint64_t(1) << batch_bool<T, A>::size) - 1));
0155         }
0156
0157         // mask
0158         template <class A, class T>
0159         XSIMD_INLINE uint64_t mask(batch_bool<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0160         {
0161             return self.data & ((uint64_t(1) << batch_bool<T, A>::size) - 1);
0162         }
0163
0164         // batch_bool_cast
0165         template <class A, class T_out, class T_in>
0166         XSIMD_INLINE batch_bool<T_out, A> batch_bool_cast(batch_bool<T_in, A> const& self, batch_bool<T_out, A> const&, requires_arch<avx512vl_128>) noexcept
0167         {
0168             return self.data;
0169         }
0170
0171         // set
0172         template <class A, class T, class... Values>
0173         XSIMD_INLINE batch_bool<T, A> set(batch_bool<T, A> const&, requires_arch<avx512vl_128>, Values... values) noexcept
0174         {
0175             static_assert(sizeof...(Values) == batch_bool<T, A>::size, "consistent init");
0176             using register_type = typename batch_bool<T, A>::register_type;
0177             register_type r = 0;
0178             unsigned shift = 0;
0179             (void)std::initializer_list<register_type> { (r |= register_type(values ? 1 : 0) << (shift++))... };
0180             return r;
0181         }
0182
0183         // store
0184         template <class T, class A>
0185         XSIMD_INLINE void store(batch_bool<T, A> const& self, bool* mem, requires_arch<avx512vl_128>) noexcept
0186         {
0187             constexpr auto size = batch_bool<T, A>::size;
0188             for (std::size_t i = 0; i < size; ++i)
0189                 mem[i] = (self.data >> i) & 0x1;
0190         }
0191
0192         // abs
0193         template <class A>
0194         XSIMD_INLINE batch<int64_t, A> abs(batch<int64_t, A> const& self, requires_arch<avx512vl_128>) noexcept
0195         {
0196             return _mm_abs_epi64(self);
0197         }
0198
0199         // Masked load/store: native 128-bit EVEX predication shared by the
0200         // constant (batch_bool_constant) and runtime (batch_bool) overloads.
0201         // Partial ordering picks the avx512vl_128 tag over the avx2_128 bridges
0202         // this arch inherits — crucial because the k-register mask cannot feed
0203         // the avx2 vpmaskmov path. 8/16-bit elements fall back to the common
0204         // scalar path. Unsigned element types reinterpret to the signed EVEX
0205         // intrinsic.
0206         namespace detail
0207         {
0208             // One core per native register type; signed and unsigned integrals
0209             // share an overload (the EVEX intrinsic is sign-agnostic). Mode
0210             // selects aligned vs unaligned.
0211             template <class T, class Mode, enable_sized_integral_t<T, 4> = 0>
0212             XSIMD_INLINE __m128i maskload128(T const* mem, uint64_t m, Mode) noexcept
0213             {
0214                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0215                 {
0216                     return _mm_maskz_load_epi32((__mmask8)m, mem);
0217                 }
0218                 else
0219                 {
0220                     return _mm_maskz_loadu_epi32((__mmask8)m, mem);
0221                 }
0222             }
0223             template <class T, class Mode, enable_sized_integral_t<T, 8> = 0>
0224             XSIMD_INLINE __m128i maskload128(T const* mem, uint64_t m, Mode) noexcept
0225             {
0226                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0227                 {
0228                     return _mm_maskz_load_epi64((__mmask8)m, mem);
0229                 }
0230                 else
0231                 {
0232                     return _mm_maskz_loadu_epi64((__mmask8)m, mem);
0233                 }
0234             }
0235             template <class Mode>
0236             XSIMD_INLINE __m128 maskload128(float const* mem, uint64_t m, Mode) noexcept
0237             {
0238                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0239                 {
0240                     return _mm_maskz_load_ps((__mmask8)m, mem);
0241                 }
0242                 else
0243                 {
0244                     return _mm_maskz_loadu_ps((__mmask8)m, mem);
0245                 }
0246             }
0247             template <class Mode>
0248             XSIMD_INLINE __m128d maskload128(double const* mem, uint64_t m, Mode) noexcept
0249             {
0250                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0251                 {
0252                     return _mm_maskz_load_pd((__mmask8)m, mem);
0253                 }
0254                 else
0255                 {
0256                     return _mm_maskz_loadu_pd((__mmask8)m, mem);
0257                 }
0258             }
0259
0260             template <class T, class Mode, enable_sized_integral_t<T, 4> = 0>
0261             XSIMD_INLINE void maskstore128(T* mem, __m128i src, uint64_t m, Mode) noexcept
0262             {
0263                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0264                 {
0265                     _mm_mask_store_epi32(mem, (__mmask8)m, src);
0266                 }
0267                 else
0268                 {
0269                     _mm_mask_storeu_epi32(mem, (__mmask8)m, src);
0270                 }
0271             }
0272             template <class T, class Mode, enable_sized_integral_t<T, 8> = 0>
0273             XSIMD_INLINE void maskstore128(T* mem, __m128i src, uint64_t m, Mode) noexcept
0274             {
0275                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0276                 {
0277                     _mm_mask_store_epi64(mem, (__mmask8)m, src);
0278                 }
0279                 else
0280                 {
0281                     _mm_mask_storeu_epi64(mem, (__mmask8)m, src);
0282                 }
0283             }
0284             template <class Mode>
0285             XSIMD_INLINE void maskstore128(float* mem, __m128 src, uint64_t m, Mode) noexcept
0286             {
0287                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0288                 {
0289                     _mm_mask_store_ps(mem, (__mmask8)m, src);
0290                 }
0291                 else
0292                 {
0293                     _mm_mask_storeu_ps(mem, (__mmask8)m, src);
0294                 }
0295             }
0296             template <class Mode>
0297             XSIMD_INLINE void maskstore128(double* mem, __m128d src, uint64_t m, Mode) noexcept
0298             {
0299                 XSIMD_IF_CONSTEXPR(std::is_same<Mode, aligned_mode>::value)
0300                 {
0301                     _mm_mask_store_pd(mem, (__mmask8)m, src);
0302                 }
0303                 else
0304                 {
0305                     _mm_mask_storeu_pd(mem, (__mmask8)m, src);
0306                 }
0307             }
0308         }
0309
0310         // Constant masks: prefix/suffix shapes lower to plain moves; interior
0311         // masks keep the EVEX path.
0312         template <class A, class T, bool... V, class Mode,
0313                   typename>
0314         XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, V...> mask, convert<T>, Mode, requires_arch<avx512vl_128>) noexcept
0315         {
0316             XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
0317             {
0318                 return detail::plain_move_load<sse2>(mem, mask, convert<T> {}, Mode {});
0319             }
0320             else
0321             {
0322                 return detail::maskload128(mem, mask.mask(), Mode {});
0323             }
0324         }
0325
0326         template <class A, class T, class Mode,
0327                   typename>
0328         XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool<T, A> mask, convert<T>, Mode, requires_arch<avx512vl_128>) noexcept
0329         {
0330             return detail::maskload128(mem, mask.mask(), Mode {});
0331         }
0332
0333         template <class A, class T, bool... V, class Mode,
0334                   typename>
0335         XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool_constant<T, A, V...> mask, Mode, requires_arch<avx512vl_128>) noexcept
0336         {
0337             XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
0338             {
0339                 detail::plain_move_store<sse2>(mem, src, mask, Mode {});
0340             }
0341             else
0342             {
0343                 detail::maskstore128(mem, src, mask.mask(), Mode {});
0344             }
0345         }
0346
0347         template <class A, class T, class Mode,
0348                   typename>
0349         XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool<T, A> mask, Mode, requires_arch<avx512vl_128>) noexcept
0350         {
0351             detail::maskstore128(mem, src, mask.mask(), Mode {});
0352         }
0353
0354         // gather
0355         template <class T, class A, class U, detail::enable_sized_integral_t<T, 4> = 0, detail::enable_sized_integral_t<U, 4> = 0>
0356         XSIMD_INLINE batch<T, A> gather(batch<T, A> const&, T const* src, batch<U, A> const& index,
0357                                         kernel::requires_arch<avx512vl_128>) noexcept
0358         {
0359             return _mm_mmask_i32gather_epi32(_mm_setzero_si128(), (__mmask8)0xF, index, static_cast<const void*>(src), sizeof(T));
0360         }
0361
0362         template <class T, class A, class U, detail::enable_sized_integral_t<T, 8> = 0, detail::enable_sized_integral_t<U, 8> = 0>
0363         XSIMD_INLINE batch<T, A> gather(batch<T, A> const&, T const* src, batch<U, A> const& index,
0364                                         kernel::requires_arch<avx512vl_128>) noexcept
0365         {
0366             return _mm_mmask_i64gather_epi64(_mm_setzero_si128(), (__mmask8)0x3, index, static_cast<const void*>(src), sizeof(T));
0367         }
0368
0369         template <class A, class U, detail::enable_sized_integral_t<U, 4> = 0>
0370         XSIMD_INLINE batch<float, A> gather(batch<float, A> const&, float const* src,
0371                                             batch<U, A> const& index,
0372                                             kernel::requires_arch<avx512vl_128>) noexcept
0373         {
0374             return _mm_mmask_i32gather_ps(_mm_setzero_ps(), (__mmask8)0xF, index, src, sizeof(float));
0375         }
0376
0377         template <class A, class U, detail::enable_sized_integral_t<U, 8> = 0>
0378         XSIMD_INLINE batch<double, A> gather(batch<double, A> const&, double const* src,
0379                                              batch<U, A> const& index,
0380                                              requires_arch<avx512vl_128>) noexcept
0381         {
0382             return _mm_mmask_i64gather_pd(_mm_setzero_pd(), (__mmask8)0x3, index, src, sizeof(double));
0383         }
0384
0385         // scatter
0386         template <class A, class T,
0387                   class = std::enable_if_t<std::is_same<uint32_t, T>::value || std::is_same<int32_t, T>::value>>
0388         XSIMD_INLINE void scatter(batch<T, A> const& src, T* dst,
0389                                   batch<int32_t, A> const& index,
0390                                   kernel::requires_arch<avx512vl_128>) noexcept
0391         {
0392             _mm_i32scatter_epi32(dst, index, src, sizeof(T));
0393         }
0394
0395         template <class A, class T,
0396                   class = std::enable_if_t<std::is_same<uint64_t, T>::value || std::is_same<int64_t, T>::value>>
0397         XSIMD_INLINE void scatter(batch<T, A> const& src, T* dst,
0398                                   batch<int64_t, A> const& index,
0399                                   kernel::requires_arch<avx512vl_128>) noexcept
0400         {
0401             _mm_i64scatter_epi64(dst, index, src, sizeof(T));
0402         }
0403
0404         template <class A>
0405         XSIMD_INLINE void scatter(batch<float, A> const& src, float* dst,
0406                                   batch<int32_t, A> const& index,
0407                                   kernel::requires_arch<avx512vl_128>) noexcept
0408         {
0409             _mm_i32scatter_ps(dst, index, src, sizeof(float));
0410         }
0411
0412         template <class A>
0413         XSIMD_INLINE void scatter(batch<double, A> const& src, double* dst,
0414                                   batch<int64_t, A> const& index,
0415                                   kernel::requires_arch<avx512vl_128>) noexcept
0416         {
0417             _mm_i64scatter_pd(dst, index, src, sizeof(double));
0418         }
0419
0420         // max
0421         template <class A>
0422         XSIMD_INLINE batch<int64_t, A> max(batch<int64_t, A> const& self, batch<int64_t, A> const& other, requires_arch<avx512vl_128>) noexcept
0423         {
0424             return _mm_max_epi64(self, other);
0425         }
0426         template <class A>
0427         XSIMD_INLINE batch<uint64_t, A> max(batch<uint64_t, A> const& self, batch<uint64_t, A> const& other, requires_arch<avx512vl_128>) noexcept
0428         {
0429             return _mm_max_epu64(self, other);
0430         }
0431
0432         // min
0433         template <class A>
0434         XSIMD_INLINE batch<int64_t, A> min(batch<int64_t, A> const& self, batch<int64_t, A> const& other, requires_arch<avx512vl_128>) noexcept
0435         {
0436             return _mm_min_epi64(self, other);
0437         }
0438         template <class A>
0439         XSIMD_INLINE batch<uint64_t, A> min(batch<uint64_t, A> const& self, batch<uint64_t, A> const& other, requires_arch<avx512vl_128>) noexcept
0440         {
0441             return _mm_min_epu64(self, other);
0442         }
0443
0444         // insert
0445         template <class A, size_t I>
0446         XSIMD_INLINE batch<float, A> insert(batch<float, A> const& self, float val, index<I>, requires_arch<avx512vl_128>) noexcept
0447         {
0448
0449             int32_t tmp = bit_cast<int32_t>(val);
0450             return _mm_castsi128_ps(_mm_mask_set1_epi32(_mm_castps_si128(self), __mmask8(1 << (I & 7)), tmp));
0451         }
0452
0453         template <class A, size_t I>
0454         XSIMD_INLINE batch<double, A> insert(batch<double, A> const& self, double val, index<I>, requires_arch<avx512vl_128>) noexcept
0455         {
0456             int64_t tmp = bit_cast<int64_t>(val);
0457             return _mm_castsi128_pd(_mm_mask_set1_epi64(_mm_castpd_si128(self), __mmask8(1 << (I & 3)), tmp));
0458         }
0459
0460         template <class A, class T, size_t I, class = std::enable_if_t<std::is_integral<T>::value>>
0461         XSIMD_INLINE batch<T, A> insert(batch<T, A> const& self, T val, index<I> pos, requires_arch<avx512vl_128>) noexcept
0462         {
0463             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0464             {
0465                 return _mm_mask_set1_epi32(self, __mmask8(1 << (I & 7)), val);
0466             }
0467             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0468             {
0469                 return _mm_mask_set1_epi64(self, __mmask8(1 << (I & 3)), val);
0470             }
0471             else
0472             {
0473                 return insert(self, val, pos, common {});
0474             }
0475         }
0476
0477         // isnan
0478         template <class A>
0479         XSIMD_INLINE batch_bool<float, A> isnan(batch<float, A> const& self, requires_arch<avx512vl_128>) noexcept
0480         {
0481             return (typename batch_bool<float, A>::register_type)_mm_cmp_ps_mask(self, self, _CMP_UNORD_Q);
0482         }
0483         template <class A>
0484         XSIMD_INLINE batch_bool<double, A> isnan(batch<double, A> const& self, requires_arch<avx512vl_128>) noexcept
0485         {
0486             return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, self, _CMP_UNORD_Q);
0487         }
0488
0489         // bitwise_rshift — signed int64 uses the native EVEX arithmetic shift
0490         // (VPSRAQ / VPSRAVQ, lat 1 / CPI 0.5). Every other width/sign keeps the
0491         // inherited avx2_128 codegen (srai/srav for 32-bit, srli for unsigned 64).
0492         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0493         XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_128>) noexcept
0494         {
0495             XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
0496             {
0497                 return _mm_srai_epi64(self, other);
0498             }
0499             else
0500             {
0501                 return bitwise_rshift(self, other, avx2_128 {});
0502             }
0503         }
0504         template <size_t shift, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0505         XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0506         {
0507             constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
0508             static_assert(shift < bits, "Shift amount must be less than the number of bits in T");
0509             XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
0510             {
0511                 return _mm_srai_epi64(self, shift);
0512             }
0513             else
0514             {
0515                 return bitwise_rshift<shift>(self, avx2_128 {});
0516             }
0517         }
0518         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0519         XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0520         {
0521             XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
0522             {
0523                 return _mm_srav_epi64(self, other);
0524             }
0525             else
0526             {
0527                 return bitwise_rshift(self, other, avx2_128 {});
0528             }
0529         }
0530
0531         // rotl
0532         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0533         XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0534         {
0535             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0536             {
0537                 return _mm_rolv_epi32(self, other);
0538             }
0539             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0540             {
0541                 return _mm_rolv_epi64(self, other);
0542             }
0543             else
0544             {
0545                 return rotl(self, other, avx2_128 {});
0546             }
0547         }
0548         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0549         XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_128>) noexcept
0550         {
0551             return rotl(self, batch<T, A>(other), A {});
0552         }
0553         template <size_t count, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0554         XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0555         {
0556             constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
0557             static_assert(count < bits, "Count must be less than the number of bits in T");
0558             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0559             {
0560                 return _mm_rol_epi32(self, count);
0561             }
0562             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0563             {
0564                 return _mm_rol_epi64(self, count);
0565             }
0566             else
0567             {
0568                 return rotl<count>(self, avx2_128 {});
0569             }
0570         }
0571
0572         // rotr
0573         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0574         XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0575         {
0576             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0577             {
0578                 return _mm_rorv_epi32(self, other);
0579             }
0580             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0581             {
0582                 return _mm_rorv_epi64(self, other);
0583             }
0584             else
0585             {
0586                 return rotr(self, other, avx2_128 {});
0587             }
0588         }
0589         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0590         XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_128>) noexcept
0591         {
0592             return rotr(self, batch<T, A>(other), A {});
0593         }
0594
0595         template <size_t count, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0596         XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0597         {
0598             constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
0599             static_assert(count < bits, "Count must be less than the number of bits in T");
0600             XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0601             {
0602                 return _mm_ror_epi32(self, count);
0603             }
0604             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0605             {
0606                 return _mm_ror_epi64(self, count);
0607             }
0608             else
0609             {
0610                 return rotr<count>(self, avx2_128 {});
0611             }
0612         }
0613
0614         // compress — native EVEX VPCOMPRESS{PS,PD,Q,D} for the widths with VL
0615         // forms. 8/16-bit need AVX512_VBMI2, so they fall through to common{}.
0616         template <class A>
0617         XSIMD_INLINE batch<float, A> compress(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_128>) noexcept
0618         {
0619             return _mm_maskz_compress_ps(mask.mask(), self);
0620         }
0621         template <class A>
0622         XSIMD_INLINE batch<double, A> compress(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_128>) noexcept
0623         {
0624             return _mm_maskz_compress_pd(mask.mask(), self);
0625         }
0626         template <class A>
0627         XSIMD_INLINE batch<int32_t, A> compress(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0628         {
0629             return _mm_maskz_compress_epi32(mask.mask(), self);
0630         }
0631         template <class A>
0632         XSIMD_INLINE batch<uint32_t, A> compress(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0633         {
0634             return _mm_maskz_compress_epi32(mask.mask(), self);
0635         }
0636         template <class A>
0637         XSIMD_INLINE batch<int64_t, A> compress(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0638         {
0639             return _mm_maskz_compress_epi64(mask.mask(), self);
0640         }
0641         template <class A>
0642         XSIMD_INLINE batch<uint64_t, A> compress(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0643         {
0644             return _mm_maskz_compress_epi64(mask.mask(), self);
0645         }
0646
0647         // expand
0648         template <class A>
0649         XSIMD_INLINE batch<float, A> expand(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_128>) noexcept
0650         {
0651             return _mm_maskz_expand_ps(mask.mask(), self);
0652         }
0653         template <class A>
0654         XSIMD_INLINE batch<double, A> expand(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_128>) noexcept
0655         {
0656             return _mm_maskz_expand_pd(mask.mask(), self);
0657         }
0658         template <class A>
0659         XSIMD_INLINE batch<int32_t, A> expand(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0660         {
0661             return _mm_maskz_expand_epi32(mask.mask(), self);
0662         }
0663         template <class A>
0664         XSIMD_INLINE batch<uint32_t, A> expand(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0665         {
0666             return _mm_maskz_expand_epi32(mask.mask(), self);
0667         }
0668         template <class A>
0669         XSIMD_INLINE batch<int64_t, A> expand(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0670         {
0671             return _mm_maskz_expand_epi64(mask.mask(), self);
0672         }
0673         template <class A>
0674         XSIMD_INLINE batch<uint64_t, A> expand(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
0675         {
0676             return _mm_maskz_expand_epi64(mask.mask(), self);
0677         }
0678
0679         // all
0680         template <class A, class T>
0681         XSIMD_INLINE bool all(batch_bool<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0682         {
0683             using register_type = typename batch_bool<T, A>::register_type;
0684             constexpr register_type bitmask = (register_type(1) << batch_bool<T, A>::size) - 1;
0685             return (self.data & bitmask) == bitmask;
0686         }
0687
0688         // any
0689         template <class A, class T>
0690         XSIMD_INLINE bool any(batch_bool<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0691         {
0692             using register_type = typename batch_bool<T, A>::register_type;
0693             constexpr register_type bitmask = (register_type(1) << batch_bool<T, A>::size) - 1;
0694             return (self.data & bitmask) != 0;
0695         }
0696
0697         // eq
0698         template <class A>
0699         XSIMD_INLINE batch_bool<float, A> eq(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_128>) noexcept
0700         {
0701             return (typename batch_bool<float, A>::register_type)_mm_cmp_ps_mask(self, other, _CMP_EQ_OQ);
0702         }
0703         template <class A>
0704         XSIMD_INLINE batch_bool<double, A> eq(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_128>) noexcept
0705         {
0706             return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, other, _CMP_EQ_OQ);
0707         }
0708
0709         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0710         XSIMD_INLINE batch_bool<T, A> eq(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0711         {
0712             return detail::compare_int_avx512vl_128<A, T, _MM_CMPINT_EQ>(self, other);
0713         }
0714         template <class A, class T>
0715         XSIMD_INLINE batch_bool<T, A> eq(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0716         {
0717             using register_type = typename batch_bool<T, A>::register_type;
0718             return register_type(~self.data ^ other.data);
0719         }
0720
0721         // neq
0722         template <class A>
0723         XSIMD_INLINE batch_bool<float, A> neq(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_128>) noexcept
0724         {
0725             return (typename batch_bool<float, A>::register_type)_mm_cmp_ps_mask(self, other, _CMP_NEQ_UQ);
0726         }
0727         template <class A>
0728         XSIMD_INLINE batch_bool<double, A> neq(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_128>) noexcept
0729         {
0730             return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, other, _CMP_NEQ_UQ);
0731         }
0732
0733         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0734         XSIMD_INLINE batch_bool<T, A> neq(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0735         {
0736             return (~(self == other));
0737         }
0738         template <class A, class T>
0739         XSIMD_INLINE batch_bool<T, A> neq(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0740         {
0741             using register_type = typename batch_bool<T, A>::register_type;
0742             return register_type(self.data ^ other.data);
0743         }
0744
0745         // gt
0746         template <class A>
0747         XSIMD_INLINE batch_bool<float, A> gt(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_128>) noexcept
0748         {
0749             return (typename batch_bool<float, A>::register_type)_mm_cmp_ps_mask(self, other, _CMP_GT_OQ);
0750         }
0751         template <class A>
0752         XSIMD_INLINE batch_bool<double, A> gt(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_128>) noexcept
0753         {
0754             return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, other, _CMP_GT_OQ);
0755         }
0756         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0757         XSIMD_INLINE batch_bool<T, A> gt(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0758         {
0759             return detail::compare_int_avx512vl_128<A, T, _MM_CMPINT_GT>(self, other);
0760         }
0761
0762         // ge
0763         template <class A>
0764         XSIMD_INLINE batch_bool<float, A> ge(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_128>) noexcept
0765         {
0766             return (typename batch_bool<float, A>::register_type)_mm_cmp_ps_mask(self, other, _CMP_GE_OQ);
0767         }
0768         template <class A>
0769         XSIMD_INLINE batch_bool<double, A> ge(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_128>) noexcept
0770         {
0771             return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, other, _CMP_GE_OQ);
0772         }
0773
0774         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0775         XSIMD_INLINE batch_bool<T, A> ge(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0776         {
0777             return detail::compare_int_avx512vl_128<A, T, _MM_CMPINT_GE>(self, other);
0778         }
0779
0780         // lt
0781         template <class A>
0782         XSIMD_INLINE batch_bool<float, A> lt(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_128>) noexcept
0783         {
0784             return (typename batch_bool<float, A>::register_type)_mm_cmp_ps_mask(self, other, _CMP_LT_OQ);
0785         }
0786         template <class A>
0787         XSIMD_INLINE batch_bool<double, A> lt(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_128>) noexcept
0788         {
0789             return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, other, _CMP_LT_OQ);
0790         }
0791
0792         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0793         XSIMD_INLINE batch_bool<T, A> lt(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0794         {
0795             return detail::compare_int_avx512vl_128<A, T, _MM_CMPINT_LT>(self, other);
0796         }
0797
0798         // le
0799         template <class A>
0800         XSIMD_INLINE batch_bool<float, A> le(batch<float, A> const& self, batch<float, A> const& other, requires_arch<avx512vl_128>) noexcept
0801         {
0802             return (typename batch_bool<float, A>::register_type)_mm_cmp_ps_mask(self, other, _CMP_LE_OQ);
0803         }
0804         template <class A>
0805         XSIMD_INLINE batch_bool<double, A> le(batch<double, A> const& self, batch<double, A> const& other, requires_arch<avx512vl_128>) noexcept
0806         {
0807             return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, other, _CMP_LE_OQ);
0808         }
0809
0810         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0811         XSIMD_INLINE batch_bool<T, A> le(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0812         {
0813             return detail::compare_int_avx512vl_128<A, T, _MM_CMPINT_LE>(self, other);
0814         }
0815
0816         // select
0817         template <class A>
0818         XSIMD_INLINE batch<float, A> select(batch_bool<float, A> const& cond, batch<float, A> const& true_br, batch<float, A> const& false_br, requires_arch<avx512vl_128>) noexcept
0819         {
0820             return _mm_mask_blend_ps(cond, false_br, true_br);
0821         }
0822         template <class A>
0823         XSIMD_INLINE batch<double, A> select(batch_bool<double, A> const& cond, batch<double, A> const& true_br, batch<double, A> const& false_br, requires_arch<avx512vl_128>) noexcept
0824         {
0825             return _mm_mask_blend_pd(cond, false_br, true_br);
0826         }
0827         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0828         XSIMD_INLINE batch<T, A> select(batch_bool<T, A> const& cond, batch<T, A> const& true_br, batch<T, A> const& false_br, requires_arch<avx512vl_128>) noexcept
0829         {
0830             XSIMD_IF_CONSTEXPR(sizeof(T) == 1)
0831             {
0832                 batch_bool<T, avx2_128> batch_cond = batch_bool<T, avx2_128>::from_mask(cond.mask());
0833                 return _mm_blendv_epi8(false_br, true_br, batch_cond);
0834             }
0835             else XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
0836             {
0837                 batch_bool<T, avx2_128> batch_cond = batch_bool<T, avx2_128>::from_mask(cond.mask());
0838                 return _mm_blendv_epi8(false_br, true_br, batch_cond);
0839             }
0840             else XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
0841             {
0842                 return _mm_mask_blend_epi32(cond, false_br, true_br);
0843             }
0844             else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
0845             {
0846                 return _mm_mask_blend_epi64(cond, false_br, true_br);
0847             }
0848         }
0849         template <class A, class T, bool... Values>
0850         XSIMD_INLINE batch<T, A> select(batch_bool_constant<T, A, Values...> const&, batch<T, A> const& true_br, batch<T, A> const& false_br, requires_arch<avx512vl_128>) noexcept
0851         {
0852             return select(batch_bool<T, A> { Values... }, true_br, false_br, avx512vl_128 {});
0853         }
0854
0855         // decr_if / incr_if — the inherited avx kernels compute
0856         // `self ± batch<T>(mask.data)`, which assumes a vector batch_bool whose
0857         // true lanes are all-ones. Here batch_bool::data is a k-mask bitfield,
0858         // so that broadcast yields garbage. Delegate to the select-based common
0859         // implementation instead.
0860         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0861         XSIMD_INLINE batch<T, A> decr_if(batch<T, A> const& self, batch_bool<T, A> const& mask, requires_arch<avx512vl_128>) noexcept
0862         {
0863             return decr_if(self, mask, common {});
0864         }
0865         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0866         XSIMD_INLINE batch<T, A> incr_if(batch<T, A> const& self, batch_bool<T, A> const& mask, requires_arch<avx512vl_128>) noexcept
0867         {
0868             return incr_if(self, mask, common {});
0869         }
0870
0871         // reciprocal
0872         template <class A>
0873         XSIMD_INLINE batch<float, A>
0874         reciprocal(batch<float, A> const& self,
0875                    kernel::requires_arch<avx512vl_128>) noexcept
0876         {
0877             return _mm_rcp14_ps(self);
0878         }
0879
0880         template <class A>
0881         XSIMD_INLINE batch<double, A>
0882         reciprocal(batch<double, A> const& self,
0883                    kernel::requires_arch<avx512vl_128>) noexcept
0884         {
0885             return _mm_rcp14_pd(self);
0886         }
0887
0888         // bitwise_and
0889         template <class A, class T>
0890         XSIMD_INLINE batch_bool<T, A> bitwise_and(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0891         {
0892             using register_type = typename batch_bool<T, A>::register_type;
0893             return register_type(self.data & other.data);
0894         }
0895
0896         // bitwise_andnot
0897         template <class A, class T>
0898         XSIMD_INLINE batch_bool<T, A> bitwise_andnot(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0899         {
0900             using register_type = typename batch_bool<T, A>::register_type;
0901             return register_type(self.data & ~other.data);
0902         }
0903
0904         // bitwise_not
0905         template <class A, class T>
0906         XSIMD_INLINE batch_bool<T, A> bitwise_not(batch_bool<T, A> const& self, requires_arch<avx512vl_128>) noexcept
0907         {
0908             using register_type = typename batch_bool<T, A>::register_type;
0909             return register_type(~self.data);
0910         }
0911
0912         // bitwise_or
0913         template <class A, class T>
0914         XSIMD_INLINE batch_bool<T, A> bitwise_or(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0915         {
0916             using register_type = typename batch_bool<T, A>::register_type;
0917             return register_type(self.data | other.data);
0918         }
0919
0920         // bitwise_xor
0921         template <class A, class T>
0922         XSIMD_INLINE batch_bool<T, A> bitwise_xor(batch_bool<T, A> const& self, batch_bool<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0923         {
0924             using register_type = typename batch_bool<T, A>::register_type;
0925             return register_type(self.data ^ other.data);
0926         }
0927
0928         // sadd
0929         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
0930         XSIMD_INLINE batch<T, A> sadd(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
0931         {
0932             if (std::is_signed<T>::value)
0933             {
0934                 auto mask = other < 0;
0935                 auto self_pos_branch = min(std::numeric_limits<T>::max() - other, self);
0936                 auto self_neg_branch = max(std::numeric_limits<T>::min() - other, self);
0937                 return other + select(mask, self_neg_branch, self_pos_branch);
0938             }
0939             else
0940             {
0941                 const auto diffmax = std::numeric_limits<T>::max() - self;
0942                 const auto mindiff = min(diffmax, other);
0943                 return self + mindiff;
0944             }
0945         }
0946
0947     }
0948 }
0949
0950 #endif