Back to home page

EIC code displayed by LXR
 
 

    

File indexing completed on 2026-11-07 10:36:57 

0001 /***************************************************************************
0002  * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
0003  * Martin Renou                                                             *
0004  * Copyright (c) QuantStack                                                 *
0005  * Copyright (c) Serge Guelton                                              *
0006  *                                                                          *
0007  * Distributed under the terms of the BSD 3-Clause License.                 *
0008  *                                                                          *
0009  * The full license is in the file LICENSE, distributed with this software. *
0010  ****************************************************************************/
0011 
0012 #ifndef XSIMD_COMMON_DETAILS_HPP
0013 #define XSIMD_COMMON_DETAILS_HPP
0014 
0015 #include "../../math/xsimd_rem_pio2.hpp"
0016 #include "../../types/xsimd_common_arch.hpp"
0017 #include "../../types/xsimd_utils.hpp"
0018 #include "../xsimd_constants.hpp"
0019 
0020 #include <complex>
0021 
0022 namespace xsimd
0023 {
0024     // Forward declaration. Should we put them in a separate file?
0025     template <class T, class A>
0026     XSIMD_INLINE batch<T, A> abs(batch<T, A> const& self) noexcept;
0027     template <class T, class A>
0028     XSIMD_INLINE batch<T, A> abs(batch<std::complex<T>, A> const& self) noexcept;
0029     template <class T, class A>
0030     XSIMD_INLINE bool any(batch_bool<T, A> const& self) noexcept;
0031     template <class T, class A>
0032     XSIMD_INLINE batch<T, A> atan2(batch<T, A> const& self, batch<T, A> const& other) noexcept;
0033     template <class A, class T_out, class T_in>
0034     XSIMD_INLINE batch<T_out, A> batch_cast(batch<T_in, A> const&, batch<T_out, A> const& out) noexcept;
0035     template <class T, class A>
0036     XSIMD_INLINE batch<T, A> bitofsign(batch<T, A> const& self) noexcept;
0037     template <class T_out, class T_in, class A>
0038     XSIMD_INLINE batch<T_out, A> bitwise_cast(batch<T_in, A> const& self) noexcept;
0039     template <class T, class A>
0040     XSIMD_INLINE batch<T, A> cos(batch<T, A> const& self) noexcept;
0041     template <class T, class A>
0042     XSIMD_INLINE batch<T, A> cosh(batch<T, A> const& self) noexcept;
0043     template <class T, class A>
0044     XSIMD_INLINE batch<T, A> exp(batch<T, A> const& self) noexcept;
0045     template <class T, class A>
0046     XSIMD_INLINE batch<T, A> fma(batch<T, A> const& x, batch<T, A> const& y, batch<T, A> const& z) noexcept;
0047     template <class T, class A>
0048     XSIMD_INLINE batch<T, A> fms(batch<T, A> const& x, batch<T, A> const& y, batch<T, A> const& z) noexcept;
0049     template <class T, class A>
0050     XSIMD_INLINE batch<T, A> fmas(batch<T, A> const& x, batch<T, A> const& y, batch<T, A> const& z) noexcept;
0051     template <class T, class A>
0052     XSIMD_INLINE batch<T, A> frexp(const batch<T, A>& x, const batch<as_integer_t<T>, A>& e) noexcept;
0053     template <class T, class A, uint64_t... Coefs>
0054     XSIMD_INLINE batch<T, A> horner(const batch<T, A>& self) noexcept;
0055     template <class T, class A>
0056     XSIMD_INLINE batch<T, A> hypot(const batch<T, A>& self) noexcept;
0057     template <class T, class A>
0058     XSIMD_INLINE batch_bool<T, A> is_even(batch<T, A> const& self) noexcept;
0059     template <class T, class A>
0060     XSIMD_INLINE batch_bool<T, A> is_flint(batch<T, A> const& self) noexcept;
0061     template <class T, class A>
0062     XSIMD_INLINE batch_bool<T, A> is_odd(batch<T, A> const& self) noexcept;
0063     template <class T, class A>
0064     XSIMD_INLINE typename batch<T, A>::batch_bool_type isinf(batch<T, A> const& self) noexcept;
0065     template <class T, class A>
0066     XSIMD_INLINE typename batch<T, A>::batch_bool_type isfinite(batch<T, A> const& self) noexcept;
0067     template <class T, class A>
0068     XSIMD_INLINE typename batch<T, A>::batch_bool_type isnan(batch<T, A> const& self) noexcept;
0069     template <class T, class A>
0070     XSIMD_INLINE batch<T, A> ldexp(const batch<T, A>& x, const batch<as_integer_t<T>, A>& e) noexcept;
0071     template <class T, class A>
0072     XSIMD_INLINE batch<T, A> log(batch<T, A> const& self) noexcept;
0073     template <class T, class A>
0074     XSIMD_INLINE batch<T, A> nearbyint(batch<T, A> const& self) noexcept;
0075     template <class T, class A>
0076     XSIMD_INLINE batch<as_integer_t<T>, A> nearbyint_as_int(const batch<T, A>& x) noexcept;
0077     template <class T, class A>
0078     XSIMD_INLINE T reduce_add(batch<T, A> const&) noexcept;
0079     template <class T, class A>
0080     XSIMD_INLINE T reduce_mul(batch<T, A> const&) noexcept;
0081     template <class T, class A>
0082     XSIMD_INLINE batch<T, A> select(batch_bool<T, A> const&, batch<T, A> const&, batch<T, A> const&) noexcept;
0083     template <class T, class A>
0084     XSIMD_INLINE batch<std::complex<T>, A> select(batch_bool<T, A> const&, batch<std::complex<T>, A> const&, batch<std::complex<T>, A> const&) noexcept;
0085     template <class T, class A>
0086     XSIMD_INLINE batch<T, A> sign(batch<T, A> const& self) noexcept;
0087     template <class T, class A>
0088     XSIMD_INLINE batch<T, A> signnz(batch<T, A> const& self) noexcept;
0089     template <class T, class A>
0090     XSIMD_INLINE batch<T, A> sin(batch<T, A> const& self) noexcept;
0091     template <class T, class A>
0092     XSIMD_INLINE batch<T, A> sinh(batch<T, A> const& self) noexcept;
0093     template <class T, class A>
0094     XSIMD_INLINE std::pair<batch<T, A>, batch<T, A>> sincos(batch<T, A> const& self) noexcept;
0095     template <class T, class A>
0096     XSIMD_INLINE batch<T, A> sqrt(batch<T, A> const& self) noexcept;
0097     template <class T, class A, class Vt, Vt... Values>
0098     XSIMD_INLINE std::enable_if_t<std::is_arithmetic<T>::value, batch<T, A>>
0099     swizzle(batch<T, A> const& x, batch_constant<Vt, A, Values...> mask) noexcept;
0100     template <class T, class A>
0101     XSIMD_INLINE batch<T, A> tan(batch<T, A> const& self) noexcept;
0102     template <class T, class A>
0103     XSIMD_INLINE batch<as_float_t<T>, A> to_float(batch<T, A> const& self) noexcept;
0104     template <class T, class A>
0105     XSIMD_INLINE batch<as_integer_t<T>, A> to_int(batch<T, A> const& self) noexcept;
0106     template <class T, class A>
0107     XSIMD_INLINE batch<T, A> trunc(batch<T, A> const& self) noexcept;
0108 
0109     namespace kernel
0110     {
0111 
0112         namespace detail
0113         {
0114             // Prevent -ffast-math from reassociating floating-point
0115             // arithmetic across this point.  The reason string
0116             // documents *why* at each call site; unused at runtime.
0117             //
0118             // Zero-cost register constraints per target:
0119             //   x86  "+x"  (XMM/YMM/ZMM, also scalar float/double)
0120             //   ARM  "+w"  (V-reg / SVE Z-reg, also scalar float/double)
0121             //   PPC  "+wa" (VS register, also scalar float/double)
0122             //   RISC-V "+f" (F/D register, scalar float/double)
0123             //   RISC-V RVV "+vr" (V register; GCC 15+ / Clang 20+)
0124             //
0125             // On unknown targets the "+m" fallback spills; it is
0126             // only emitted when the compiler can actually reassociate.
0127             template <class T>
0128             XSIMD_INLINE void reassociation_barrier(T& x, const char*) noexcept
0129             {
0130 #if XSIMD_REASSOCIATIVE_MATH && XSIMD_WITH_INLINE_ASM && !defined(__EMSCRIPTEN__)
0131 #if XSIMD_WITH_SSE2
0132                 __asm__ volatile("" : "+x"(x));
0133 #elif XSIMD_WITH_NEON || XSIMD_WITH_SVE
0134                 __asm__ volatile("" : "+w"(x));
0135 #elif XSIMD_WITH_VSX
0136                 __asm__ volatile("" : "+wa"(x));
0137 #else
0138                 __asm__ volatile("" : "+m"(x));
0139 #endif
0140 #else
0141                 (void)x;
0142 #endif
0143             }
0144 
0145             // RISC-V scalar float/double: use F/D registers instead of
0146             // spilling through "+m".  These overloads also serve
0147             // emulated batches on RISC-V via the std::array overload.
0148 #if XSIMD_REASSOCIATIVE_MATH && XSIMD_WITH_INLINE_ASM && defined(__riscv)
0149             XSIMD_INLINE void reassociation_barrier(float& x, const char*) noexcept
0150             {
0151                 __asm__ volatile("" : "+f"(x));
0152             }
0153             XSIMD_INLINE void reassociation_barrier(double& x, const char*) noexcept
0154             {
0155                 __asm__ volatile("" : "+f"(x));
0156             }
0157 #endif
0158 
0159             template <class T, size_t N>
0160             XSIMD_INLINE void reassociation_barrier(std::array<T, N>& arr, const char* reason) noexcept
0161             {
0162                 for (auto& v : arr)
0163                     reassociation_barrier(v, reason);
0164             }
0165 
0166             template <class T, class A>
0167             XSIMD_INLINE void reassociation_barrier(batch<T, A>& b, const char* reason) noexcept
0168             {
0169 #if XSIMD_REASSOCIATIVE_MATH && XSIMD_WITH_RVV && XSIMD_WITH_INLINE_ASM && ((__GNUC__ >= 15) || (__clang_major__ >= 20))
0170                 __asm__ volatile("" : "+vr"(b.data.value.value));
0171                 (void)reason;
0172 #else
0173                 reassociation_barrier(b.data, reason);
0174 #endif
0175             }
0176 
0177             template <class F, class A, class T, class... Batches>
0178             XSIMD_INLINE batch<T, A> apply(F&& func, batch<T, A> const& self, batch<T, A> const& other) noexcept
0179             {
0180                 constexpr std::size_t size = batch<T, A>::size;
0181                 alignas(A::alignment()) T self_buffer[size];
0182                 alignas(A::alignment()) T other_buffer[size];
0183                 self.store_aligned(&self_buffer[0]);
0184                 other.store_aligned(&other_buffer[0]);
0185                 for (std::size_t i = 0; i < size; ++i)
0186                 {
0187                     self_buffer[i] = func(self_buffer[i], other_buffer[i]);
0188                 }
0189                 return batch<T, A>::load_aligned(self_buffer);
0190             }
0191 
0192             template <class U, class F, class A, class T>
0193             XSIMD_INLINE batch<U, A> apply_transform(F&& func, batch<T, A> const& self) noexcept
0194             {
0195                 static_assert(batch<T, A>::size == batch<U, A>::size,
0196                               "Source and destination sizes must match");
0197                 constexpr std::size_t src_size = batch<T, A>::size;
0198                 constexpr std::size_t dest_size = batch<U, A>::size;
0199                 alignas(A::alignment()) T self_buffer[src_size];
0200                 alignas(A::alignment()) U other_buffer[dest_size];
0201                 self.store_aligned(&self_buffer[0]);
0202                 for (std::size_t i = 0; i < src_size; ++i)
0203                 {
0204                     other_buffer[i] = func(self_buffer[i]);
0205                 }
0206                 return batch<U, A>::load_aligned(other_buffer);
0207             }
0208         }
0209 
0210         // some common fast_cast conversion
0211         namespace detail
0212         {
0213             template <class A>
0214             XSIMD_INLINE batch<uint8_t, A> fast_cast(batch<int8_t, A> const& self, batch<uint8_t, A> const&, requires_arch<common>) noexcept
0215             {
0216                 return bitwise_cast<uint8_t>(self);
0217             }
0218             template <class A>
0219             XSIMD_INLINE batch<uint16_t, A> fast_cast(batch<int16_t, A> const& self, batch<uint16_t, A> const&, requires_arch<common>) noexcept
0220             {
0221                 return bitwise_cast<uint16_t>(self);
0222             }
0223             template <class A>
0224             XSIMD_INLINE batch<uint32_t, A> fast_cast(batch<int32_t, A> const& self, batch<uint32_t, A> const&, requires_arch<common>) noexcept
0225             {
0226                 return bitwise_cast<uint32_t>(self);
0227             }
0228             template <class A>
0229             XSIMD_INLINE batch<uint64_t, A> fast_cast(batch<int64_t, A> const& self, batch<uint64_t, A> const&, requires_arch<common>) noexcept
0230             {
0231                 return bitwise_cast<uint64_t>(self);
0232             }
0233             template <class A>
0234             XSIMD_INLINE batch<int8_t, A> fast_cast(batch<uint8_t, A> const& self, batch<int8_t, A> const&, requires_arch<common>) noexcept
0235             {
0236                 return bitwise_cast<int8_t>(self);
0237             }
0238             template <class A>
0239             XSIMD_INLINE batch<int16_t, A> fast_cast(batch<uint16_t, A> const& self, batch<int16_t, A> const&, requires_arch<common>) noexcept
0240             {
0241                 return bitwise_cast<int16_t>(self);
0242             }
0243             template <class A>
0244             XSIMD_INLINE batch<int32_t, A> fast_cast(batch<uint32_t, A> const& self, batch<int32_t, A> const&, requires_arch<common>) noexcept
0245             {
0246                 return bitwise_cast<int32_t>(self);
0247             }
0248             template <class A>
0249             XSIMD_INLINE batch<int64_t, A> fast_cast(batch<uint64_t, A> const& self, batch<int64_t, A> const&, requires_arch<common>) noexcept
0250             {
0251                 return bitwise_cast<int64_t>(self);
0252             }
0253 
0254             // Provide a common uint32_t -> float cast only if we have a
0255             // non-common int32_t -> float fast_cast
0256             template <class A, class = decltype(fast_cast(std::declval<batch<int32_t, A> const&>(), std::declval<batch<float, A> const&>(), A {}))>
0257             XSIMD_INLINE batch<float, A> fast_cast(batch<uint32_t, A> const& v, batch<float, A> const&, requires_arch<common>) noexcept
0258             {
0259                 // see https://stackoverflow.com/questions/34066228/how-to-perform-uint32-float-conversion-with-sse
0260                 batch<uint32_t, A> msk_lo(0xFFFF);
0261                 batch<float, A> cnst65536f(65536.0f);
0262 
0263                 auto v_lo = batch_cast<int32_t>(v & msk_lo); /* extract the 16 lowest significant bits of self                             */
0264                 auto v_hi = batch_cast<int32_t>(v >> 16); /* 16 most significant bits of v                                                 */
0265                 auto v_lo_flt = batch_cast<float>(v_lo); /* No rounding                                                                */
0266                 auto v_hi_flt = batch_cast<float>(v_hi); /* No rounding                                                                */
0267                 v_hi_flt = cnst65536f * v_hi_flt; /* No rounding                                                            */
0268                 return v_hi_flt + v_lo_flt; /* Rounding may occur here, mul and add may fuse to fma for haswell and newer   */
0269             }
0270 
0271             // Provide a common float -> uint32_t cast only if we have a
0272             // non-common float -> int32_t fast_cast
0273             template <class A, class = decltype(fast_cast(std::declval<batch<float, A> const&>(), std::declval<batch<int32_t, A> const&>(), A {}))>
0274             XSIMD_INLINE batch<uint32_t, A> fast_cast(batch<float, A> const& v, batch<uint32_t, A> const&, requires_arch<common>) noexcept
0275             {
0276                 auto is_large = v >= batch<float, A>(1u << 31);
0277                 auto small_v = bitwise_cast<float>(batch_cast<int32_t>(v));
0278                 auto large_v = bitwise_cast<float>(
0279                     batch_cast<int32_t>(v - batch<float, A>(1u << 31))
0280                     ^ batch<int32_t, A>(1u << 31));
0281                 return bitwise_cast<uint32_t>(select(is_large, large_v, small_v));
0282             }
0283         }
0284 
0285         namespace detail
0286         {
0287             // Generic conversion handling machinery. Each architecture must define
0288             // conversion function when such conversions exits in the form of
0289             // intrinsic. Then we use that information to automatically decide whether
0290             // to use scalar or vector conversion when doing load / store / batch_cast
0291             struct with_fast_conversion
0292             {
0293             };
0294             struct with_slow_conversion
0295             {
0296             };
0297 
0298             template <class A, class From, class To, class = void>
0299             struct conversion_type_impl
0300             {
0301                 using type = with_slow_conversion;
0302             };
0303 
0304             using xsimd::detail::void_t;
0305 
0306             template <class A, class From, class To>
0307             struct conversion_type_impl<A, From, To,
0308                                         void_t<decltype(fast_cast(std::declval<const batch<From, A>&>(),
0309                                                                   std::declval<const batch<To, A>&>(),
0310                                                                   std::declval<const A&>()))>>
0311             {
0312                 using type = with_fast_conversion;
0313             };
0314 
0315             template <class A, class From, class To>
0316             using conversion_type = typename conversion_type_impl<A, From, To>::type;
0317         }
0318 
0319         namespace detail
0320         {
0321             /* origin: boost/simdfunction/horn.hpp*/
0322             /*
0323              * ====================================================
0324              * copyright 2016 NumScale SAS
0325              *
0326              * Distributed under the Boost Software License, Version 1.0.
0327              * (See copy at http://boost.org/LICENSE_1_0.txt)
0328              * ====================================================
0329              */
0330             template <class B, uint64_t c>
0331             XSIMD_INLINE B coef() noexcept
0332             {
0333                 using value_type = typename B::value_type;
0334                 return B(bit_cast<value_type>(as_unsigned_integer_t<value_type>(c)));
0335             }
0336             template <class B>
0337             XSIMD_INLINE B horner(const B&) noexcept
0338             {
0339                 return B(typename B::value_type(0.));
0340             }
0341 
0342             template <class B, uint64_t c0>
0343             XSIMD_INLINE B horner(const B&) noexcept
0344             {
0345                 return coef<B, c0>();
0346             }
0347 
0348             template <class B, uint64_t c0, uint64_t c1, uint64_t... args>
0349             XSIMD_INLINE B horner(const B& self) noexcept
0350             {
0351                 return fma(self, horner<B, c1, args...>(self), coef<B, c0>());
0352             }
0353 
0354             /* origin: boost/simdfunction/horn1.hpp*/
0355             /*
0356              * ====================================================
0357              * copyright 2016 NumScale SAS
0358              *
0359              * Distributed under the Boost Software License, Version 1.0.
0360              * (See copy at http://boost.org/LICENSE_1_0.txt)
0361              * ====================================================
0362              */
0363             template <class B>
0364             XSIMD_INLINE B horner1(const B&) noexcept
0365             {
0366                 return B(1.);
0367             }
0368 
0369             template <class B, uint64_t c0>
0370             XSIMD_INLINE B horner1(const B& x) noexcept
0371             {
0372                 return x + detail::coef<B, c0>();
0373             }
0374 
0375             template <class B, uint64_t c0, uint64_t c1, uint64_t... args>
0376             XSIMD_INLINE B horner1(const B& x) noexcept
0377             {
0378                 return fma(x, horner1<B, c1, args...>(x), detail::coef<B, c0>());
0379             }
0380         }
0381 
0382     }
0383 
0384 }
0385 
0386 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.3.7 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!