arch/common/xsimd_common_swizzle.hpp

0001 /***************************************************************************
0002  * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
0003  * Martin Renou                                                             *
0004  * Copyright (c) QuantStack                                                 *
0005  * Copyright (c) Serge Guelton                                              *
0006  * Copyright (c) Marco Barbone                                              *
0007  *                                                                          *
0008  * Distributed under the terms of the BSD 3-Clause License.                 *
0009  *                                                                          *
0010  * The full license is in the file LICENSE, distributed with this software.*
0011  ****************************************************************************/
0012 #ifndef XSIMD_COMMON_SWIZZLE_HPP
0013 #define XSIMD_COMMON_SWIZZLE_HPP
0014
0015 #include "../../config/xsimd_macros.hpp"
0016
0017 #include <cstddef>
0018 #include <cstdint>
0019 #include <type_traits>
0020
0021 namespace xsimd
0022 {
0023     template <typename T, class A, T... Values>
0024     struct batch_constant;
0025
0026     namespace kernel
0027     {
0028         namespace detail
0029         {
0030             // ────────────────────────────────────────────────────────────────────────
0031             //  get_at<I,Values...> → the I-th element of the pack
0032             template <typename T, std::size_t I, T V0, T... Vs>
0033             struct get_at
0034             {
0035                 static constexpr T value = get_at<T, I - 1, Vs...>::value;
0036             };
0037             template <typename T, T V0, T... Vs>
0038             struct get_at<T, 0, V0, Vs...>
0039             {
0040                 static constexpr T value = V0;
0041             };
0042
0043             // ────────────────────────────────────────────────────────────────────────
0044             // identity_impl
0045             template <std::size_t /*I*/, typename T>
0046             XSIMD_INLINE constexpr bool identity_impl() noexcept { return true; }
0047             template <std::size_t I, typename T, T V0, T... Vs>
0048             XSIMD_INLINE constexpr bool identity_impl() noexcept
0049             {
0050                 return V0 == static_cast<T>(I)
0051                     && identity_impl<I + 1, T, Vs...>();
0052             }
0053
0054             // ────────────────────────────────────────────────────────────────────────
0055             // dup_lo_impl
0056             template <std::size_t I, std::size_t N, typename T,
0057                       T... Vs, std::enable_if_t<I == N / 2, int> = 0>
0058             XSIMD_INLINE constexpr bool dup_lo_impl() noexcept { return true; }
0059
0060             template <std::size_t I, std::size_t N, typename T,
0061                       T... Vs, std::enable_if_t<(I < N / 2), int> = 0>
0062             XSIMD_INLINE constexpr bool dup_lo_impl() noexcept
0063             {
0064                 return get_at<T, I, Vs...>::value < static_cast<T>(N / 2)
0065                     && get_at<T, I + N / 2, Vs...>::value == get_at<T, I, Vs...>::value
0066                     && dup_lo_impl<I + 1, N, T, Vs...>();
0067             }
0068
0069             // ────────────────────────────────────────────────────────────────────────
0070             // dup_hi_impl
0071             template <std::size_t I, std::size_t N, typename T,
0072                       T... Vs, std::enable_if_t<I == N / 2, int> = 0>
0073             XSIMD_INLINE constexpr bool dup_hi_impl() noexcept { return true; }
0074
0075             template <std::size_t I, std::size_t N, typename T,
0076                       T... Vs, std::enable_if_t<(I < N / 2), int> = 0>
0077             XSIMD_INLINE constexpr bool dup_hi_impl() noexcept
0078             {
0079                 return get_at<T, I, Vs...>::value >= static_cast<T>(N / 2)
0080                     && get_at<T, I, Vs...>::value < static_cast<T>(N)
0081                     && get_at<T, I + N / 2, Vs...>::value == get_at<T, I, Vs...>::value
0082                     && dup_hi_impl<I + 1, N, T, Vs...>();
0083             }
0084
0085             // ────────────────────────────────────────────────────────────────────────
0086             // only_from_lo
0087             template <typename T, T Size, T First, T... Vals>
0088             struct only_from_lo_impl;
0089
0090             template <typename T, T Size, T Last>
0091             struct only_from_lo_impl<T, Size, Last>
0092             {
0093                 static constexpr bool value = (Last < (Size / 2));
0094             };
0095
0096             template <typename T, T Size, T First, T... Vals>
0097             struct only_from_lo_impl
0098             {
0099                 static constexpr bool value = (First < (Size / 2)) && only_from_lo_impl<T, Size, Vals...>::value;
0100             };
0101
0102             template <typename T, T... Vals>
0103             constexpr bool is_only_from_lo()
0104             {
0105                 return only_from_lo_impl<T, sizeof...(Vals), Vals...>::value;
0106             };
0107
0108             // ────────────────────────────────────────────────────────────────────────
0109             // only_from_hi
0110             template <typename T, T Size, T First, T... Vals>
0111             struct only_from_hi_impl;
0112
0113             template <typename T, T Size, T Last>
0114             struct only_from_hi_impl<T, Size, Last>
0115             {
0116                 static constexpr bool value = (Last >= (Size / 2));
0117             };
0118
0119             template <typename T, T Size, T First, T... Vals>
0120             struct only_from_hi_impl
0121             {
0122                 static constexpr bool value = (First >= (Size / 2)) && only_from_hi_impl<T, Size, Vals...>::value;
0123             };
0124
0125             template <typename T, T... Vals>
0126             constexpr bool is_only_from_hi()
0127             {
0128                 return only_from_hi_impl<T, sizeof...(Vals), Vals...>::value;
0129             };
0130
0131             // ────────────────────────────────────────────────────────────────────────
0132             //  1) helper to get the I-th value from the Vs pack
0133             template <std::size_t I, uint32_t Head, uint32_t... Tail>
0134             struct get_nth_value
0135             {
0136                 static constexpr uint32_t value = get_nth_value<I - 1, Tail...>::value;
0137             };
0138             template <uint32_t Head, uint32_t... Tail>
0139             struct get_nth_value<0, Head, Tail...>
0140             {
0141                 static constexpr uint32_t value = Head;
0142             };
0143
0144             // ────────────────────────────────────────────────────────────────────────
0145             //  2) recursive cross‐lane test: true if any output‐lane i pulls from the opposite half
0146             template <std::size_t I,
0147                       std::size_t N,
0148                       std::size_t H,
0149                       uint32_t... Vs>
0150             struct cross_impl
0151             {
0152                 // does element I cross? (i.e. i<H but V>=H) or (i>=H but V<H)
0153                 static constexpr uint32_t Vi = get_nth_value<I, Vs...>::value;
0154                 static constexpr bool curr = (I < H ? (Vi >= H) : (Vi < H));
0155                 static constexpr bool next = cross_impl<I + 1, N, H, Vs...>::value;
0156                 static constexpr bool value = curr || next;
0157             };
0158             template <std::size_t N, std::size_t H, uint32_t... Vs>
0159             struct cross_impl<N, N, H, Vs...>
0160             {
0161                 static constexpr bool value = false;
0162             };
0163             template <uint32_t... Vs>
0164             XSIMD_INLINE constexpr bool is_cross_lane() noexcept
0165             {
0166                 static_assert(sizeof...(Vs) >= 1, "Need at least one lane");
0167                 return cross_impl<0, sizeof...(Vs), sizeof...(Vs) / 2, Vs...>::value;
0168             }
0169
0170             /**
0171              * @brief Internal: Check if a swizzle pattern crosses lane boundaries
0172              *
0173              * @tparam LaneSizeBytes Size of a lane in bytes (must be > 0)
0174              * @tparam ElemT Element type to determine element size
0175              * @tparam U Type of the index values
0176              * @tparam Vs... Index values for the swizzle pattern
0177              *
0178              * @return true if any element accesses data from a different lane
0179              *
0180              * This is an internal helper. Architecture-specific code can call this directly
0181              * with explicit lane sizes (e.g., detail::is_cross_lane_with_lane_size<16, float, ...>()
0182              * for 128-bit lanes).
0183              */
0184             template <std::size_t LaneSizeBytes, typename ElemT, typename U, U... Vs>
0185             XSIMD_INLINE constexpr bool is_cross_lane_with_lane_size() noexcept
0186             {
0187                 static_assert(std::is_integral<U>::value, "swizzle mask values must be integral");
0188                 static_assert(sizeof...(Vs) >= 1, "need at least one value");
0189                 static_assert(LaneSizeBytes > 0, "lane size must be positive");
0190
0191                 constexpr std::size_t lane_elems = LaneSizeBytes / sizeof(ElemT);
0192                 constexpr U values[] = { Vs... };
0193                 constexpr std::size_t N = sizeof...(Vs);
0194
0195                 for (std::size_t i = 0; i < N; ++i)
0196                 {
0197                     std::size_t elem_lane = i / lane_elems;
0198                     std::size_t target_lane = static_cast<std::size_t>(values[i]) / lane_elems;
0199                     if (elem_lane != target_lane)
0200                         return true;
0201                 }
0202                 return false;
0203             }
0204
0205             template <typename T, T... Vs>
0206             XSIMD_INLINE constexpr bool is_identity() noexcept { return detail::identity_impl<0, T, Vs...>(); }
0207             template <typename T, T... Vs>
0208             XSIMD_INLINE constexpr bool is_dup_lo() noexcept { return detail::dup_lo_impl<0, sizeof...(Vs), T, Vs...>(); }
0209             template <typename T, T... Vs>
0210             XSIMD_INLINE constexpr bool is_dup_hi() noexcept { return detail::dup_hi_impl<0, sizeof...(Vs), T, Vs...>(); }
0211             template <typename T, class A, T... Vs>
0212             XSIMD_INLINE constexpr bool is_identity(batch_constant<T, A, Vs...>) noexcept { return is_identity<T, Vs...>(); }
0213             template <typename T, class A, T... Vs>
0214             XSIMD_INLINE constexpr bool is_dup_lo(batch_constant<T, A, Vs...>) noexcept { return is_dup_lo<T, Vs...>(); }
0215             template <typename T, class A, T... Vs>
0216             XSIMD_INLINE constexpr bool is_dup_hi(batch_constant<T, A, Vs...>) noexcept { return is_dup_hi<T, Vs...>(); }
0217             template <typename T, class A, T... Vs>
0218             XSIMD_INLINE constexpr bool is_only_from_lo(batch_constant<T, A, Vs...>) noexcept { return detail::is_only_from_lo<T, Vs...>(); }
0219             template <typename T, class A, T... Vs>
0220             XSIMD_INLINE constexpr bool is_only_from_hi(batch_constant<T, A, Vs...>) noexcept { return detail::is_only_from_hi<T, Vs...>(); }
0221             template <typename T, class A, T... Vs>
0222             XSIMD_INLINE constexpr bool is_cross_lane(batch_constant<T, A, Vs...>) noexcept
0223             {
0224                 return detail::is_cross_lane_with_lane_size<16, T, T, Vs...>();
0225             }
0226
0227             /**
0228              * @brief Public: Check if a swizzle pattern crosses 128-bit lane boundaries
0229              *
0230              * Checks if indices cross 128-bit (16-byte) lane boundaries, which is the
0231              * standard lane size for SSE/AVX/AVX512 shuffle operations.
0232              *
0233              * @tparam ElemT Element type to determine element size
0234              * @tparam U Type of the index values
0235              * @tparam Vs... Index values for the swizzle pattern
0236              *
0237              * @return true if any element accesses data from a different 128-bit lane
0238              *
0239              * Examples:
0240              * - is_cross_lane<float, 0, 1, 2, 3, 4, 5, 6, 7>() // no crossing (within 128-bit)
0241              * - is_cross_lane<float, 4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15>() // crosses
0242              */
0243             template <typename ElemT, typename U, U... Vs>
0244             XSIMD_INLINE constexpr bool is_cross_lane() noexcept
0245             {
0246                 return is_cross_lane_with_lane_size<16, ElemT, U, Vs...>();
0247             }
0248
0249             // Overload with std::size_t indices
0250             template <typename ElemT, std::size_t... Vs>
0251             XSIMD_INLINE constexpr bool is_cross_lane() noexcept
0252             {
0253                 return is_cross_lane<ElemT, std::size_t, Vs...>();
0254             }
0255
0256         } // namespace detail
0257     } // namespace kernel
0258 } // namespace xsimd
0259
0260 #endif // XSIMD_COMMON_SWIZZLE_HPP