EIC code displayed by LXR master/​include/​xsimd/​config/​xsimd_cpu_features_x86.hpp	Source navigation Diff markup Identifier search General search
	Version: master test Revert   Change

File indexing completed on 2026-11-07 10:37:15

 /***************************************************************************
  * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
  * Martin Renou                                                             *
  * Copyright (c) QuantStack                                                 *
  * Copyright (c) Serge Guelton                                              *
  *                                                                          *
  * Distributed under the terms of the BSD 3-Clause License.                 *
  *                                                                          *
  * The full license is in the file LICENSE, distributed with this software. *
  ****************************************************************************/
 
 #ifndef XSIMD_CPU_FEATURES_X86_HPP
 #define XSIMD_CPU_FEATURES_X86_HPP
 
 #include <array>
 #include <cassert>
 #include <cstdint>
 #include <cstring>
 #include <type_traits>
 #if XSIMD_CPP_VERSION >= 201703L
 #include <string_view>
 #endif
 
 #include "../utils/bits.hpp"
 #include "./xsimd_config.hpp"
 
 #if XSIMD_TARGET_X86 && defined(_MSC_VER)
 #include <intrin.h> // Contains the definition of __cpuidex
 #endif
 
 namespace xsimd
 {
     namespace detail
     {
         using x86_reg32_t = std::uint32_t;
 
         using cpuid_reg_t = std::array<x86_reg32_t, 4>;
 
         /**
          * CPU Identification (CPUID) instruction results.
          *
          * The CPUID instruction provides detailed information about the processor,
          * including supported instruction set extensions (SSE, AVX, AVX-512, etc.).
          * This function is well defined on all architectures but will return all zeros
          * on all non-x86 architectures.
          *
          * @param leaf The value inputted to the EAX register.
          * @param subleaf The value inputted to the ECX register.
          *
          * @see https://en.wikipedia.org/wiki/CPUID
          */
         inline cpuid_reg_t x86_cpuid(int leaf, int subleaf = 0) noexcept;
 
         inline x86_reg32_t x86_xcr0_low() noexcept;
 
         /** A strongly type bitset for a 32 bits register. */
         template <typename E>
         using x86_reg32_bitset = utils::uint_bitset<E, x86_reg32_t>;
 
         /** A wrapper to attach a register bitfield descriptor and its CPUID index. */
         template <typename E, x86_reg32_t I>
         struct x86_reg_id
         {
             static constexpr x86_reg32_t index = I;
             using bits = E;
 
             static_assert(index >= 0 && index < 4, "At most 4 register in CPUID");
         };
 
         /** Find the register id with index k. */
         template <x86_reg32_t K, typename... reg_ids>
         struct find_reg_k;
 
         /** Find the register id with index k (empty / nothing found case). */
         template <x86_reg32_t K, typename... reg_ids>
         struct find_reg_k
         {
             using type = x86_reg_id<void, 0>;
         };
 
         /** Find the register id with index k (recursive case). */
         template <x86_reg32_t K, typename reg_id_head, typename... reg_id_tail>
         struct find_reg_k<K, reg_id_head, reg_id_tail...>
         {
             using type = std::conditional_t<
                 reg_id_head::index == K,
                 reg_id_head,
                 typename find_reg_k<K, reg_id_tail...>::type>;
         };
 
         /**
          * A class with strongly typed bitfield for `CPUID` registers.
          *
          * The class stores a variable number of register (up to four) from the CPUID
          * output. This is a space optimization to avoid storing many zeros in the
          * final `x86_cpu_features`.
          * As a result, some of the type aliases `eax`, `ebx`, `ecx`, `edx` may be `void`.
          */
         template <x86_reg32_t leaf_num, x86_reg32_t subleaf_num, typename... reg_ids>
         class x86_cpuid_regs : private x86_reg32_bitset<typename reg_ids::bits>...
         {
         private:
             static_assert(sizeof...(reg_ids) <= 4, "At most 4 register in CPUID");
 
             /* Parse CPUINFO register value into individual bit components.*/
             constexpr explicit x86_cpuid_regs(const cpuid_reg_t& regs) noexcept
                 : x86_reg32_bitset<typename reg_ids::bits>(regs[reg_ids::index])...
             {
             }
 
         public:
             static constexpr x86_reg32_t leaf = leaf_num;
             static constexpr x86_reg32_t subleaf = subleaf_num;
 
             using eax = typename find_reg_k<0, reg_ids...>::type::bits;
             using ebx = typename find_reg_k<1, reg_ids...>::type::bits;
             using ecx = typename find_reg_k<2, reg_ids...>::type::bits;
             using edx = typename find_reg_k<3, reg_ids...>::type::bits;
 
             inline static x86_cpuid_regs read()
             {
                 return x86_cpuid_regs(detail::x86_cpuid(leaf, subleaf));
             }
 
             constexpr x86_cpuid_regs() noexcept = default;
 
             // TODO(C++17) compact version for which this was designed.
             // The else clause contains a very verbose port.
 #if 0
             using x86_reg32_bitset<typename reg_ids::bits>::all_bits_set...;
             using x86_reg32_bitset<typename reg_ids::bits>::get_range...;
 #else
 
         private:
             template <int N>
             struct m_empty_reg
             {
                 enum class type {};
             };
 
             using eax_or_empty = std::conditional_t<std::is_void<eax>::value, typename m_empty_reg<0>::type, eax>;
             using ebx_or_empty = std::conditional_t<std::is_void<ebx>::value, typename m_empty_reg<1>::type, ebx>;
             using ecx_or_empty = std::conditional_t<std::is_void<ecx>::value, typename m_empty_reg<2>::type, ecx>;
             using edx_or_empty = std::conditional_t<std::is_void<edx>::value, typename m_empty_reg<3>::type, edx>;
 
         public:
             template <eax_or_empty... bits, typename T = eax, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr bool all_bits_set() const noexcept
             {
                 return x86_reg32_bitset<eax>::template all_bits_set<bits...>();
             }
 
             template <eax_or_empty start, eax_or_empty end, typename T = eax, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr x86_reg32_t get_range() const noexcept
             {
                 return x86_reg32_bitset<eax>::template get_range<start, end>();
             }
 
             template <ebx_or_empty... bits, typename T = ebx, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr bool all_bits_set() const noexcept
             {
                 return x86_reg32_bitset<ebx>::template all_bits_set<bits...>();
             }
 
             template <ebx_or_empty start, ebx_or_empty end, typename T = ebx, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr x86_reg32_t get_range() const noexcept
             {
                 return x86_reg32_bitset<ebx>::template get_range<start, end>();
             }
 
             template <ecx_or_empty... bits, typename T = ecx, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr bool all_bits_set() const noexcept
             {
                 return x86_reg32_bitset<ecx>::template all_bits_set<bits...>();
             }
 
             template <ecx_or_empty start, ecx_or_empty end, typename T = ecx, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr x86_reg32_t get_range() const noexcept
             {
                 return x86_reg32_bitset<ecx>::template get_range<start, end>();
             }
 
             template <edx_or_empty... bits, typename T = edx, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr bool all_bits_set() const noexcept
             {
                 return x86_reg32_bitset<edx>::template all_bits_set<bits...>();
             }
 
             template <edx_or_empty start, edx_or_empty end, typename T = edx, std::enable_if_t<!std::is_void<T>::value, int> = 0>
             constexpr x86_reg32_t get_range() const noexcept
             {
                 return x86_reg32_bitset<edx>::template get_range<start, end>();
             }
 
 #endif // C++17
         };
 
         template <bool extended>
         struct x86_cpuid_highest_func
         {
         private:
             using x86_reg32_t = detail::x86_reg32_t;
             using manufacturer_str = std::array<char, 3 * sizeof(x86_reg32_t)>;
 
         public:
             static constexpr x86_reg32_t leaf = extended ? 0x80000000 : 0x0;
 
             inline static x86_cpuid_highest_func read()
             {
                 auto regs = detail::x86_cpuid(0);
                 x86_cpuid_highest_func out {};
                 // Highest function parameter in EAX
                 out.m_highest_leaf = regs[0];
 
                 // Manufacturer string in EBX, EDX, ECX (in that order)
                 char* manuf = out.m_manufacturer_id.data();
                 std::memcpy(manuf + 0 * sizeof(x86_reg32_t), &regs[1], sizeof(x86_reg32_t));
                 std::memcpy(manuf + 1 * sizeof(x86_reg32_t), &regs[3], sizeof(x86_reg32_t));
                 std::memcpy(manuf + 2 * sizeof(x86_reg32_t), &regs[2], sizeof(x86_reg32_t));
 
                 return out;
             }
 
             constexpr x86_cpuid_highest_func() noexcept = default;
 
             /**
              * Highest available leaf in CPUID non-extended range.
              *
              * This is the highest function parameter (EAX) that can be passed to CPUID.
              * This is valid in the specified range:
              *   - if `extended` is `false`, that is below `0x80000000`,
              *   - if `extended` is `true`, that is above `0x80000000`,
              */
             constexpr x86_reg32_t highest_leaf() const noexcept
             {
                 return m_highest_leaf;
             }
 
             /**
              * The manufacturer ID string in a static array.
              *
              * This raw character array is case specific and may contain both leading
              * and trailing whitespaces.
              * It cannot be assumed to be null terminated.
              * This is not implemented for all manufacturer when `extended` is `true`.
              */
             constexpr manufacturer_str manufacturer_id_raw() const noexcept
             {
                 return m_manufacturer_id;
             }
 
 #if XSIMD_CPP_VERSION >= 201703L
             constexpr std::string_view manufacturer_id() const noexcept
             {
                 return { m_manufacturer_id.data(), m_manufacturer_id.size() };
             }
 #endif
 
         private:
             manufacturer_str m_manufacturer_id {};
             x86_reg32_t m_highest_leaf {};
         };
     }
 
     /**
      * Highest CPUID Function Parameter and Manufacturer ID (EAX=0).
      *
      * Returns the highest leaf value supported by CPUID in the standard range
      * (below 0x80000000), and the processor manufacturer ID string.
      *
      * @see https://en.wikipedia.org/wiki/CPUID
      */
     using x86_cpuid_leaf0 = detail::x86_cpuid_highest_func<false>;
 
     /**
      * Known processor manufacturer ID strings returned by CPUID leaf 0.
      *
      * The 12-byte manufacturer ID is stored in EBX, EDX, ECX (in that order).
      * Some strings are shared across physical CPUs, emulators, and virtual machines.
      * Obscure, defunct, and soft-core CPUs are not represented; they map to `unknown`.
      *
      * @see https://en.wikipedia.org/wiki/CPUID
      */
     enum class x86_manufacturer
     {
         /**
          * AMD ("AuthenticAMD", "AMD ISBETTER").
          *
          * "AMD ISBETTER" was used by early K5 engineering samples.
          */
         amd,
         /**
          * Intel ("GenuineIntel", "GenuineIotel").
          *
          * "GenuineIotel" is a rare typo variant seen on some chips.
          */
         intel,
         /**
          * VIA / Centaur ("CentaurHauls", "VIA VIA VIA ").
          *
          * Centaur Technology was acquired by VIA in 1999;
          * older chips report "CentaurHauls", newer ones "VIA VIA VIA ".
          */
         via,
         /** Zhaoxin ("  Shanghai  "). */
         zhaoxin,
         /** Hygon ("HygonGenuine"). */
         hygon,
         /**
          * Transmeta ("TransmetaCPU", "GenuineTMx86").
          *
          * Two different ID strings were used across product lines.
          */
         transmeta,
         /** MCST Elbrus ("E2K MACHINE "). */
         elbrus,
         /** Microsoft Virtual PC / x86-to-ARM ("Virtual CPU "). */
         microsoft_vpc,
         /** Unrecognized manufacturer ID string. */
         unknown,
     };
 
     /**
      * Parse a 12-byte CPUID manufacturer ID into an @ref x86_manufacturer value.
      *
      * The input is the raw character array returned by @ref x86_cpuid_leaf0::manufacturer_id_raw.
      * Unrecognized strings map to @ref x86_manufacturer::unknown.
      */
     inline x86_manufacturer x86_parse_manufacturer(const std::array<char, 12>& id) noexcept
     {
         auto eq = [&id](const char(&s)[13]) noexcept -> bool
         {
             return std::memcmp(id.data(), s, 12) == 0;
         };
         if (eq("GenuineIntel") || eq("GenuineIotel"))
             return x86_manufacturer::intel;
         if (eq("AuthenticAMD") || eq("AMD ISBETTER"))
             return x86_manufacturer::amd;
         if (eq("CentaurHauls") || eq("VIA VIA VIA "))
             return x86_manufacturer::via;
         if (eq("  Shanghai  "))
             return x86_manufacturer::zhaoxin;
         if (eq("HygonGenuine"))
             return x86_manufacturer::hygon;
         if (eq("TransmetaCPU") || eq("GenuineTMx86"))
             return x86_manufacturer::transmeta;
         if (eq("E2K MACHINE "))
             return x86_manufacturer::elbrus;
         if (eq("Virtual CPU "))
             return x86_manufacturer::microsoft_vpc;
         return x86_manufacturer::unknown;
     };
 
     /** Return a string representation of an @ref x86_manufacturer value. */
     constexpr const char* x86_manufacturer_name(x86_manufacturer m) noexcept
     {
         switch (m)
         {
         case x86_manufacturer::intel:
             return "intel";
         case x86_manufacturer::amd:
             return "amd";
         case x86_manufacturer::via:
             return "via";
         case x86_manufacturer::zhaoxin:
             return "zhaoxin";
         case x86_manufacturer::hygon:
             return "hygon";
         case x86_manufacturer::transmeta:
             return "transmeta";
         case x86_manufacturer::elbrus:
             return "elbrus";
         case x86_manufacturer::microsoft_vpc:
             return "microsoft_vpc";
         case x86_manufacturer::unknown:
             return "unknown";
         }
         return "invalid";
     }
 
     struct x86_cpuid_leaf1_traits
     {
         static constexpr detail::x86_reg32_t leaf = 1;
         static constexpr detail::x86_reg32_t subleaf = 0;
 
         enum class eax
         {
             /* Stepping ID bit range. */
             stepping_start = 0,
             stepping_end = 4,
             /* Model bit range. */
             model_start = 4,
             model_end = 8,
             /* Family ID bit range. */
             family_id_start = 8,
             family_id_end = 12,
             /* Extended Model ID bit range. */
             ext_model_start = 16,
             ext_model_end = 20,
             /* Extended Family ID bit range. */
             ext_family_start = 20,
             ext_family_end = 28,
         };
         enum class ecx
         {
             /* Streaming SIMD Extensions 3. */
             sse3 = 0,
             /* Supplemental Streaming SIMD Extensions 3. */
             ssse3 = 9,
             /* Fused multiply-add with 3 operands (FMA3). */
             fma3 = 12,
             /* Streaming SIMD Extensions 4.1. */
             sse4_1 = 19,
             /* Streaming SIMD Extensions 4.2. */
             sse4_2 = 20,
             /* Population count instruction (POPCNT). */
             popcnt = 23,
             /* Advanced Encryption Standard instruction set. */
             aes_ni = 25,
             /* OS has enabled XSAVE/XRSTOR for extended processor state management. */
             osxsave = 27,
             /* Advanced Vector Extensions (256-bit SIMD). */
             avx = 28,
             /* Half to single floating point conversion. */
             f16c = 29,
             /* On-chip random number generator. */
             rdrnd = 30,
         };
         enum class edx
         {
             /* Streaming SIMD Extensions 2. */
             sse2 = 26,
         };
 
         using regs_t = detail::x86_cpuid_regs<leaf, subleaf,
                                               detail::x86_reg_id<eax, 0>,
                                               detail::x86_reg_id<ecx, 2>,
                                               detail::x86_reg_id<edx, 3>>;
     };
 
     /**
      * Processor Info and Feature Bits.
      *
      * Utility class that can read and parse the registers for the first leaf level
      * of the CPUID instruction.
      * This is well defined on all architectures but will return all false on all
      * non-x86 architectures.
      *
      * @see https://en.wikipedia.org/wiki/CPUID
      */
     using x86_cpuid_leaf1 = typename x86_cpuid_leaf1_traits::regs_t;
 
     struct x86_cpuid_leaf7_traits
     {
         static constexpr detail::x86_reg32_t leaf = 7;
         static constexpr detail::x86_reg32_t subleaf = 0;
 
         enum class eax
         {
             /* Start bit for the encoding of the highest subleaf available. */
             highest_subleaf_start = 0,
             /* End bit for the encoding of the highest subleaf available. */
             highest_subleaf_end = 32,
         };
         enum class ebx
         {
             /* Bit Manipulation Instruction Set 1. */
             bmi1 = 3,
             /* Advanced Vector Extensions 2 (integer 256-bit SIMD). */
             avx2 = 5,
             /* Bit Manipulation Instruction Set 2. */
             bmi2 = 8,
             /* AVX-512 Foundation instructions. */
             avx512f = 16,
             /* AVX-512 Doubleword and Quadword instructions. */
             avx512dq = 17,
             /* Low-level access to the entropy-generating hardware. */
             rdseed = 18,
             /* Intel arbitrary precision add carry. */
             adx = 19,
             /* AVX-512 Integer Fused Multiply-Add instructions. */
             avx512ifma = 21,
             /* AVX-512 Prefetch instructions. */
             avx512pf = 26,
             /* AVX-512 Exponential and Reciprocal instructions. */
             avx512er = 27,
             /* AVX-512 Conflict Detection instructions. */
             avx512cd = 28,
             /* Sha-1 and Sha-256 extension. */
             sha = 29,
             /* AVX-512 Byte and Word instructions. */
             avx512bw = 30,
             /* AVX-512 Vector Length Extensions for xmm and ymm registers. */
             avx512vl = 31,
         };
         enum class ecx
         {
             /* AVX-512 Vector Bit Manipulation instructions. */
             avx512vbmi = 1,
             /* AVX-512 Vector Bit Manipulation instructions 2. */
             avx512vbmi2 = 6,
             /* Galois Field instructions. */
             gfni = 8,
             /* Vector Advanced Encryption Standard instructions. */
             vaes = 9,
             /* Carry-less multiplication quadword instruction. */
             vpclmulqdq = 10,
             /* AVX-512 Vector Neural Network instructions. */
             avx512vnni_bw = 11,
             /* AVX-512 bit algorithm instructions (BITALG). */
             avx512_bitalg = 12,
             /* AVX-512 vector population count for doubleword and quadword. */
             avx512_vpopcntdq = 14,
         };
         enum class edx
         {
             /* AVX-512 4-register neural network instructions (word variable precision). */
             avx512_4vnniw = 2,
             /* AVX-512 4-register multiply-accumulate single precision. */
             avx512_4fmaps = 3,
             /* AVX-512 intersect pairs of packed doubleword/quadword integers. */
             avx512_vp2intersect = 8,
             /* AVX-512 16-bit floating-point instructions. */
             avx512_fp16 = 23,
 
         };
 
         using regs_t = detail::x86_cpuid_regs<leaf, subleaf,
                                               detail::x86_reg_id<eax, 0>,
                                               detail::x86_reg_id<ebx, 1>,
                                               detail::x86_reg_id<ecx, 2>,
                                               detail::x86_reg_id<edx, 3>>;
     };
 
     /**
      * Extended Feature Bits (EAX=7, ECX=0).
      *
      * Utility class that can read and parse the registers for the extended
      * feature bits leaf of the CPUID instruction.
      * This is well defined on all architectures but will return all false on all
      * non-x86 architectures.
      *
      * @see https://en.wikipedia.org/wiki/CPUID
      */
     using x86_cpuid_leaf7 = typename x86_cpuid_leaf7_traits::regs_t;
 
     struct x86_cpuid_leaf7sub1_traits
     {
         static constexpr detail::x86_reg32_t leaf = 7;
         static constexpr detail::x86_reg32_t subleaf = 1;
 
         enum class eax
         {
             /* AVX (VEX-encoded) Vector Neural Network instructions. */
             avxvnni = 4,
             /* AVX-512 BFloat16 instructions. */
             avx512_bf16 = 5,
         };
 
         using regs_t = detail::x86_cpuid_regs<leaf, subleaf,
                                               detail::x86_reg_id<eax, 0>>;
     };
 
     /**
      * Extended Feature Bits (EAX=7, ECX=1).
      *
      * Utility class that can read and parse the registers for the extended
      * feature bits, subleaf 1, of the CPUID instruction.
      * This is well defined on all architectures but will return all false on all
      * non-x86 architectures.
      *
      * @see https://en.wikipedia.org/wiki/CPUID
      */
     using x86_cpuid_leaf7sub1 = typename x86_cpuid_leaf7sub1_traits::regs_t;
 
     /**
      * Highest Extended CPUID Function Parameter (EAX=0x80000000).
      *
      * Returns the highest leaf value supported by CPUID in the extended range
      * (at or above 0x80000000), and the processor manufacturer ID string.
      *
      * @see https://en.wikipedia.org/wiki/CPUID
      */
     using x86_cpuid_leaf80000000 = detail::x86_cpuid_highest_func<true>;
 
     struct x86_cpuid_leaf80000001_traits
     {
         static constexpr detail::x86_reg32_t leaf = 0x80000001;
         static constexpr detail::x86_reg32_t subleaf = 0;
 
         enum class ecx
         {
             /* AMD Fused multiply-add with 4 operands (FMA4). */
             fma4 = 16,
         };
 
         using regs_t = detail::x86_cpuid_regs<leaf, subleaf,
                                               detail::x86_reg_id<ecx, 2>>;
     };
 
     /**
      * Extended Processor Info and Feature Bits.
      *
      * Utility class that can read and parse the registers for the extended
      * processor info leaf of the CPUID instruction.
      * This is well defined on all architectures but will return all false on all
      * non-x86 architectures.
      *
      * @see https://en.wikipedia.org/wiki/CPUID
      */
     using x86_cpuid_leaf80000001 = typename x86_cpuid_leaf80000001_traits::regs_t;
 
     /*
      * Extended Control Register 0 (XCR0).
      *
      * Operating systems can explicitly disable the usage of instruction set (such
      * as SSE or AVX extensions) by setting an appropriate flag in XCR0 register.
      * This utility parses such bit values.
      *
      * @see https://docs.kernel.org/admin-guide/hw-vuln/gather_data_sampling.html
      */
     class x86_xcr0
     {
     public:
         enum class xcr0
         {
             /** x87 FPU/MMX support (must be 1). */
             x87 = 0,
             /** XSAVE support for MXCSR and XMM registers. */
             sse = 1,
             /** AVX enabled and XSAVE support for upper halves of YMM registers. */
             avx = 2,
             /** MPX enabled and XSAVE support for BND0-BND3 registers. */
             bndreg = 3,
             /** MPX enabled and XSAVE support for BNDCFGU and BNDSTATUS registers. */
             bndcsr = 4,
             /** AVX-512 enabled and XSAVE support for opmask registers k0-k7. */
             opmask = 5,
             /** AVX-512 enabled and XSAVE support for upper halves of lower ZMM registers. */
             zmm_hi256 = 6,
             /** AVX-512 enabled and XSAVE support for upper ZMM registers. */
             hi16_zmm = 7,
             /** Saving/restoring Intel Processor Trace state via XSAVE enabled.*/
             processor_trace = 8,
             /** XSAVE support for PKRU register. */
             pkru = 9,
         };
 
         /**
          * Create a default value with only SSE enabled.
          *
          * AVX and AVX512 strictly require OSXSAVE to be enabled by the OS.
          * If OSXSAVE is disabled (e.g., via bcdedit /set xsavedisable 1), AVX state won't
          * be preserved across context switches, so AVX cannot be used.
          * SSE is therefore the only value safe to assume.
          */
         constexpr static x86_xcr0 safe_default() noexcept
         {
             x86_reg32_t low = {};
             low = utils::make_bit_mask(static_cast<x86_reg32_t>(xcr0::sse));
             return x86_xcr0(low);
         }
 
         /**
          * Read the XCR0 register from the CPU if on the correct architecture.
          *
          * This is only safe to call if bit 18 of CR4.OSXSAVE has been set.
          *
          * @see cpu_id::osxsave
          */
         inline static x86_xcr0 read()
         {
             assert(x86_cpuid_leaf1::read().all_bits_set<x86_cpuid_leaf1::ecx::osxsave>());
             return x86_xcr0(detail::x86_xcr0_low());
         }
 
         template <xcr0... bits>
         constexpr bool all_bits_set() const noexcept
         {
             return m_low.all_bits_set<bits...>();
         }
 
         /** Create a value which return false to everything. */
         constexpr x86_xcr0() noexcept = default;
 
     private:
         using x86_reg32_t = detail::x86_reg32_t;
 
         using xcr0_reg_t = detail::x86_reg32_bitset<xcr0>;
 
         /** Parse a XCR0 value into individual components. */
         constexpr explicit x86_xcr0(x86_reg32_t low) noexcept
             : m_low(low)
         {
         }
 
         xcr0_reg_t m_low {};
     };
 
     /**
      * Orchestrator for `CPUID` calls.
      *
      * This class orchestrate `CPUID` and `XCR0` calls so that they are made in the appropriate
      * order. It also implements lazy calling and cache mechanism around those calls.
      * Works on all platforms, and return all zeros on non `x86` platforms.
      */
     class x86_cpu_features_backend_cpuid
     {
     public:
         x86_cpu_features_backend_cpuid() noexcept = default;
 
         inline x86_xcr0 const& xcr0() const noexcept;
         inline x86_cpuid_leaf0 const& leaf0() const;
         inline x86_cpuid_leaf80000000 const& leaf80000000() const;
         inline x86_cpuid_leaf1 const& leaf1() const;
         inline x86_cpuid_leaf7 const& leaf7() const;
         inline x86_cpuid_leaf7sub1 const& leaf7sub1() const;
         inline x86_cpuid_leaf80000001 const& leaf80000001() const;
 
     private:
         enum class status
         {
             leaf0_valid = 0,
             leaf1_valid = 1,
             leaf7_valid = 2,
             leaf7sub1_valid = 3,
             leaf80000000_valid = 4,
             leaf80000001_valid = 5,
             xcr0_valid = 6,
         };
 
         using status_bitset = utils::uint_bitset<status, std::uint32_t>;
 
         mutable x86_cpuid_leaf0 m_leaf0 {};
         mutable x86_cpuid_leaf1 m_leaf1 {};
         mutable x86_cpuid_leaf7 m_leaf7 {};
         mutable x86_cpuid_leaf7sub1 m_leaf7sub1 {};
         mutable x86_cpuid_leaf80000000 m_leaf80000000 {};
         mutable x86_cpuid_leaf80000001 m_leaf80000001 {};
         mutable x86_xcr0 m_xcr0 {};
         mutable status_bitset m_status {};
 
         inline bool osxsave() const noexcept;
 
         /**
          * Internal utility to lazily read and cache a CPUID leaf.
          *
          * @tparam status_id The status bit tracking whether this leaf has been read and cached.
          * @tparam L The CPUID leaf type (e.g. x86_cpuid_leaf1, x86_cpuid_leaf7).
          * @param leaf_cache A non-const reference to the class member that stores the leaf
          *        value. It must be non-const because this function may write to it on first
          *        call. It is passed explicitly (rather than accessed via `this`) to allow
          *        factoring the caching logic across different leaf members.
          * @return A const reference to `leaf_cache`. The non-const input / const-ref output
          *         asymmetry is intentional: callers must not modify the cached value, but
          *         this function needs write access to populate it.
          *
          * On first call, checks whether the leaf number is within the range advertised as
          * supported by CPUID (via leaf 0 for the standard range, leaf 0x80000000 for the
          * extended range). If supported, reads the leaf from the CPU; otherwise leaves
          * `leaf_cache` at its zero-initialized default (all feature bits false). Either
          * way, `status_id` is set so subsequent calls return immediately.
          */
         template <status status_id, typename L>
         inline auto const& safe_read_leaf(L& leaf_cache) const;
     };
 
     /**
      * No-Op orchestrator for `CPUID` calls
      *
      * This does nothing and return zero-constructed objects on all calls.
      * This is meant as an optimization on non `x86` platforms as the
      * `x86_cpu_features_backend_cpuid` can be slightly large (hundred of bytes).
      */
     class x86_cpu_features_backend_noop
     {
     public:
         constexpr x86_xcr0 xcr0() const noexcept { return {}; }
         constexpr x86_cpuid_leaf0 leaf0() const { return {}; }
         constexpr x86_cpuid_leaf80000000 leaf80000000() const { return {}; }
         constexpr x86_cpuid_leaf1 leaf1() const { return {}; }
         constexpr x86_cpuid_leaf7 leaf7() const { return {}; }
         constexpr x86_cpuid_leaf7sub1 leaf7sub1() const { return {}; }
         constexpr x86_cpuid_leaf80000001 leaf80000001() const { return {}; }
     };
 
 #if XSIMD_TARGET_X86
     using x86_cpu_features_backend_default = x86_cpu_features_backend_cpuid;
 #else
     using x86_cpu_features_backend_default = x86_cpu_features_backend_noop;
 #endif
 
     /**
      * An opiniated CPU feature detection utility for x86.
      *
      * These are high level features that combine multiple registers reads in sequence.
      * Instead of looking directly at raw CPUID results, this utility also checks that
      * permissions (e.g. OSXSAVE) are enabled, and otherwise return conservative defaults.
      *
      * This is well defined on all architectures. It will always return false on
      * non-x86 architectures.
      */
     class x86_cpu_features : private x86_cpu_features_backend_default
     {
     public:
         x86_cpu_features() noexcept = default;
 
         inline bool sse_enabled() const noexcept
         {
             return xcr0().all_bits_set<x86_xcr0::xcr0::sse>();
         }
 
         inline bool avx_enabled() const noexcept
         {
             // Check both SSE and AVX bits even though AVX must imply SSE
             return xcr0().all_bits_set<x86_xcr0::xcr0::sse, x86_xcr0::xcr0::avx>();
         }
 
         inline bool avx512_enabled() const noexcept
         {
             // Check all SSE, AVX, optmask, and AVX512 bits even though AVX512 must
             // imply AVX, SSE, and masked operations.
             return xcr0().all_bits_set<x86_xcr0::xcr0::sse, x86_xcr0::xcr0::avx, x86_xcr0::xcr0::opmask, x86_xcr0::xcr0::zmm_hi256>();
         }
 
         /**
          * The manufacturer ID string in a static array.
          *
          * This raw character array is case specific and may contain both leading
          * and trailing whitespaces.
          * It cannot be assumed to be null terminated.
          */
         inline auto manufacturer_id_raw() const noexcept
         {
             return leaf0().manufacturer_id_raw();
         }
 
 #if XSIMD_CPP_VERSION >= 201703L
         inline std::string_view manufacturer_id() const noexcept
         {
             return leaf0().manufacturer_id();
         }
 #endif
 
         /** The manufacturer ID string parsed into known common vendors. */
         inline x86_manufacturer known_manufacturer() const noexcept
         {
             return x86_parse_manufacturer(manufacturer_id_raw());
         }
 
         /**
          * Indicates whether the OS has enabled extended state management.
          *
          * When true, the OS has set bit 18 (OSXSAVE) in the CR4 control register,
          * enabling the XGETBV/XSETBV instructions to access XCR0 and support
          * processor extended state management using XSAVE/XRSTOR.
          *
          * This value is read from CPUID leaf 0x1, ECX bit 27, which reflects
          * the state of CR4.OSXSAVE.
          */
         inline bool osxsave() const noexcept { return leaf1().all_bits_set<x86_cpuid_leaf1::ecx::osxsave>(); }
 
         /**
          * Effective processor family.
          *
          * Per wikipedia CPUID page:
          * > The actual processor family is derived from the Family ID and Extended Family ID fields.
          * > If the Family ID field is equal to 15, the family is equal to the sum of the Extended
          * > Family ID and the Family ID fields. Otherwise, the family is equal to the value of the
          * > Family ID field.
          *
          * @see https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits
          */
         inline detail::x86_reg32_t cpu_family() const noexcept
         {
             using eax = x86_cpuid_leaf1::eax;
             auto family_id = leaf1().get_range<eax::family_id_start, eax::family_id_end>();
             auto ext_family_id = leaf1().get_range<eax::ext_family_start, eax::ext_family_end>();
             return family_id + (family_id == 15 ? ext_family_id : 0);
         }
 
         /**
          * Effective processor model.
          *
          * Per wikipedia CPUID page:
          * > The actual processor model is derived from the Model, Extended Model ID and Family ID
          * > fields. If the Family ID field is either 6 or 15, the model is equal to the sum of the
          * > Extended Model ID field shifted left by 4 bits and the Model field.
          * > Otherwise, the model is equal to the value of the Model field.
          *
          * @see https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits
          */
         inline detail::x86_reg32_t cpu_model() const noexcept
         {
             using eax = x86_cpuid_leaf1::eax;
             auto model = leaf1().get_range<eax::model_start, eax::model_end>();
             auto ext_model = leaf1().get_range<eax::ext_model_start, eax::ext_model_end>();
             auto family_id = leaf1().get_range<eax::family_id_start, eax::family_id_end>();
             return (family_id == 15 || family_id == 6) ? ((ext_model << 4) + model) : model;
         }
 
         inline bool sse2() const noexcept { return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::edx::sse2>(); }
 
         inline bool sse3() const noexcept { return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::sse3>(); }
 
         inline bool ssse3() const noexcept { return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::ssse3>(); }
 
         inline bool sse4_1() const noexcept { return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::sse4_1>(); }
 
         inline bool sse4_2() const noexcept { return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::sse4_2>(); }
 
         inline bool popcnt() const noexcept { return leaf1().all_bits_set<x86_cpuid_leaf1::ecx::popcnt>(); }
 
         inline bool fma3() const noexcept { return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::fma3>(); }
 
         inline bool avx() const noexcept { return avx_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::avx>(); }
 
         inline bool avx_128() const noexcept
         {
             // Avx 128 bit instructions use the same xmm registers from SSE so checking if those
             // are enabled is sufficient.
             return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::avx>();
         }
 
         inline bool aes_ni() const noexcept { return sse_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::aes_ni>(); }
 
         inline bool f16c() const noexcept { return avx_enabled() && leaf1().all_bits_set<x86_cpuid_leaf1::ecx::f16c>(); }
 
         inline bool rdrnd() const noexcept { return leaf1().all_bits_set<x86_cpuid_leaf1::ecx::rdrnd>(); }
 
         inline bool bmi1() const noexcept { return leaf7().all_bits_set<x86_cpuid_leaf7::ebx::bmi1>(); }
 
         inline bool avx2() const noexcept { return avx_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx2>(); }
 
         inline bool avx2_128() const noexcept
         {
             // Avx 128 bit instructions use the same xmm registers from SSE so checking if those
             // are enabled is sufficient.
             return sse_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx2>();
         }
 
         inline bool bmi2() const noexcept { return leaf7().all_bits_set<x86_cpuid_leaf7::ebx::bmi2>(); }
 
         /**
          * BMI2 support with efficient PEXT and PDEP instructions.
          *
          * > AMD processors before Zen 3 that implement PDEP and PEXT do so in microcode, with a
          * > latency of 18 cycles rather than (Zen 3) 3 cycles. As a result it is often faster
          * > to use other instructions on these processors.
          *
          * @see https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set
          */
         inline bool efficient_bmi2() const noexcept
         {
             if (known_manufacturer() == x86_manufacturer::amd)
             {
                 // Zen 3 and Zen 4 report family 0x19; Zen 5 reports family 0x1A.
                 // Earlier AMD microarchitectures (Zen / Zen+ / Zen 2) report family 0x17.
                 return bmi2() && (cpu_family() >= 0x19);
             }
             return bmi2();
         }
 
         inline bool avx512f() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512f>(); }
 
         inline bool avx512dq() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512dq>(); }
 
         inline bool rdseed() const noexcept { return leaf7().all_bits_set<x86_cpuid_leaf7::ebx::rdseed>(); }
 
         inline bool adx() const noexcept { return leaf7().all_bits_set<x86_cpuid_leaf7::ebx::adx>(); }
 
         inline bool avx512ifma() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512ifma>(); }
 
         inline bool avx512pf() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512pf>(); }
 
         inline bool avx512er() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512er>(); }
 
         inline bool avx512cd() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512cd>(); }
 
         inline bool sha() const noexcept { return leaf7().all_bits_set<x86_cpuid_leaf7::ebx::sha>(); }
 
         inline bool avx512bw() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512bw>(); }
 
         inline bool avx512vl() const noexcept
         {
             return xcr0().all_bits_set<x86_xcr0::xcr0::opmask>()
                 && leaf7().all_bits_set<x86_cpuid_leaf7::ebx::avx512f, x86_cpuid_leaf7::ebx::avx512vl>();
         }
 
         inline bool avx512vl_128() const noexcept
         {
             // AVX512 128 bit instructions use the same xmm registers from SSE so checking if those
             // are enabled is sufficient.
             return sse_enabled() && avx512vl();
         }
 
         inline bool avx512vl_256() const noexcept
         {
             // AVX512 256 bit instructions use the same ymm registers from AVX so checking if those
             // are enabled is sufficient.
             return avx_enabled() && avx512vl();
         }
 
         inline bool avx512vbmi() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::avx512vbmi>(); }
 
         inline bool avx512vbmi2() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::avx512vbmi2>(); }
 
         inline bool gfni() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::gfni>(); }
 
         inline bool vaes() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::vaes>(); }
 
         inline bool vpclmulqdq() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::vpclmulqdq>(); }
 
         inline bool avx512vnni_bw() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::avx512vnni_bw>(); }
 
         inline bool avx512_bitalg() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::avx512_bitalg>(); }
 
         inline bool avx512_vpopcntdq() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::ecx::avx512_vpopcntdq>(); }
 
         inline bool avx512_4vnniw() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::edx::avx512_4vnniw>(); }
 
         inline bool avx512_4fmaps() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::edx::avx512_4fmaps>(); }
 
         inline bool avx512_vp2intersect() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::edx::avx512_vp2intersect>(); }
 
         inline bool avx512_fp16() const noexcept { return avx512_enabled() && leaf7().all_bits_set<x86_cpuid_leaf7::edx::avx512_fp16>(); }
 
         inline bool avxvnni() const noexcept { return avx_enabled() && leaf7sub1().all_bits_set<x86_cpuid_leaf7sub1::eax::avxvnni>(); }
 
         inline bool avx512_bf16() const noexcept { return avx512_enabled() && leaf7sub1().all_bits_set<x86_cpuid_leaf7sub1::eax::avx512_bf16>(); }
 
         inline bool fma4() const noexcept { return avx_enabled() && leaf80000001().all_bits_set<x86_cpuid_leaf80000001::ecx::fma4>(); }
     };
 
     /********************
      *  Implementation  *
      ********************/
 
     template <x86_cpu_features_backend_cpuid::status status_id, typename L>
     inline auto const& x86_cpu_features_backend_cpuid::safe_read_leaf(L& leaf_cache) const
     {
         // Check if already initialized
         if (m_status.bit_is_set<status_id>())
         {
             return leaf_cache;
         }
 
         // Limit where we need to check leaf0 or leaf 80000000.
         constexpr auto extended_threshold = x86_cpuid_leaf80000000::leaf;
 
         // Check if it is safe to call CPUID with this value.
         // First we identify if the leaf is in the regular or extended range.
         // TODO(C++17): if constexpr
         if (L::leaf < extended_threshold)
         {
             // Check leaf0 in regular range
             if (L::leaf <= leaf0().highest_leaf())
             {
                 leaf_cache = L::read();
             }
         }
         else
         {
             // Check leaf80000000 in extended range
             if (L::leaf <= leaf80000000().highest_leaf())
             {
                 leaf_cache = L::read();
             }
         }
 
         // Mark as valid in all cases, including if it was not read.
         // In this case it will be filled with zeros (all false).
         m_status.set_bit<status_id>();
         return leaf_cache;
     }
 
     inline x86_xcr0 const& x86_cpu_features_backend_cpuid::xcr0() const noexcept
     {
         if (!m_status.bit_is_set<status::xcr0_valid>())
         {
             m_xcr0 = osxsave() ? x86_xcr0::read() : x86_xcr0::safe_default();
             m_status.set_bit<status::xcr0_valid>();
         }
         return m_xcr0;
     }
 
     inline x86_cpuid_leaf0 const& x86_cpu_features_backend_cpuid::leaf0() const
     {
         if (!m_status.bit_is_set<status::leaf0_valid>())
         {
             m_leaf0 = x86_cpuid_leaf0::read();
             m_status.set_bit<status::leaf0_valid>();
         }
         return m_leaf0;
     }
 
     inline x86_cpuid_leaf80000000 const& x86_cpu_features_backend_cpuid::leaf80000000() const
     {
         if (!m_status.bit_is_set<status::leaf80000000_valid>())
         {
             m_leaf80000000 = x86_cpuid_leaf80000000::read();
             m_status.set_bit<status::leaf80000000_valid>();
         }
         return m_leaf80000000;
     }
 
     inline x86_cpuid_leaf1 const& x86_cpu_features_backend_cpuid::leaf1() const
     {
         return safe_read_leaf<status::leaf1_valid>(m_leaf1);
     }
 
     inline x86_cpuid_leaf7 const& x86_cpu_features_backend_cpuid::leaf7() const
     {
         return safe_read_leaf<status::leaf7_valid>(m_leaf7);
     }
 
     inline x86_cpuid_leaf7sub1 const& x86_cpu_features_backend_cpuid::leaf7sub1() const
     {
         // Check if already initialized
         if (m_status.bit_is_set<status::leaf7sub1_valid>())
         {
             return m_leaf7sub1;
         }
 
         // Check if safe to call CPUID with this value as subleaf.
         constexpr auto start = x86_cpuid_leaf7::eax::highest_subleaf_start;
         constexpr auto end = x86_cpuid_leaf7::eax::highest_subleaf_end;
         const auto highest_subleaf7 = leaf7().get_range<start, end>();
         if (x86_cpuid_leaf7sub1::subleaf <= highest_subleaf7)
         {
             m_leaf7sub1 = x86_cpuid_leaf7sub1::read();
         }
 
         // Mark as valid in all cases, including if it was not read.
         // In this case it will be filled with zeros (all false).
         m_status.set_bit<status::leaf7sub1_valid>();
         return m_leaf7sub1;
     }
 
     inline x86_cpuid_leaf80000001 const& x86_cpu_features_backend_cpuid::leaf80000001() const
     {
         return safe_read_leaf<status::leaf80000001_valid>(m_leaf80000001);
     }
 
     inline bool x86_cpu_features_backend_cpuid::osxsave() const noexcept
     {
         return leaf1().all_bits_set<x86_cpuid_leaf1::ecx::osxsave>();
     }
 
     namespace detail
     {
 #if XSIMD_TARGET_X86
 
         inline cpuid_reg_t x86_cpuid(int leaf, int subleaf) noexcept
         {
             cpuid_reg_t reg = {};
 #if defined(_MSC_VER)
             int buf[4];
             __cpuidex(buf, leaf, subleaf);
             std::memcpy(reg.data(), buf, sizeof(buf));
 
 // Intel compiler has long had support for `__cpuid`, but only recently for `__cpuidex`.
 // Modern Clang and GCC also now support `__cpuidex`.
 // It was decided to keep the inline ASM version for maximum compatibility, as the difference
 // in ASM is negligible compared to the cost of CPUID.
 // https://github.com/xtensor-stack/xsimd/pull/1278
 #elif XSIMD_WITH_INLINE_ASM
 
 #if defined(__i386__) && defined(__PIC__)
             // %ebx may be the PIC register
             __asm__("xchg{l}\t{%%}ebx, %1\n\t"
                     "cpuid\n\t"
                     "xchg{l}\t{%%}ebx, %1\n\t"
                     : "=a"(reg[0]), "=r"(reg[1]), "=c"(reg[2]), "=d"(reg[3])
                     : "0"(leaf), "2"(subleaf));
 
 #else
             __asm__("cpuid\n\t"
                     : "=a"(reg[0]), "=b"(reg[1]), "=c"(reg[2]), "=d"(reg[3])
                     : "0"(leaf), "2"(subleaf));
 #endif
 #endif
             return reg;
         }
 
         inline x86_reg32_t x86_xcr0_low() noexcept
         {
 #if defined(_MSC_VER)
 #if _MSC_VER >= 1400
             return static_cast<x86_reg32_t>(_xgetbv(0));
 #else
 #error "_MSC_VER < 1400 is not supported"
 #endif
 
 #elif XSIMD_WITH_INLINE_ASM
             x86_reg32_t xcr0 = {};
             __asm__(
                 "xorl %%ecx, %%ecx\n"
                 "xgetbv\n"
                 : "=a"(xcr0)
                 :
 #if defined(__i386__)
                 : "ecx", "edx"
 #else
                 : "rcx", "rdx"
 #endif
             );
             return xcr0;
 #endif
         }
 
 #else // XSIMD_TARGET_X86
 
         inline cpuid_reg_t x86_cpuid(int /* leaf */, int /* subleaf */) noexcept
         {
             return {}; // All bits to zero
         }
 
         inline x86_reg32_t x86_xcr0_low() noexcept
         {
             return {}; // All bits to zero
         }
 
 #endif // XSIMD_TARGET_X86
     }
 }
 #endif

This page was automatically generated by the 2.3.7 LXR engine. The LXR team