Back to home page

EIC code displayed by LXR

 
 

    


File indexing completed on 2025-01-29 10:22:08

0001 /*
0002  * xxHash - Extremely Fast Hash algorithm
0003  * Header File
0004  * Copyright (C) 2012-2021 Yann Collet
0005  *
0006  * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
0007  *
0008  * Redistribution and use in source and binary forms, with or without
0009  * modification, are permitted provided that the following conditions are
0010  * met:
0011  *
0012  *    * Redistributions of source code must retain the above copyright
0013  *      notice, this list of conditions and the following disclaimer.
0014  *    * Redistributions in binary form must reproduce the above
0015  *      copyright notice, this list of conditions and the following disclaimer
0016  *      in the documentation and/or other materials provided with the
0017  *      distribution.
0018  *
0019  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
0020  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
0021  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
0022  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
0023  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
0024  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
0025  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
0026  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
0027  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
0028  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
0029  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
0030  *
0031  * You can contact the author at:
0032  *   - xxHash homepage: https://www.xxhash.com
0033  *   - xxHash source repository: https://github.com/Cyan4973/xxHash
0034  */
0035 
0036 /*!
0037  * @mainpage xxHash
0038  *
0039  * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed
0040  * limits.
0041  *
0042  * It is proposed in four flavors, in three families:
0043  * 1. @ref XXH32_family
0044  *   - Classic 32-bit hash function. Simple, compact, and runs on almost all
0045  *     32-bit and 64-bit systems.
0046  * 2. @ref XXH64_family
0047  *   - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most
0048  *     64-bit systems (but _not_ 32-bit systems).
0049  * 3. @ref XXH3_family
0050  *   - Modern 64-bit and 128-bit hash function family which features improved
0051  *     strength and performance across the board, especially on smaller data.
0052  *     It benefits greatly from SIMD and 64-bit without requiring it.
0053  *
0054  * Benchmarks
0055  * ---
0056  * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04.
0057  * The open source benchmark program is compiled with clang v10.0 using -O3 flag.
0058  *
0059  * | Hash Name            | ISA ext | Width | Large Data Speed | Small Data Velocity |
0060  * | -------------------- | ------- | ----: | ---------------: | ------------------: |
0061  * | XXH3_64bits()        | @b AVX2 |    64 |        59.4 GB/s |               133.1 |
0062  * | MeowHash             | AES-NI  |   128 |        58.2 GB/s |                52.5 |
0063  * | XXH3_128bits()       | @b AVX2 |   128 |        57.9 GB/s |               118.1 |
0064  * | CLHash               | PCLMUL  |    64 |        37.1 GB/s |                58.1 |
0065  * | XXH3_64bits()        | @b SSE2 |    64 |        31.5 GB/s |               133.1 |
0066  * | XXH3_128bits()       | @b SSE2 |   128 |        29.6 GB/s |               118.1 |
0067  * | RAM sequential read  |         |   N/A |        28.0 GB/s |                 N/A |
0068  * | ahash                | AES-NI  |    64 |        22.5 GB/s |               107.2 |
0069  * | City64               |         |    64 |        22.0 GB/s |                76.6 |
0070  * | T1ha2                |         |    64 |        22.0 GB/s |                99.0 |
0071  * | City128              |         |   128 |        21.7 GB/s |                57.7 |
0072  * | FarmHash             | AES-NI  |    64 |        21.3 GB/s |                71.9 |
0073  * | XXH64()              |         |    64 |        19.4 GB/s |                71.0 |
0074  * | SpookyHash           |         |    64 |        19.3 GB/s |                53.2 |
0075  * | Mum                  |         |    64 |        18.0 GB/s |                67.0 |
0076  * | CRC32C               | SSE4.2  |    32 |        13.0 GB/s |                57.9 |
0077  * | XXH32()              |         |    32 |         9.7 GB/s |                71.9 |
0078  * | City32               |         |    32 |         9.1 GB/s |                66.0 |
0079  * | Blake3*              | @b AVX2 |   256 |         4.4 GB/s |                 8.1 |
0080  * | Murmur3              |         |    32 |         3.9 GB/s |                56.1 |
0081  * | SipHash*             |         |    64 |         3.0 GB/s |                43.2 |
0082  * | Blake3*              | @b SSE2 |   256 |         2.4 GB/s |                 8.1 |
0083  * | HighwayHash          |         |    64 |         1.4 GB/s |                 6.0 |
0084  * | FNV64                |         |    64 |         1.2 GB/s |                62.7 |
0085  * | Blake2*              |         |   256 |         1.1 GB/s |                 5.1 |
0086  * | SHA1*                |         |   160 |         0.8 GB/s |                 5.6 |
0087  * | MD5*                 |         |   128 |         0.6 GB/s |                 7.8 |
0088  * @note
0089  *   - Hashes which require a specific ISA extension are noted. SSE2 is also noted,
0090  *     even though it is mandatory on x64.
0091  *   - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic
0092  *     by modern standards.
0093  *   - Small data velocity is a rough average of algorithm's efficiency for small
0094  *     data. For more accurate information, see the wiki.
0095  *   - More benchmarks and strength tests are found on the wiki:
0096  *         https://github.com/Cyan4973/xxHash/wiki
0097  *
0098  * Usage
0099  * ------
0100  * All xxHash variants use a similar API. Changing the algorithm is a trivial
0101  * substitution.
0102  *
0103  * @pre
0104  *    For functions which take an input and length parameter, the following
0105  *    requirements are assumed:
0106  *    - The range from [`input`, `input + length`) is valid, readable memory.
0107  *      - The only exception is if the `length` is `0`, `input` may be `NULL`.
0108  *    - For C++, the objects must have the *TriviallyCopyable* property, as the
0109  *      functions access bytes directly as if it was an array of `unsigned char`.
0110  *
0111  * @anchor single_shot_example
0112  * **Single Shot**
0113  *
0114  * These functions are stateless functions which hash a contiguous block of memory,
0115  * immediately returning the result. They are the easiest and usually the fastest
0116  * option.
0117  *
0118  * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits()
0119  *
0120  * @code{.c}
0121  *   #include <string.h>
0122  *   #include "xxhash.h"
0123  *
0124  *   // Example for a function which hashes a null terminated string with XXH32().
0125  *   XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed)
0126  *   {
0127  *       // NULL pointers are only valid if the length is zero
0128  *       size_t length = (string == NULL) ? 0 : strlen(string);
0129  *       return XXH32(string, length, seed);
0130  *   }
0131  * @endcode
0132  *
0133  * @anchor streaming_example
0134  * **Streaming**
0135  *
0136  * These groups of functions allow incremental hashing of unknown size, even
0137  * more than what would fit in a size_t.
0138  *
0139  * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset()
0140  *
0141  * @code{.c}
0142  *   #include <stdio.h>
0143  *   #include <assert.h>
0144  *   #include "xxhash.h"
0145  *   // Example for a function which hashes a FILE incrementally with XXH3_64bits().
0146  *   XXH64_hash_t hashFile(FILE* f)
0147  *   {
0148  *       // Allocate a state struct. Do not just use malloc() or new.
0149  *       XXH3_state_t* state = XXH3_createState();
0150  *       assert(state != NULL && "Out of memory!");
0151  *       // Reset the state to start a new hashing session.
0152  *       XXH3_64bits_reset(state);
0153  *       char buffer[4096];
0154  *       size_t count;
0155  *       // Read the file in chunks
0156  *       while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) {
0157  *           // Run update() as many times as necessary to process the data
0158  *           XXH3_64bits_update(state, buffer, count);
0159  *       }
0160  *       // Retrieve the finalized hash. This will not change the state.
0161  *       XXH64_hash_t result = XXH3_64bits_digest(state);
0162  *       // Free the state. Do not use free().
0163  *       XXH3_freeState(state);
0164  *       return result;
0165  *   }
0166  * @endcode
0167  *
0168  * @file xxhash.h
0169  * xxHash prototypes and implementation
0170  */
0171 
0172 #if defined (__cplusplus)
0173 extern "C" {
0174 #endif
0175 
0176 /* ****************************
0177  *  INLINE mode
0178  ******************************/
0179 /*!
0180  * @defgroup public Public API
0181  * Contains details on the public xxHash functions.
0182  * @{
0183  */
0184 #ifdef XXH_DOXYGEN
0185 /*!
0186  * @brief Gives access to internal state declaration, required for static allocation.
0187  *
0188  * Incompatible with dynamic linking, due to risks of ABI changes.
0189  *
0190  * Usage:
0191  * @code{.c}
0192  *     #define XXH_STATIC_LINKING_ONLY
0193  *     #include "xxhash.h"
0194  * @endcode
0195  */
0196 #  define XXH_STATIC_LINKING_ONLY
0197 /* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */
0198 
0199 /*!
0200  * @brief Gives access to internal definitions.
0201  *
0202  * Usage:
0203  * @code{.c}
0204  *     #define XXH_STATIC_LINKING_ONLY
0205  *     #define XXH_IMPLEMENTATION
0206  *     #include "xxhash.h"
0207  * @endcode
0208  */
0209 #  define XXH_IMPLEMENTATION
0210 /* Do not undef XXH_IMPLEMENTATION for Doxygen */
0211 
0212 /*!
0213  * @brief Exposes the implementation and marks all functions as `inline`.
0214  *
0215  * Use these build macros to inline xxhash into the target unit.
0216  * Inlining improves performance on small inputs, especially when the length is
0217  * expressed as a compile-time constant:
0218  *
0219  *  https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
0220  *
0221  * It also keeps xxHash symbols private to the unit, so they are not exported.
0222  *
0223  * Usage:
0224  * @code{.c}
0225  *     #define XXH_INLINE_ALL
0226  *     #include "xxhash.h"
0227  * @endcode
0228  * Do not compile and link xxhash.o as a separate object, as it is not useful.
0229  */
0230 #  define XXH_INLINE_ALL
0231 #  undef XXH_INLINE_ALL
0232 /*!
0233  * @brief Exposes the implementation without marking functions as inline.
0234  */
0235 #  define XXH_PRIVATE_API
0236 #  undef XXH_PRIVATE_API
0237 /*!
0238  * @brief Emulate a namespace by transparently prefixing all symbols.
0239  *
0240  * If you want to include _and expose_ xxHash functions from within your own
0241  * library, but also want to avoid symbol collisions with other libraries which
0242  * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix
0243  * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE
0244  * (therefore, avoid empty or numeric values).
0245  *
0246  * Note that no change is required within the calling program as long as it
0247  * includes `xxhash.h`: Regular symbol names will be automatically translated
0248  * by this header.
0249  */
0250 #  define XXH_NAMESPACE /* YOUR NAME HERE */
0251 #  undef XXH_NAMESPACE
0252 #endif
0253 
0254 #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
0255     && !defined(XXH_INLINE_ALL_31684351384)
0256    /* this section should be traversed only once */
0257 #  define XXH_INLINE_ALL_31684351384
0258    /* give access to the advanced API, required to compile implementations */
0259 #  undef XXH_STATIC_LINKING_ONLY   /* avoid macro redef */
0260 #  define XXH_STATIC_LINKING_ONLY
0261    /* make all functions private */
0262 #  undef XXH_PUBLIC_API
0263 #  if defined(__GNUC__)
0264 #    define XXH_PUBLIC_API static __inline __attribute__((unused))
0265 #  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
0266 #    define XXH_PUBLIC_API static inline
0267 #  elif defined(_MSC_VER)
0268 #    define XXH_PUBLIC_API static __inline
0269 #  else
0270      /* note: this version may generate warnings for unused static functions */
0271 #    define XXH_PUBLIC_API static
0272 #  endif
0273 
0274    /*
0275     * This part deals with the special case where a unit wants to inline xxHash,
0276     * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
0277     * such as part of some previously included *.h header file.
0278     * Without further action, the new include would just be ignored,
0279     * and functions would effectively _not_ be inlined (silent failure).
0280     * The following macros solve this situation by prefixing all inlined names,
0281     * avoiding naming collision with previous inclusions.
0282     */
0283    /* Before that, we unconditionally #undef all symbols,
0284     * in case they were already defined with XXH_NAMESPACE.
0285     * They will then be redefined for XXH_INLINE_ALL
0286     */
0287 #  undef XXH_versionNumber
0288     /* XXH32 */
0289 #  undef XXH32
0290 #  undef XXH32_createState
0291 #  undef XXH32_freeState
0292 #  undef XXH32_reset
0293 #  undef XXH32_update
0294 #  undef XXH32_digest
0295 #  undef XXH32_copyState
0296 #  undef XXH32_canonicalFromHash
0297 #  undef XXH32_hashFromCanonical
0298     /* XXH64 */
0299 #  undef XXH64
0300 #  undef XXH64_createState
0301 #  undef XXH64_freeState
0302 #  undef XXH64_reset
0303 #  undef XXH64_update
0304 #  undef XXH64_digest
0305 #  undef XXH64_copyState
0306 #  undef XXH64_canonicalFromHash
0307 #  undef XXH64_hashFromCanonical
0308     /* XXH3_64bits */
0309 #  undef XXH3_64bits
0310 #  undef XXH3_64bits_withSecret
0311 #  undef XXH3_64bits_withSeed
0312 #  undef XXH3_64bits_withSecretandSeed
0313 #  undef XXH3_createState
0314 #  undef XXH3_freeState
0315 #  undef XXH3_copyState
0316 #  undef XXH3_64bits_reset
0317 #  undef XXH3_64bits_reset_withSeed
0318 #  undef XXH3_64bits_reset_withSecret
0319 #  undef XXH3_64bits_update
0320 #  undef XXH3_64bits_digest
0321 #  undef XXH3_generateSecret
0322     /* XXH3_128bits */
0323 #  undef XXH128
0324 #  undef XXH3_128bits
0325 #  undef XXH3_128bits_withSeed
0326 #  undef XXH3_128bits_withSecret
0327 #  undef XXH3_128bits_reset
0328 #  undef XXH3_128bits_reset_withSeed
0329 #  undef XXH3_128bits_reset_withSecret
0330 #  undef XXH3_128bits_reset_withSecretandSeed
0331 #  undef XXH3_128bits_update
0332 #  undef XXH3_128bits_digest
0333 #  undef XXH128_isEqual
0334 #  undef XXH128_cmp
0335 #  undef XXH128_canonicalFromHash
0336 #  undef XXH128_hashFromCanonical
0337     /* Finally, free the namespace itself */
0338 #  undef XXH_NAMESPACE
0339 
0340     /* employ the namespace for XXH_INLINE_ALL */
0341 #  define XXH_NAMESPACE XXH_INLINE_
0342    /*
0343     * Some identifiers (enums, type names) are not symbols,
0344     * but they must nonetheless be renamed to avoid redeclaration.
0345     * Alternative solution: do not redeclare them.
0346     * However, this requires some #ifdefs, and has a more dispersed impact.
0347     * Meanwhile, renaming can be achieved in a single place.
0348     */
0349 #  define XXH_IPREF(Id)   XXH_NAMESPACE ## Id
0350 #  define XXH_OK XXH_IPREF(XXH_OK)
0351 #  define XXH_ERROR XXH_IPREF(XXH_ERROR)
0352 #  define XXH_errorcode XXH_IPREF(XXH_errorcode)
0353 #  define XXH32_canonical_t  XXH_IPREF(XXH32_canonical_t)
0354 #  define XXH64_canonical_t  XXH_IPREF(XXH64_canonical_t)
0355 #  define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
0356 #  define XXH32_state_s XXH_IPREF(XXH32_state_s)
0357 #  define XXH32_state_t XXH_IPREF(XXH32_state_t)
0358 #  define XXH64_state_s XXH_IPREF(XXH64_state_s)
0359 #  define XXH64_state_t XXH_IPREF(XXH64_state_t)
0360 #  define XXH3_state_s  XXH_IPREF(XXH3_state_s)
0361 #  define XXH3_state_t  XXH_IPREF(XXH3_state_t)
0362 #  define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
0363    /* Ensure the header is parsed again, even if it was previously included */
0364 #  undef XXHASH_H_5627135585666179
0365 #  undef XXHASH_H_STATIC_13879238742
0366 #endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
0367 
0368 /* ****************************************************************
0369  *  Stable API
0370  *****************************************************************/
0371 #ifndef XXHASH_H_5627135585666179
0372 #define XXHASH_H_5627135585666179 1
0373 
0374 /*! @brief Marks a global symbol. */
0375 #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
0376 #  if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
0377 #    ifdef XXH_EXPORT
0378 #      define XXH_PUBLIC_API __declspec(dllexport)
0379 #    elif XXH_IMPORT
0380 #      define XXH_PUBLIC_API __declspec(dllimport)
0381 #    endif
0382 #  else
0383 #    define XXH_PUBLIC_API   /* do nothing */
0384 #  endif
0385 #endif
0386 
0387 #ifdef XXH_NAMESPACE
0388 #  define XXH_CAT(A,B) A##B
0389 #  define XXH_NAME2(A,B) XXH_CAT(A,B)
0390 #  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
0391 /* XXH32 */
0392 #  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
0393 #  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
0394 #  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
0395 #  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
0396 #  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
0397 #  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
0398 #  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
0399 #  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
0400 #  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
0401 /* XXH64 */
0402 #  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
0403 #  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
0404 #  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
0405 #  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
0406 #  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
0407 #  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
0408 #  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
0409 #  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
0410 #  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
0411 /* XXH3_64bits */
0412 #  define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
0413 #  define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
0414 #  define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
0415 #  define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
0416 #  define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
0417 #  define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
0418 #  define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
0419 #  define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
0420 #  define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
0421 #  define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
0422 #  define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
0423 #  define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
0424 #  define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
0425 #  define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
0426 #  define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
0427 /* XXH3_128bits */
0428 #  define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
0429 #  define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
0430 #  define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
0431 #  define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
0432 #  define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
0433 #  define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
0434 #  define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
0435 #  define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
0436 #  define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
0437 #  define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
0438 #  define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
0439 #  define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
0440 #  define XXH128_cmp     XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
0441 #  define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
0442 #  define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
0443 #endif
0444 
0445 
0446 /* *************************************
0447 *  Compiler specifics
0448 ***************************************/
0449 
0450 /* specific declaration modes for Windows */
0451 #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
0452 #  if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
0453 #    ifdef XXH_EXPORT
0454 #      define XXH_PUBLIC_API __declspec(dllexport)
0455 #    elif XXH_IMPORT
0456 #      define XXH_PUBLIC_API __declspec(dllimport)
0457 #    endif
0458 #  else
0459 #    define XXH_PUBLIC_API   /* do nothing */
0460 #  endif
0461 #endif
0462 
0463 #if defined (__GNUC__)
0464 # define XXH_CONSTF  __attribute__((const))
0465 # define XXH_PUREF   __attribute__((pure))
0466 # define XXH_MALLOCF __attribute__((malloc))
0467 #else
0468 # define XXH_CONSTF  /* disable */
0469 # define XXH_PUREF
0470 # define XXH_MALLOCF
0471 #endif
0472 
0473 /* *************************************
0474 *  Version
0475 ***************************************/
0476 #define XXH_VERSION_MAJOR    0
0477 #define XXH_VERSION_MINOR    8
0478 #define XXH_VERSION_RELEASE  2
0479 /*! @brief Version number, encoded as two digits each */
0480 #define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
0481 
0482 /*!
0483  * @brief Obtains the xxHash version.
0484  *
0485  * This is mostly useful when xxHash is compiled as a shared library,
0486  * since the returned value comes from the library, as opposed to header file.
0487  *
0488  * @return @ref XXH_VERSION_NUMBER of the invoked library.
0489  */
0490 XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);
0491 
0492 
0493 /* ****************************
0494 *  Common basic types
0495 ******************************/
0496 #include <stddef.h>   /* size_t */
0497 /*!
0498  * @brief Exit code for the streaming API.
0499  */
0500 typedef enum {
0501     XXH_OK = 0, /*!< OK */
0502     XXH_ERROR   /*!< Error */
0503 } XXH_errorcode;
0504 
0505 
0506 /*-**********************************************************************
0507 *  32-bit hash
0508 ************************************************************************/
0509 #if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
0510 /*!
0511  * @brief An unsigned 32-bit integer.
0512  *
0513  * Not necessarily defined to `uint32_t` but functionally equivalent.
0514  */
0515 typedef uint32_t XXH32_hash_t;
0516 
0517 #elif !defined (__VMS) \
0518   && (defined (__cplusplus) \
0519   || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
0520 #   include <stdint.h>
0521     typedef uint32_t XXH32_hash_t;
0522 
0523 #else
0524 #   include <limits.h>
0525 #   if UINT_MAX == 0xFFFFFFFFUL
0526       typedef unsigned int XXH32_hash_t;
0527 #   elif ULONG_MAX == 0xFFFFFFFFUL
0528       typedef unsigned long XXH32_hash_t;
0529 #   else
0530 #     error "unsupported platform: need a 32-bit type"
0531 #   endif
0532 #endif
0533 
0534 /*!
0535  * @}
0536  *
0537  * @defgroup XXH32_family XXH32 family
0538  * @ingroup public
0539  * Contains functions used in the classic 32-bit xxHash algorithm.
0540  *
0541  * @note
0542  *   XXH32 is useful for older platforms, with no or poor 64-bit performance.
0543  *   Note that the @ref XXH3_family provides competitive speed for both 32-bit
0544  *   and 64-bit systems, and offers true 64/128 bit hash results.
0545  *
0546  * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families
0547  * @see @ref XXH32_impl for implementation details
0548  * @{
0549  */
0550 
0551 /*!
0552  * @brief Calculates the 32-bit hash of @p input using xxHash32.
0553  *
0554  * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
0555  *
0556  * See @ref single_shot_example "Single Shot Example" for an example.
0557  *
0558  * @param input The block of data to be hashed, at least @p length bytes in size.
0559  * @param length The length of @p input, in bytes.
0560  * @param seed The 32-bit seed to alter the hash's output predictably.
0561  *
0562  * @pre
0563  *   The memory between @p input and @p input + @p length must be valid,
0564  *   readable, contiguous memory. However, if @p length is `0`, @p input may be
0565  *   `NULL`. In C++, this also must be *TriviallyCopyable*.
0566  *
0567  * @return The calculated 32-bit hash value.
0568  *
0569  * @see
0570  *    XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
0571  *    Direct equivalents for the other variants of xxHash.
0572  * @see
0573  *    XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version.
0574  */
0575 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
0576 
0577 #ifndef XXH_NO_STREAM
0578 /*!
0579  * Streaming functions generate the xxHash value from an incremental input.
0580  * This method is slower than single-call functions, due to state management.
0581  * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
0582  *
0583  * An XXH state must first be allocated using `XXH*_createState()`.
0584  *
0585  * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
0586  *
0587  * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
0588  *
0589  * The function returns an error code, with 0 meaning OK, and any other value
0590  * meaning there is an error.
0591  *
0592  * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
0593  * This function returns the nn-bits hash as an int or long long.
0594  *
0595  * It's still possible to continue inserting input into the hash state after a
0596  * digest, and generate new hash values later on by invoking `XXH*_digest()`.
0597  *
0598  * When done, release the state using `XXH*_freeState()`.
0599  *
0600  * @see streaming_example at the top of @ref xxhash.h for an example.
0601  */
0602 
0603 /*!
0604  * @typedef struct XXH32_state_s XXH32_state_t
0605  * @brief The opaque state struct for the XXH32 streaming API.
0606  *
0607  * @see XXH32_state_s for details.
0608  */
0609 typedef struct XXH32_state_s XXH32_state_t;
0610 
0611 /*!
0612  * @brief Allocates an @ref XXH32_state_t.
0613  *
0614  * Must be freed with XXH32_freeState().
0615  * @return An allocated XXH32_state_t on success, `NULL` on failure.
0616  */
0617 XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
0618 /*!
0619  * @brief Frees an @ref XXH32_state_t.
0620  *
0621  * Must be allocated with XXH32_createState().
0622  * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
0623  * @return XXH_OK.
0624  */
0625 XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
0626 /*!
0627  * @brief Copies one @ref XXH32_state_t to another.
0628  *
0629  * @param dst_state The state to copy to.
0630  * @param src_state The state to copy from.
0631  * @pre
0632  *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
0633  */
0634 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
0635 
0636 /*!
0637  * @brief Resets an @ref XXH32_state_t to begin a new hash.
0638  *
0639  * This function resets and seeds a state. Call it before @ref XXH32_update().
0640  *
0641  * @param statePtr The state struct to reset.
0642  * @param seed The 32-bit seed to alter the hash result predictably.
0643  *
0644  * @pre
0645  *   @p statePtr must not be `NULL`.
0646  *
0647  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
0648  */
0649 XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, XXH32_hash_t seed);
0650 
0651 /*!
0652  * @brief Consumes a block of @p input to an @ref XXH32_state_t.
0653  *
0654  * Call this to incrementally consume blocks of data.
0655  *
0656  * @param statePtr The state struct to update.
0657  * @param input The block of data to be hashed, at least @p length bytes in size.
0658  * @param length The length of @p input, in bytes.
0659  *
0660  * @pre
0661  *   @p statePtr must not be `NULL`.
0662  * @pre
0663  *   The memory between @p input and @p input + @p length must be valid,
0664  *   readable, contiguous memory. However, if @p length is `0`, @p input may be
0665  *   `NULL`. In C++, this also must be *TriviallyCopyable*.
0666  *
0667  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
0668  */
0669 XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
0670 
0671 /*!
0672  * @brief Returns the calculated hash value from an @ref XXH32_state_t.
0673  *
0674  * @note
0675  *   Calling XXH32_digest() will not affect @p statePtr, so you can update,
0676  *   digest, and update again.
0677  *
0678  * @param statePtr The state struct to calculate the hash from.
0679  *
0680  * @pre
0681  *  @p statePtr must not be `NULL`.
0682  *
0683  * @return The calculated xxHash32 value from that state.
0684  */
0685 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
0686 #endif /* !XXH_NO_STREAM */
0687 
0688 /*******   Canonical representation   *******/
0689 
0690 /*
0691  * The default return values from XXH functions are unsigned 32 and 64 bit
0692  * integers.
0693  * This the simplest and fastest format for further post-processing.
0694  *
0695  * However, this leaves open the question of what is the order on the byte level,
0696  * since little and big endian conventions will store the same number differently.
0697  *
0698  * The canonical representation settles this issue by mandating big-endian
0699  * convention, the same convention as human-readable numbers (large digits first).
0700  *
0701  * When writing hash values to storage, sending them over a network, or printing
0702  * them, it's highly recommended to use the canonical representation to ensure
0703  * portability across a wider range of systems, present and future.
0704  *
0705  * The following functions allow transformation of hash values to and from
0706  * canonical format.
0707  */
0708 
0709 /*!
0710  * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
0711  */
0712 typedef struct {
0713     unsigned char digest[4]; /*!< Hash bytes, big endian */
0714 } XXH32_canonical_t;
0715 
0716 /*!
0717  * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
0718  *
0719  * @param dst The @ref XXH32_canonical_t pointer to be stored to.
0720  * @param hash The @ref XXH32_hash_t to be converted.
0721  *
0722  * @pre
0723  *   @p dst must not be `NULL`.
0724  */
0725 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
0726 
0727 /*!
0728  * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
0729  *
0730  * @param src The @ref XXH32_canonical_t to convert.
0731  *
0732  * @pre
0733  *   @p src must not be `NULL`.
0734  *
0735  * @return The converted hash.
0736  */
0737 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
0738 
0739 
0740 /*! @cond Doxygen ignores this part */
0741 #ifdef __has_attribute
0742 # define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
0743 #else
0744 # define XXH_HAS_ATTRIBUTE(x) 0
0745 #endif
0746 /*! @endcond */
0747 
0748 /*! @cond Doxygen ignores this part */
0749 /*
0750  * C23 __STDC_VERSION__ number hasn't been specified yet. For now
0751  * leave as `201711L` (C17 + 1).
0752  * TODO: Update to correct value when its been specified.
0753  */
0754 #define XXH_C23_VN 201711L
0755 /*! @endcond */
0756 
0757 /*! @cond Doxygen ignores this part */
0758 /* C-language Attributes are added in C23. */
0759 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute)
0760 # define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
0761 #else
0762 # define XXH_HAS_C_ATTRIBUTE(x) 0
0763 #endif
0764 /*! @endcond */
0765 
0766 /*! @cond Doxygen ignores this part */
0767 #if defined(__cplusplus) && defined(__has_cpp_attribute)
0768 # define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
0769 #else
0770 # define XXH_HAS_CPP_ATTRIBUTE(x) 0
0771 #endif
0772 /*! @endcond */
0773 
0774 /*! @cond Doxygen ignores this part */
0775 /*
0776  * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
0777  * introduced in CPP17 and C23.
0778  * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
0779  * C23   : https://en.cppreference.com/w/c/language/attributes/fallthrough
0780  */
0781 #if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough)
0782 # define XXH_FALLTHROUGH [[fallthrough]]
0783 #elif XXH_HAS_ATTRIBUTE(__fallthrough__)
0784 # define XXH_FALLTHROUGH __attribute__ ((__fallthrough__))
0785 #else
0786 # define XXH_FALLTHROUGH /* fallthrough */
0787 #endif
0788 /*! @endcond */
0789 
0790 /*! @cond Doxygen ignores this part */
0791 /*
0792  * Define XXH_NOESCAPE for annotated pointers in public API.
0793  * https://clang.llvm.org/docs/AttributeReference.html#noescape
0794  * As of writing this, only supported by clang.
0795  */
0796 #if XXH_HAS_ATTRIBUTE(noescape)
0797 # define XXH_NOESCAPE __attribute__((noescape))
0798 #else
0799 # define XXH_NOESCAPE
0800 #endif
0801 /*! @endcond */
0802 
0803 
0804 /*!
0805  * @}
0806  * @ingroup public
0807  * @{
0808  */
0809 
0810 #ifndef XXH_NO_LONG_LONG
0811 /*-**********************************************************************
0812 *  64-bit hash
0813 ************************************************************************/
0814 #if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
0815 /*!
0816  * @brief An unsigned 64-bit integer.
0817  *
0818  * Not necessarily defined to `uint64_t` but functionally equivalent.
0819  */
0820 typedef uint64_t XXH64_hash_t;
0821 #elif !defined (__VMS) \
0822   && (defined (__cplusplus) \
0823   || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
0824 #  include <stdint.h>
0825    typedef uint64_t XXH64_hash_t;
0826 #else
0827 #  include <limits.h>
0828 #  if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
0829      /* LP64 ABI says uint64_t is unsigned long */
0830      typedef unsigned long XXH64_hash_t;
0831 #  else
0832      /* the following type must have a width of 64-bit */
0833      typedef unsigned long long XXH64_hash_t;
0834 #  endif
0835 #endif
0836 
0837 /*!
0838  * @}
0839  *
0840  * @defgroup XXH64_family XXH64 family
0841  * @ingroup public
0842  * @{
0843  * Contains functions used in the classic 64-bit xxHash algorithm.
0844  *
0845  * @note
0846  *   XXH3 provides competitive speed for both 32-bit and 64-bit systems,
0847  *   and offers true 64/128 bit hash results.
0848  *   It provides better speed for systems with vector processing capabilities.
0849  */
0850 
0851 /*!
0852  * @brief Calculates the 64-bit hash of @p input using xxHash64.
0853  *
0854  * This function usually runs faster on 64-bit systems, but slower on 32-bit
0855  * systems (see benchmark).
0856  *
0857  * @param input The block of data to be hashed, at least @p length bytes in size.
0858  * @param length The length of @p input, in bytes.
0859  * @param seed The 64-bit seed to alter the hash's output predictably.
0860  *
0861  * @pre
0862  *   The memory between @p input and @p input + @p length must be valid,
0863  *   readable, contiguous memory. However, if @p length is `0`, @p input may be
0864  *   `NULL`. In C++, this also must be *TriviallyCopyable*.
0865  *
0866  * @return The calculated 64-bit hash.
0867  *
0868  * @see
0869  *    XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
0870  *    Direct equivalents for the other variants of xxHash.
0871  * @see
0872  *    XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version.
0873  */
0874 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
0875 
0876 /*******   Streaming   *******/
0877 #ifndef XXH_NO_STREAM
0878 /*!
0879  * @brief The opaque state struct for the XXH64 streaming API.
0880  *
0881  * @see XXH64_state_s for details.
0882  */
0883 typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
0884 
0885 /*!
0886  * @brief Allocates an @ref XXH64_state_t.
0887  *
0888  * Must be freed with XXH64_freeState().
0889  * @return An allocated XXH64_state_t on success, `NULL` on failure.
0890  */
0891 XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
0892 
0893 /*!
0894  * @brief Frees an @ref XXH64_state_t.
0895  *
0896  * Must be allocated with XXH64_createState().
0897  * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState().
0898  * @return XXH_OK.
0899  */
0900 XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
0901 
0902 /*!
0903  * @brief Copies one @ref XXH64_state_t to another.
0904  *
0905  * @param dst_state The state to copy to.
0906  * @param src_state The state to copy from.
0907  * @pre
0908  *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
0909  */
0910 XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state);
0911 
0912 /*!
0913  * @brief Resets an @ref XXH64_state_t to begin a new hash.
0914  *
0915  * This function resets and seeds a state. Call it before @ref XXH64_update().
0916  *
0917  * @param statePtr The state struct to reset.
0918  * @param seed The 64-bit seed to alter the hash result predictably.
0919  *
0920  * @pre
0921  *   @p statePtr must not be `NULL`.
0922  *
0923  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
0924  */
0925 XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);
0926 
0927 /*!
0928  * @brief Consumes a block of @p input to an @ref XXH64_state_t.
0929  *
0930  * Call this to incrementally consume blocks of data.
0931  *
0932  * @param statePtr The state struct to update.
0933  * @param input The block of data to be hashed, at least @p length bytes in size.
0934  * @param length The length of @p input, in bytes.
0935  *
0936  * @pre
0937  *   @p statePtr must not be `NULL`.
0938  * @pre
0939  *   The memory between @p input and @p input + @p length must be valid,
0940  *   readable, contiguous memory. However, if @p length is `0`, @p input may be
0941  *   `NULL`. In C++, this also must be *TriviallyCopyable*.
0942  *
0943  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
0944  */
0945 XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
0946 
0947 /*!
0948  * @brief Returns the calculated hash value from an @ref XXH64_state_t.
0949  *
0950  * @note
0951  *   Calling XXH64_digest() will not affect @p statePtr, so you can update,
0952  *   digest, and update again.
0953  *
0954  * @param statePtr The state struct to calculate the hash from.
0955  *
0956  * @pre
0957  *  @p statePtr must not be `NULL`.
0958  *
0959  * @return The calculated xxHash64 value from that state.
0960  */
0961 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
0962 #endif /* !XXH_NO_STREAM */
0963 /*******   Canonical representation   *******/
0964 
0965 /*!
0966  * @brief Canonical (big endian) representation of @ref XXH64_hash_t.
0967  */
0968 typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
0969 
0970 /*!
0971  * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t.
0972  *
0973  * @param dst The @ref XXH64_canonical_t pointer to be stored to.
0974  * @param hash The @ref XXH64_hash_t to be converted.
0975  *
0976  * @pre
0977  *   @p dst must not be `NULL`.
0978  */
0979 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash);
0980 
0981 /*!
0982  * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t.
0983  *
0984  * @param src The @ref XXH64_canonical_t to convert.
0985  *
0986  * @pre
0987  *   @p src must not be `NULL`.
0988  *
0989  * @return The converted hash.
0990  */
0991 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src);
0992 
0993 #ifndef XXH_NO_XXH3
0994 
0995 /*!
0996  * @}
0997  * ************************************************************************
0998  * @defgroup XXH3_family XXH3 family
0999  * @ingroup public
1000  * @{
1001  *
1002  * XXH3 is a more recent hash algorithm featuring:
1003  *  - Improved speed for both small and large inputs
1004  *  - True 64-bit and 128-bit outputs
1005  *  - SIMD acceleration
1006  *  - Improved 32-bit viability
1007  *
1008  * Speed analysis methodology is explained here:
1009  *
1010  *    https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
1011  *
1012  * Compared to XXH64, expect XXH3 to run approximately
1013  * ~2x faster on large inputs and >3x faster on small ones,
1014  * exact differences vary depending on platform.
1015  *
1016  * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
1017  * but does not require it.
1018  * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3
1019  * at competitive speeds, even without vector support. Further details are
1020  * explained in the implementation.
1021  *
1022  * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD
1023  * implementations for many common platforms:
1024  *   - AVX512
1025  *   - AVX2
1026  *   - SSE2
1027  *   - ARM NEON
1028  *   - WebAssembly SIMD128
1029  *   - POWER8 VSX
1030  *   - s390x ZVector
1031  * This can be controlled via the @ref XXH_VECTOR macro, but it automatically
1032  * selects the best version according to predefined macros. For the x86 family, an
1033  * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c.
1034  *
1035  * XXH3 implementation is portable:
1036  * it has a generic C90 formulation that can be compiled on any platform,
1037  * all implementations generate exactly the same hash value on all platforms.
1038  * Starting from v0.8.0, it's also labelled "stable", meaning that
1039  * any future version will also generate the same hash value.
1040  *
1041  * XXH3 offers 2 variants, _64bits and _128bits.
1042  *
1043  * When only 64 bits are needed, prefer invoking the _64bits variant, as it
1044  * reduces the amount of mixing, resulting in faster speed on small inputs.
1045  * It's also generally simpler to manipulate a scalar return type than a struct.
1046  *
1047  * The API supports one-shot hashing, streaming mode, and custom secrets.
1048  */
1049 /*-**********************************************************************
1050 *  XXH3 64-bit variant
1051 ************************************************************************/
1052 
1053 /*!
1054  * @brief 64-bit unseeded variant of XXH3.
1055  *
1056  * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of 0, however
1057  * it may have slightly better performance due to constant propagation of the
1058  * defaults.
1059  *
1060  * @see
1061  *    XXH32(), XXH64(), XXH3_128bits(): equivalent for the other xxHash algorithms
1062  * @see
1063  *    XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants
1064  * @see
1065  *    XXH3_64bits_reset(), XXH3_64bits_update(), XXH3_64bits_digest(): Streaming version.
1066  */
1067 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length);
1068 
1069 /*!
1070  * @brief 64-bit seeded variant of XXH3
1071  *
1072  * This variant generates a custom secret on the fly based on default secret
1073  * altered using the `seed` value.
1074  *
1075  * While this operation is decently fast, note that it's not completely free.
1076  *
1077  * @note
1078  *    seed == 0 produces the same results as @ref XXH3_64bits().
1079  *
1080  * @param input The data to hash
1081  * @param length The length
1082  * @param seed The 64-bit seed to alter the state.
1083  */
1084 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
1085 
1086 /*!
1087  * The bare minimum size for a custom secret.
1088  *
1089  * @see
1090  *  XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
1091  *  XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
1092  */
1093 #define XXH3_SECRET_SIZE_MIN 136
1094 
1095 /*!
1096  * @brief 64-bit variant of XXH3 with a custom "secret".
1097  *
1098  * It's possible to provide any blob of bytes as a "secret" to generate the hash.
1099  * This makes it more difficult for an external actor to prepare an intentional collision.
1100  * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
1101  * However, the quality of the secret impacts the dispersion of the hash algorithm.
1102  * Therefore, the secret _must_ look like a bunch of random bytes.
1103  * Avoid "trivial" or structured data such as repeated sequences or a text document.
1104  * Whenever in doubt about the "randomness" of the blob of bytes,
1105  * consider employing "XXH3_generateSecret()" instead (see below).
1106  * It will generate a proper high entropy secret derived from the blob of bytes.
1107  * Another advantage of using XXH3_generateSecret() is that
1108  * it guarantees that all bits within the initial blob of bytes
1109  * will impact every bit of the output.
1110  * This is not necessarily the case when using the blob of bytes directly
1111  * because, when hashing _small_ inputs, only a portion of the secret is employed.
1112  */
1113 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1114 
1115 
1116 /*******   Streaming   *******/
1117 #ifndef XXH_NO_STREAM
1118 /*
1119  * Streaming requires state maintenance.
1120  * This operation costs memory and CPU.
1121  * As a consequence, streaming is slower than one-shot hashing.
1122  * For better performance, prefer one-shot functions whenever applicable.
1123  */
1124 
1125 /*!
1126  * @brief The state struct for the XXH3 streaming API.
1127  *
1128  * @see XXH3_state_s for details.
1129  */
1130 typedef struct XXH3_state_s XXH3_state_t;
1131 XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
1132 XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
1133 
1134 /*!
1135  * @brief Copies one @ref XXH3_state_t to another.
1136  *
1137  * @param dst_state The state to copy to.
1138  * @param src_state The state to copy from.
1139  * @pre
1140  *   @p dst_state and @p src_state must not be `NULL` and must not overlap.
1141  */
1142 XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state);
1143 
1144 /*!
1145  * @brief Resets an @ref XXH3_state_t to begin a new hash.
1146  *
1147  * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_64bits_update().
1148  * Digest will be equivalent to `XXH3_64bits()`.
1149  *
1150  * @param statePtr The state struct to reset.
1151  *
1152  * @pre
1153  *   @p statePtr must not be `NULL`.
1154  *
1155  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1156  *
1157  */
1158 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1159 
1160 /*!
1161  * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1162  *
1163  * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_64bits_update().
1164  * Digest will be equivalent to `XXH3_64bits_withSeed()`.
1165  *
1166  * @param statePtr The state struct to reset.
1167  * @param seed     The 64-bit seed to alter the state.
1168  *
1169  * @pre
1170  *   @p statePtr must not be `NULL`.
1171  *
1172  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1173  *
1174  */
1175 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1176 
1177 /*!
1178  * XXH3_64bits_reset_withSecret():
1179  * `secret` is referenced, it _must outlive_ the hash streaming session.
1180  * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
1181  * and the quality of produced hash values depends on secret's entropy
1182  * (secret's content should look like a bunch of random bytes).
1183  * When in doubt about the randomness of a candidate `secret`,
1184  * consider employing `XXH3_generateSecret()` instead (see below).
1185  */
1186 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1187 
1188 /*!
1189  * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1190  *
1191  * Call this to incrementally consume blocks of data.
1192  *
1193  * @param statePtr The state struct to update.
1194  * @param input The block of data to be hashed, at least @p length bytes in size.
1195  * @param length The length of @p input, in bytes.
1196  *
1197  * @pre
1198  *   @p statePtr must not be `NULL`.
1199  * @pre
1200  *   The memory between @p input and @p input + @p length must be valid,
1201  *   readable, contiguous memory. However, if @p length is `0`, @p input may be
1202  *   `NULL`. In C++, this also must be *TriviallyCopyable*.
1203  *
1204  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1205  */
1206 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1207 
1208 /*!
1209  * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t.
1210  *
1211  * @note
1212  *   Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
1213  *   digest, and update again.
1214  *
1215  * @param statePtr The state struct to calculate the hash from.
1216  *
1217  * @pre
1218  *  @p statePtr must not be `NULL`.
1219  *
1220  * @return The calculated XXH3 64-bit hash value from that state.
1221  */
1222 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t  XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1223 #endif /* !XXH_NO_STREAM */
1224 
1225 /* note : canonical representation of XXH3 is the same as XXH64
1226  * since they both produce XXH64_hash_t values */
1227 
1228 
1229 /*-**********************************************************************
1230 *  XXH3 128-bit variant
1231 ************************************************************************/
1232 
1233 /*!
1234  * @brief The return value from 128-bit hashes.
1235  *
1236  * Stored in little endian order, although the fields themselves are in native
1237  * endianness.
1238  */
1239 typedef struct {
1240     XXH64_hash_t low64;   /*!< `value & 0xFFFFFFFFFFFFFFFF` */
1241     XXH64_hash_t high64;  /*!< `value >> 64` */
1242 } XXH128_hash_t;
1243 
1244 /*!
1245  * @brief Unseeded 128-bit variant of XXH3
1246  *
1247  * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead
1248  * for shorter inputs.
1249  *
1250  * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of 0, however
1251  * it may have slightly better performance due to constant propagation of the
1252  * defaults.
1253  *
1254  * @see
1255  *    XXH32(), XXH64(), XXH3_64bits(): equivalent for the other xxHash algorithms
1256  * @see
1257  *    XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
1258  * @see
1259  *    XXH3_128bits_reset(), XXH3_128bits_update(), XXH3_128bits_digest(): Streaming version.
1260  */
1261 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len);
1262 /*! @brief Seeded 128-bit variant of XXH3. @see XXH3_64bits_withSeed(). */
1263 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1264 /*! @brief Custom secret 128-bit variant of XXH3. @see XXH3_64bits_withSecret(). */
1265 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1266 
1267 /*******   Streaming   *******/
1268 #ifndef XXH_NO_STREAM
1269 /*
1270  * Streaming requires state maintenance.
1271  * This operation costs memory and CPU.
1272  * As a consequence, streaming is slower than one-shot hashing.
1273  * For better performance, prefer one-shot functions whenever applicable.
1274  *
1275  * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
1276  * Use already declared XXH3_createState() and XXH3_freeState().
1277  *
1278  * All reset and streaming functions have same meaning as their 64-bit counterpart.
1279  */
1280 
1281 /*!
1282  * @brief Resets an @ref XXH3_state_t to begin a new hash.
1283  *
1284  * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_128bits_update().
1285  * Digest will be equivalent to `XXH3_128bits()`.
1286  *
1287  * @param statePtr The state struct to reset.
1288  *
1289  * @pre
1290  *   @p statePtr must not be `NULL`.
1291  *
1292  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1293  *
1294  */
1295 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1296 
1297 /*!
1298  * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1299  *
1300  * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_128bits_update().
1301  * Digest will be equivalent to `XXH3_128bits_withSeed()`.
1302  *
1303  * @param statePtr The state struct to reset.
1304  * @param seed     The 64-bit seed to alter the state.
1305  *
1306  * @pre
1307  *   @p statePtr must not be `NULL`.
1308  *
1309  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1310  *
1311  */
1312 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1313 /*! @brief Custom secret 128-bit variant of XXH3. @see XXH_64bits_reset_withSecret(). */
1314 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1315 
1316 /*!
1317  * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1318  *
1319  * Call this to incrementally consume blocks of data.
1320  *
1321  * @param statePtr The state struct to update.
1322  * @param input The block of data to be hashed, at least @p length bytes in size.
1323  * @param length The length of @p input, in bytes.
1324  *
1325  * @pre
1326  *   @p statePtr must not be `NULL`.
1327  * @pre
1328  *   The memory between @p input and @p input + @p length must be valid,
1329  *   readable, contiguous memory. However, if @p length is `0`, @p input may be
1330  *   `NULL`. In C++, this also must be *TriviallyCopyable*.
1331  *
1332  * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1333  */
1334 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1335 
1336 /*!
1337  * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t.
1338  *
1339  * @note
1340  *   Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
1341  *   digest, and update again.
1342  *
1343  * @param statePtr The state struct to calculate the hash from.
1344  *
1345  * @pre
1346  *  @p statePtr must not be `NULL`.
1347  *
1348  * @return The calculated XXH3 128-bit hash value from that state.
1349  */
1350 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1351 #endif /* !XXH_NO_STREAM */
1352 
1353 /* Following helper functions make it possible to compare XXH128_hast_t values.
1354  * Since XXH128_hash_t is a structure, this capability is not offered by the language.
1355  * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
1356 
1357 /*!
1358  * XXH128_isEqual():
1359  * Return: 1 if `h1` and `h2` are equal, 0 if they are not.
1360  */
1361 XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
1362 
1363 /*!
1364  * @brief Compares two @ref XXH128_hash_t
1365  * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
1366  *
1367  * @return: >0 if *h128_1  > *h128_2
1368  *          =0 if *h128_1 == *h128_2
1369  *          <0 if *h128_1  < *h128_2
1370  */
1371 XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2);
1372 
1373 
1374 /*******   Canonical representation   *******/
1375 typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
1376 
1377 
1378 /*!
1379  * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t.
1380  *
1381  * @param dst The @ref XXH128_canonical_t pointer to be stored to.
1382  * @param hash The @ref XXH128_hash_t to be converted.
1383  *
1384  * @pre
1385  *   @p dst must not be `NULL`.
1386  */
1387 XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash);
1388 
1389 /*!
1390  * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t.
1391  *
1392  * @param src The @ref XXH128_canonical_t to convert.
1393  *
1394  * @pre
1395  *   @p src must not be `NULL`.
1396  *
1397  * @return The converted hash.
1398  */
1399 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src);
1400 
1401 
1402 #endif  /* !XXH_NO_XXH3 */
1403 #endif  /* XXH_NO_LONG_LONG */
1404 
1405 /*!
1406  * @}
1407  */
1408 #endif /* XXHASH_H_5627135585666179 */
1409 
1410 
1411 
1412 #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
1413 #define XXHASH_H_STATIC_13879238742
1414 /* ****************************************************************************
1415  * This section contains declarations which are not guaranteed to remain stable.
1416  * They may change in future versions, becoming incompatible with a different
1417  * version of the library.
1418  * These declarations should only be used with static linking.
1419  * Never use them in association with dynamic linking!
1420  ***************************************************************************** */
1421 
1422 /*
1423  * These definitions are only present to allow static allocation
1424  * of XXH states, on stack or in a struct, for example.
1425  * Never **ever** access their members directly.
1426  */
1427 
1428 /*!
1429  * @internal
1430  * @brief Structure for XXH32 streaming API.
1431  *
1432  * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1433  * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1434  * an opaque type. This allows fields to safely be changed.
1435  *
1436  * Typedef'd to @ref XXH32_state_t.
1437  * Do not access the members of this struct directly.
1438  * @see XXH64_state_s, XXH3_state_s
1439  */
1440 struct XXH32_state_s {
1441    XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
1442    XXH32_hash_t large_len;    /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
1443    XXH32_hash_t v[4];         /*!< Accumulator lanes */
1444    XXH32_hash_t mem32[4];     /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
1445    XXH32_hash_t memsize;      /*!< Amount of data in @ref mem32 */
1446    XXH32_hash_t reserved;     /*!< Reserved field. Do not read nor write to it. */
1447 };   /* typedef'd to XXH32_state_t */
1448 
1449 
1450 #ifndef XXH_NO_LONG_LONG  /* defined when there is no 64-bit support */
1451 
1452 /*!
1453  * @internal
1454  * @brief Structure for XXH64 streaming API.
1455  *
1456  * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1457  * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1458  * an opaque type. This allows fields to safely be changed.
1459  *
1460  * Typedef'd to @ref XXH64_state_t.
1461  * Do not access the members of this struct directly.
1462  * @see XXH32_state_s, XXH3_state_s
1463  */
1464 struct XXH64_state_s {
1465    XXH64_hash_t total_len;    /*!< Total length hashed. This is always 64-bit. */
1466    XXH64_hash_t v[4];         /*!< Accumulator lanes */
1467    XXH64_hash_t mem64[4];     /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
1468    XXH32_hash_t memsize;      /*!< Amount of data in @ref mem64 */
1469    XXH32_hash_t reserved32;   /*!< Reserved field, needed for padding anyways*/
1470    XXH64_hash_t reserved64;   /*!< Reserved field. Do not read or write to it. */
1471 };   /* typedef'd to XXH64_state_t */
1472 
1473 #ifndef XXH_NO_XXH3
1474 
1475 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
1476 #  include <stdalign.h>
1477 #  define XXH_ALIGN(n)      alignas(n)
1478 #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
1479 /* In C++ alignas() is a keyword */
1480 #  define XXH_ALIGN(n)      alignas(n)
1481 #elif defined(__GNUC__)
1482 #  define XXH_ALIGN(n)      __attribute__ ((aligned(n)))
1483 #elif defined(_MSC_VER)
1484 #  define XXH_ALIGN(n)      __declspec(align(n))
1485 #else
1486 #  define XXH_ALIGN(n)   /* disabled */
1487 #endif
1488 
1489 /* Old GCC versions only accept the attribute after the type in structures. */
1490 #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))   /* C11+ */ \
1491     && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
1492     && defined(__GNUC__)
1493 #   define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
1494 #else
1495 #   define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
1496 #endif
1497 
1498 /*!
1499  * @brief The size of the internal XXH3 buffer.
1500  *
1501  * This is the optimal update size for incremental hashing.
1502  *
1503  * @see XXH3_64b_update(), XXH3_128b_update().
1504  */
1505 #define XXH3_INTERNALBUFFER_SIZE 256
1506 
1507 /*!
1508  * @internal
1509  * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
1510  *
1511  * This is the size used in @ref XXH3_kSecret and the seeded functions.
1512  *
1513  * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
1514  */
1515 #define XXH3_SECRET_DEFAULT_SIZE 192
1516 
1517 /*!
1518  * @internal
1519  * @brief Structure for XXH3 streaming API.
1520  *
1521  * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1522  * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
1523  * Otherwise it is an opaque type.
1524  * Never use this definition in combination with dynamic library.
1525  * This allows fields to safely be changed in the future.
1526  *
1527  * @note ** This structure has a strict alignment requirement of 64 bytes!! **
1528  * Do not allocate this with `malloc()` or `new`,
1529  * it will not be sufficiently aligned.
1530  * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
1531  *
1532  * Typedef'd to @ref XXH3_state_t.
1533  * Do never access the members of this struct directly.
1534  *
1535  * @see XXH3_INITSTATE() for stack initialization.
1536  * @see XXH3_createState(), XXH3_freeState().
1537  * @see XXH32_state_s, XXH64_state_s
1538  */
1539 struct XXH3_state_s {
1540    XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
1541        /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */
1542    XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
1543        /*!< Used to store a custom secret generated from a seed. */
1544    XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
1545        /*!< The internal buffer. @see XXH32_state_s::mem32 */
1546    XXH32_hash_t bufferedSize;
1547        /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
1548    XXH32_hash_t useSeed;
1549        /*!< Reserved field. Needed for padding on 64-bit. */
1550    size_t nbStripesSoFar;
1551        /*!< Number or stripes processed. */
1552    XXH64_hash_t totalLen;
1553        /*!< Total length hashed. 64-bit even on 32-bit targets. */
1554    size_t nbStripesPerBlock;
1555        /*!< Number of stripes per block. */
1556    size_t secretLimit;
1557        /*!< Size of @ref customSecret or @ref extSecret */
1558    XXH64_hash_t seed;
1559        /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
1560    XXH64_hash_t reserved64;
1561        /*!< Reserved field. */
1562    const unsigned char* extSecret;
1563        /*!< Reference to an external secret for the _withSecret variants, NULL
1564         *   for other variants. */
1565    /* note: there may be some padding at the end due to alignment on 64 bytes */
1566 }; /* typedef'd to XXH3_state_t */
1567 
1568 #undef XXH_ALIGN_MEMBER
1569 
1570 /*!
1571  * @brief Initializes a stack-allocated `XXH3_state_s`.
1572  *
1573  * When the @ref XXH3_state_t structure is merely emplaced on stack,
1574  * it should be initialized with XXH3_INITSTATE() or a memset()
1575  * in case its first reset uses XXH3_NNbits_reset_withSeed().
1576  * This init can be omitted if the first reset uses default or _withSecret mode.
1577  * This operation isn't necessary when the state is created with XXH3_createState().
1578  * Note that this doesn't prepare the state for a streaming operation,
1579  * it's still necessary to use XXH3_NNbits_reset*() afterwards.
1580  */
1581 #define XXH3_INITSTATE(XXH3_state_ptr)                       \
1582     do {                                                     \
1583         XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \
1584         tmp_xxh3_state_ptr->seed = 0;                        \
1585         tmp_xxh3_state_ptr->extSecret = NULL;                \
1586     } while(0)
1587 
1588 
1589 /*!
1590  * simple alias to pre-selected XXH3_128bits variant
1591  */
1592 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1593 
1594 
1595 /* ===   Experimental API   === */
1596 /* Symbols defined below must be considered tied to a specific library version. */
1597 
1598 /*!
1599  * XXH3_generateSecret():
1600  *
1601  * Derive a high-entropy secret from any user-defined content, named customSeed.
1602  * The generated secret can be used in combination with `*_withSecret()` functions.
1603  * The `_withSecret()` variants are useful to provide a higher level of protection
1604  * than 64-bit seed, as it becomes much more difficult for an external actor to
1605  * guess how to impact the calculation logic.
1606  *
1607  * The function accepts as input a custom seed of any length and any content,
1608  * and derives from it a high-entropy secret of length @p secretSize into an
1609  * already allocated buffer @p secretBuffer.
1610  *
1611  * The generated secret can then be used with any `*_withSecret()` variant.
1612  * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(),
1613  * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret()
1614  * are part of this list. They all accept a `secret` parameter
1615  * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN)
1616  * _and_ feature very high entropy (consist of random-looking bytes).
1617  * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can
1618  * be employed to ensure proper quality.
1619  *
1620  * @p customSeed can be anything. It can have any size, even small ones,
1621  * and its content can be anything, even "poor entropy" sources such as a bunch
1622  * of zeroes. The resulting `secret` will nonetheless provide all required qualities.
1623  *
1624  * @pre
1625  *   - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN
1626  *   - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
1627  *
1628  * Example code:
1629  * @code{.c}
1630  *    #include <stdio.h>
1631  *    #include <stdlib.h>
1632  *    #include <string.h>
1633  *    #define XXH_STATIC_LINKING_ONLY // expose unstable API
1634  *    #include "xxhash.h"
1635  *    // Hashes argv[2] using the entropy from argv[1].
1636  *    int main(int argc, char* argv[])
1637  *    {
1638  *        char secret[XXH3_SECRET_SIZE_MIN];
1639  *        if (argv != 3) { return 1; }
1640  *        XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1]));
1641  *        XXH64_hash_t h = XXH3_64bits_withSecret(
1642  *             argv[2], strlen(argv[2]),
1643  *             secret, sizeof(secret)
1644  *        );
1645  *        printf("%016llx\n", (unsigned long long) h);
1646  *    }
1647  * @endcode
1648  */
1649 XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize);
1650 
1651 /*!
1652  * @brief Generate the same secret as the _withSeed() variants.
1653  *
1654  * The generated secret can be used in combination with
1655  *`*_withSecret()` and `_withSecretandSeed()` variants.
1656  *
1657  * Example C++ `std::string` hash class:
1658  * @code{.cpp}
1659  *    #include <string>
1660  *    #define XXH_STATIC_LINKING_ONLY // expose unstable API
1661  *    #include "xxhash.h"
1662  *    // Slow, seeds each time
1663  *    class HashSlow {
1664  *        XXH64_hash_t seed;
1665  *    public:
1666  *        HashSlow(XXH64_hash_t s) : seed{s} {}
1667  *        size_t operator()(const std::string& x) const {
1668  *            return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)};
1669  *        }
1670  *    };
1671  *    // Fast, caches the seeded secret for future uses.
1672  *    class HashFast {
1673  *        unsigned char secret[XXH3_SECRET_SIZE_MIN];
1674  *    public:
1675  *        HashFast(XXH64_hash_t s) {
1676  *            XXH3_generateSecret_fromSeed(secret, seed);
1677  *        }
1678  *        size_t operator()(const std::string& x) const {
1679  *            return size_t{
1680  *                XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret))
1681  *            };
1682  *        }
1683  *    };
1684  * @endcode
1685  * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes
1686  * @param seed The seed to seed the state.
1687  */
1688 XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed);
1689 
1690 /*!
1691  * These variants generate hash values using either
1692  * @p seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes)
1693  * or @p secret for "large" keys (>= XXH3_MIDSIZE_MAX).
1694  *
1695  * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
1696  * `_withSeed()` has to generate the secret on the fly for "large" keys.
1697  * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
1698  * `_withSecret()` has to generate the masks on the fly for "small" keys,
1699  * which requires more instructions than _withSeed() variants.
1700  * Therefore, _withSecretandSeed variant combines the best of both worlds.
1701  *
1702  * When @p secret has been generated by XXH3_generateSecret_fromSeed(),
1703  * this variant produces *exactly* the same results as `_withSeed()` variant,
1704  * hence offering only a pure speed benefit on "large" input,
1705  * by skipping the need to regenerate the secret for every large input.
1706  *
1707  * Another usage scenario is to hash the secret to a 64-bit hash value,
1708  * for example with XXH3_64bits(), which then becomes the seed,
1709  * and then employ both the seed and the secret in _withSecretandSeed().
1710  * On top of speed, an added benefit is that each bit in the secret
1711  * has a 50% chance to swap each bit in the output, via its impact to the seed.
1712  *
1713  * This is not guaranteed when using the secret directly in "small data" scenarios,
1714  * because only portions of the secret are employed for small data.
1715  */
1716 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t
1717 XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len,
1718                               XXH_NOESCAPE const void* secret, size_t secretSize,
1719                               XXH64_hash_t seed);
1720 /*! @copydoc XXH3_64bits_withSecretandSeed() */
1721 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t
1722 XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length,
1723                                XXH_NOESCAPE const void* secret, size_t secretSize,
1724                                XXH64_hash_t seed64);
1725 #ifndef XXH_NO_STREAM
1726 /*! @copydoc XXH3_64bits_withSecretandSeed() */
1727 XXH_PUBLIC_API XXH_errorcode
1728 XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
1729                                     XXH_NOESCAPE const void* secret, size_t secretSize,
1730                                     XXH64_hash_t seed64);
1731 /*! @copydoc XXH3_64bits_withSecretandSeed() */
1732 XXH_PUBLIC_API XXH_errorcode
1733 XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
1734                                      XXH_NOESCAPE const void* secret, size_t secretSize,
1735                                      XXH64_hash_t seed64);
1736 #endif /* !XXH_NO_STREAM */
1737 
1738 #endif  /* !XXH_NO_XXH3 */
1739 #endif  /* XXH_NO_LONG_LONG */
1740 #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
1741 #  define XXH_IMPLEMENTATION
1742 #endif
1743 
1744 #endif  /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
1745 
1746 
1747 /* ======================================================================== */
1748 /* ======================================================================== */
1749 /* ======================================================================== */
1750 
1751 
1752 /*-**********************************************************************
1753  * xxHash implementation
1754  *-**********************************************************************
1755  * xxHash's implementation used to be hosted inside xxhash.c.
1756  *
1757  * However, inlining requires implementation to be visible to the compiler,
1758  * hence be included alongside the header.
1759  * Previously, implementation was hosted inside xxhash.c,
1760  * which was then #included when inlining was activated.
1761  * This construction created issues with a few build and install systems,
1762  * as it required xxhash.c to be stored in /include directory.
1763  *
1764  * xxHash implementation is now directly integrated within xxhash.h.
1765  * As a consequence, xxhash.c is no longer needed in /include.
1766  *
1767  * xxhash.c is still available and is still useful.
1768  * In a "normal" setup, when xxhash is not inlined,
1769  * xxhash.h only exposes the prototypes and public symbols,
1770  * while xxhash.c can be built into an object file xxhash.o
1771  * which can then be linked into the final binary.
1772  ************************************************************************/
1773 
1774 #if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
1775    || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
1776 #  define XXH_IMPLEM_13a8737387
1777 
1778 /* *************************************
1779 *  Tuning parameters
1780 ***************************************/
1781 
1782 /*!
1783  * @defgroup tuning Tuning parameters
1784  * @{
1785  *
1786  * Various macros to control xxHash's behavior.
1787  */
1788 #ifdef XXH_DOXYGEN
1789 /*!
1790  * @brief Define this to disable 64-bit code.
1791  *
1792  * Useful if only using the @ref XXH32_family and you have a strict C90 compiler.
1793  */
1794 #  define XXH_NO_LONG_LONG
1795 #  undef XXH_NO_LONG_LONG /* don't actually */
1796 /*!
1797  * @brief Controls how unaligned memory is accessed.
1798  *
1799  * By default, access to unaligned memory is controlled by `memcpy()`, which is
1800  * safe and portable.
1801  *
1802  * Unfortunately, on some target/compiler combinations, the generated assembly
1803  * is sub-optimal.
1804  *
1805  * The below switch allow selection of a different access method
1806  * in the search for improved performance.
1807  *
1808  * @par Possible options:
1809  *
1810  *  - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
1811  *   @par
1812  *     Use `memcpy()`. Safe and portable. Note that most modern compilers will
1813  *     eliminate the function call and treat it as an unaligned access.
1814  *
1815  *  - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))`
1816  *   @par
1817  *     Depends on compiler extensions and is therefore not portable.
1818  *     This method is safe _if_ your compiler supports it,
1819  *     and *generally* as fast or faster than `memcpy`.
1820  *
1821  *  - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
1822  *  @par
1823  *     Casts directly and dereferences. This method doesn't depend on the
1824  *     compiler, but it violates the C standard as it directly dereferences an
1825  *     unaligned pointer. It can generate buggy code on targets which do not
1826  *     support unaligned memory accesses, but in some circumstances, it's the
1827  *     only known way to get the most performance.
1828  *
1829  *  - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
1830  *  @par
1831  *     Also portable. This can generate the best code on old compilers which don't
1832  *     inline small `memcpy()` calls, and it might also be faster on big-endian
1833  *     systems which lack a native byteswap instruction. However, some compilers
1834  *     will emit literal byteshifts even if the target supports unaligned access.
1835  *
1836  *
1837  * @warning
1838  *   Methods 1 and 2 rely on implementation-defined behavior. Use these with
1839  *   care, as what works on one compiler/platform/optimization level may cause
1840  *   another to read garbage data or even crash.
1841  *
1842  * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
1843  *
1844  * Prefer these methods in priority order (0 > 3 > 1 > 2)
1845  */
1846 #  define XXH_FORCE_MEMORY_ACCESS 0
1847 
1848 /*!
1849  * @def XXH_SIZE_OPT
1850  * @brief Controls how much xxHash optimizes for size.
1851  *
1852  * xxHash, when compiled, tends to result in a rather large binary size. This
1853  * is mostly due to heavy usage to forced inlining and constant folding of the
1854  * @ref XXH3_family to increase performance.
1855  *
1856  * However, some developers prefer size over speed. This option can
1857  * significantly reduce the size of the generated code. When using the `-Os`
1858  * or `-Oz` options on GCC or Clang, this is defined to 1 by default,
1859  * otherwise it is defined to 0.
1860  *
1861  * Most of these size optimizations can be controlled manually.
1862  *
1863  * This is a number from 0-2.
1864  *  - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed
1865  *    comes first.
1866  *  - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more
1867  *    conservative and disables hacks that increase code size. It implies the
1868  *    options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0,
1869  *    and @ref XXH3_NEON_LANES == 8 if they are not already defined.
1870  *  - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible.
1871  *    Performance may cry. For example, the single shot functions just use the
1872  *    streaming API.
1873  */
1874 #  define XXH_SIZE_OPT 0
1875 
1876 /*!
1877  * @def XXH_FORCE_ALIGN_CHECK
1878  * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
1879  * and XXH64() only).
1880  *
1881  * This is an important performance trick for architectures without decent
1882  * unaligned memory access performance.
1883  *
1884  * It checks for input alignment, and when conditions are met, uses a "fast
1885  * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
1886  * faster_ read speed.
1887  *
1888  * The check costs one initial branch per hash, which is generally negligible,
1889  * but not zero.
1890  *
1891  * Moreover, it's not useful to generate an additional code path if memory
1892  * access uses the same instruction for both aligned and unaligned
1893  * addresses (e.g. x86 and aarch64).
1894  *
1895  * In these cases, the alignment check can be removed by setting this macro to 0.
1896  * Then the code will always use unaligned memory access.
1897  * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips
1898  * which are platforms known to offer good unaligned memory accesses performance.
1899  *
1900  * It is also disabled by default when @ref XXH_SIZE_OPT >= 1.
1901  *
1902  * This option does not affect XXH3 (only XXH32 and XXH64).
1903  */
1904 #  define XXH_FORCE_ALIGN_CHECK 0
1905 
1906 /*!
1907  * @def XXH_NO_INLINE_HINTS
1908  * @brief When non-zero, sets all functions to `static`.
1909  *
1910  * By default, xxHash tries to force the compiler to inline almost all internal
1911  * functions.
1912  *
1913  * This can usually improve performance due to reduced jumping and improved
1914  * constant folding, but significantly increases the size of the binary which
1915  * might not be favorable.
1916  *
1917  * Additionally, sometimes the forced inlining can be detrimental to performance,
1918  * depending on the architecture.
1919  *
1920  * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
1921  * compiler full control on whether to inline or not.
1922  *
1923  * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if
1924  * @ref XXH_SIZE_OPT >= 1, this will automatically be defined.
1925  */
1926 #  define XXH_NO_INLINE_HINTS 0
1927 
1928 /*!
1929  * @def XXH3_INLINE_SECRET
1930  * @brief Determines whether to inline the XXH3 withSecret code.
1931  *
1932  * When the secret size is known, the compiler can improve the performance
1933  * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret().
1934  *
1935  * However, if the secret size is not known, it doesn't have any benefit. This
1936  * happens when xxHash is compiled into a global symbol. Therefore, if
1937  * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0.
1938  *
1939  * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers
1940  * that are *sometimes* force inline on -Og, and it is impossible to automatically
1941  * detect this optimization level.
1942  */
1943 #  define XXH3_INLINE_SECRET 0
1944 
1945 /*!
1946  * @def XXH32_ENDJMP
1947  * @brief Whether to use a jump for `XXH32_finalize`.
1948  *
1949  * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
1950  * This is generally preferable for performance,
1951  * but depending on exact architecture, a jmp may be preferable.
1952  *
1953  * This setting is only possibly making a difference for very small inputs.
1954  */
1955 #  define XXH32_ENDJMP 0
1956 
1957 /*!
1958  * @internal
1959  * @brief Redefines old internal names.
1960  *
1961  * For compatibility with code that uses xxHash's internals before the names
1962  * were changed to improve namespacing. There is no other reason to use this.
1963  */
1964 #  define XXH_OLD_NAMES
1965 #  undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
1966 
1967 /*!
1968  * @def XXH_NO_STREAM
1969  * @brief Disables the streaming API.
1970  *
1971  * When xxHash is not inlined and the streaming functions are not used, disabling
1972  * the streaming functions can improve code size significantly, especially with
1973  * the @ref XXH3_family which tends to make constant folded copies of itself.
1974  */
1975 #  define XXH_NO_STREAM
1976 #  undef XXH_NO_STREAM /* don't actually */
1977 #endif /* XXH_DOXYGEN */
1978 /*!
1979  * @}
1980  */
1981 
1982 #ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
1983    /* prefer __packed__ structures (method 1) for GCC
1984     * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy
1985     * which for some reason does unaligned loads. */
1986 #  if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED))
1987 #    define XXH_FORCE_MEMORY_ACCESS 1
1988 #  endif
1989 #endif
1990 
1991 #ifndef XXH_SIZE_OPT
1992    /* default to 1 for -Os or -Oz */
1993 #  if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__)
1994 #    define XXH_SIZE_OPT 1
1995 #  else
1996 #    define XXH_SIZE_OPT 0
1997 #  endif
1998 #endif
1999 
2000 #ifndef XXH_FORCE_ALIGN_CHECK  /* can be defined externally */
2001    /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */
2002 #  if XXH_SIZE_OPT >= 1 || \
2003       defined(__i386)  || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \
2004    || defined(_M_IX86) || defined(_M_X64)     || defined(_M_ARM64)    || defined(_M_ARM) /* visual */
2005 #    define XXH_FORCE_ALIGN_CHECK 0
2006 #  else
2007 #    define XXH_FORCE_ALIGN_CHECK 1
2008 #  endif
2009 #endif
2010 
2011 #ifndef XXH_NO_INLINE_HINTS
2012 #  if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__)  /* -O0, -fno-inline */
2013 #    define XXH_NO_INLINE_HINTS 1
2014 #  else
2015 #    define XXH_NO_INLINE_HINTS 0
2016 #  endif
2017 #endif
2018 
2019 #ifndef XXH3_INLINE_SECRET
2020 #  if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \
2021      || !defined(XXH_INLINE_ALL)
2022 #    define XXH3_INLINE_SECRET 0
2023 #  else
2024 #    define XXH3_INLINE_SECRET 1
2025 #  endif
2026 #endif
2027 
2028 #ifndef XXH32_ENDJMP
2029 /* generally preferable for performance */
2030 #  define XXH32_ENDJMP 0
2031 #endif
2032 
2033 /*!
2034  * @defgroup impl Implementation
2035  * @{
2036  */
2037 
2038 
2039 /* *************************************
2040 *  Includes & Memory related functions
2041 ***************************************/
2042 #if defined(XXH_NO_STREAM)
2043 /* nothing */
2044 #elif defined(XXH_NO_STDLIB)
2045 
2046 /* When requesting to disable any mention of stdlib,
2047  * the library loses the ability to invoked malloc / free.
2048  * In practice, it means that functions like `XXH*_createState()`
2049  * will always fail, and return NULL.
2050  * This flag is useful in situations where
2051  * xxhash.h is integrated into some kernel, embedded or limited environment
2052  * without access to dynamic allocation.
2053  */
2054 
2055 static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
2056 static void XXH_free(void* p) { (void)p; }
2057 
2058 #else
2059 
2060 /*
2061  * Modify the local functions below should you wish to use
2062  * different memory routines for malloc() and free()
2063  */
2064 #include <stdlib.h>
2065 
2066 /*!
2067  * @internal
2068  * @brief Modify this function to use a different routine than malloc().
2069  */
2070 static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }
2071 
2072 /*!
2073  * @internal
2074  * @brief Modify this function to use a different routine than free().
2075  */
2076 static void XXH_free(void* p) { free(p); }
2077 
2078 #endif  /* XXH_NO_STDLIB */
2079 
2080 #include <string.h>
2081 
2082 /*!
2083  * @internal
2084  * @brief Modify this function to use a different routine than memcpy().
2085  */
2086 static void* XXH_memcpy(void* dest, const void* src, size_t size)
2087 {
2088     return memcpy(dest,src,size);
2089 }
2090 
2091 #include <limits.h>   /* ULLONG_MAX */
2092 
2093 
2094 /* *************************************
2095 *  Compiler Specific Options
2096 ***************************************/
2097 #ifdef _MSC_VER /* Visual Studio warning fix */
2098 #  pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
2099 #endif
2100 
2101 #if XXH_NO_INLINE_HINTS  /* disable inlining hints */
2102 #  if defined(__GNUC__) || defined(__clang__)
2103 #    define XXH_FORCE_INLINE static __attribute__((unused))
2104 #  else
2105 #    define XXH_FORCE_INLINE static
2106 #  endif
2107 #  define XXH_NO_INLINE static
2108 /* enable inlining hints */
2109 #elif defined(__GNUC__) || defined(__clang__)
2110 #  define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
2111 #  define XXH_NO_INLINE static __attribute__((noinline))
2112 #elif defined(_MSC_VER)  /* Visual Studio */
2113 #  define XXH_FORCE_INLINE static __forceinline
2114 #  define XXH_NO_INLINE static __declspec(noinline)
2115 #elif defined (__cplusplus) \
2116   || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))   /* C99 */
2117 #  define XXH_FORCE_INLINE static inline
2118 #  define XXH_NO_INLINE static
2119 #else
2120 #  define XXH_FORCE_INLINE static
2121 #  define XXH_NO_INLINE static
2122 #endif
2123 
2124 #if XXH3_INLINE_SECRET
2125 #  define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE
2126 #else
2127 #  define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
2128 #endif
2129 
2130 
2131 /* *************************************
2132 *  Debug
2133 ***************************************/
2134 /*!
2135  * @ingroup tuning
2136  * @def XXH_DEBUGLEVEL
2137  * @brief Sets the debugging level.
2138  *
2139  * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
2140  * compiler's command line options. The value must be a number.
2141  */
2142 #ifndef XXH_DEBUGLEVEL
2143 #  ifdef DEBUGLEVEL /* backwards compat */
2144 #    define XXH_DEBUGLEVEL DEBUGLEVEL
2145 #  else
2146 #    define XXH_DEBUGLEVEL 0
2147 #  endif
2148 #endif
2149 
2150 #if (XXH_DEBUGLEVEL>=1)
2151 #  include <assert.h>   /* note: can still be disabled with NDEBUG */
2152 #  define XXH_ASSERT(c)   assert(c)
2153 #else
2154 #  if defined(__INTEL_COMPILER)
2155 #    define XXH_ASSERT(c)   XXH_ASSUME((unsigned char) (c))
2156 #  else
2157 #    define XXH_ASSERT(c)   XXH_ASSUME(c)
2158 #  endif
2159 #endif
2160 
2161 /* note: use after variable declarations */
2162 #ifndef XXH_STATIC_ASSERT
2163 #  if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)    /* C11 */
2164 #    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
2165 #  elif defined(__cplusplus) && (__cplusplus >= 201103L)            /* C++11 */
2166 #    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
2167 #  else
2168 #    define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
2169 #  endif
2170 #  define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
2171 #endif
2172 
2173 /*!
2174  * @internal
2175  * @def XXH_COMPILER_GUARD(var)
2176  * @brief Used to prevent unwanted optimizations for @p var.
2177  *
2178  * It uses an empty GCC inline assembly statement with a register constraint
2179  * which forces @p var into a general purpose register (eg eax, ebx, ecx
2180  * on x86) and marks it as modified.
2181  *
2182  * This is used in a few places to avoid unwanted autovectorization (e.g.
2183  * XXH32_round()). All vectorization we want is explicit via intrinsics,
2184  * and _usually_ isn't wanted elsewhere.
2185  *
2186  * We also use it to prevent unwanted constant folding for AArch64 in
2187  * XXH3_initCustomSecret_scalar().
2188  */
2189 #if defined(__GNUC__) || defined(__clang__)
2190 #  define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var))
2191 #else
2192 #  define XXH_COMPILER_GUARD(var) ((void)0)
2193 #endif
2194 
2195 /* Specifically for NEON vectors which use the "w" constraint, on
2196  * Clang. */
2197 #if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__)
2198 #  define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var))
2199 #else
2200 #  define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0)
2201 #endif
2202 
2203 /* *************************************
2204 *  Basic Types
2205 ***************************************/
2206 #if !defined (__VMS) \
2207  && (defined (__cplusplus) \
2208  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
2209 # include <stdint.h>
2210   typedef uint8_t xxh_u8;
2211 #else
2212   typedef unsigned char xxh_u8;
2213 #endif
2214 typedef XXH32_hash_t xxh_u32;
2215 
2216 #ifdef XXH_OLD_NAMES
2217 #  warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly"
2218 #  define BYTE xxh_u8
2219 #  define U8   xxh_u8
2220 #  define U32  xxh_u32
2221 #endif
2222 
2223 /* ***   Memory access   *** */
2224 
2225 /*!
2226  * @internal
2227  * @fn xxh_u32 XXH_read32(const void* ptr)
2228  * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
2229  *
2230  * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2231  *
2232  * @param ptr The pointer to read from.
2233  * @return The 32-bit native endian integer from the bytes at @p ptr.
2234  */
2235 
2236 /*!
2237  * @internal
2238  * @fn xxh_u32 XXH_readLE32(const void* ptr)
2239  * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
2240  *
2241  * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2242  *
2243  * @param ptr The pointer to read from.
2244  * @return The 32-bit little endian integer from the bytes at @p ptr.
2245  */
2246 
2247 /*!
2248  * @internal
2249  * @fn xxh_u32 XXH_readBE32(const void* ptr)
2250  * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
2251  *
2252  * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2253  *
2254  * @param ptr The pointer to read from.
2255  * @return The 32-bit big endian integer from the bytes at @p ptr.
2256  */
2257 
2258 /*!
2259  * @internal
2260  * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
2261  * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
2262  *
2263  * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2264  * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
2265  * always @ref XXH_alignment::XXH_unaligned.
2266  *
2267  * @param ptr The pointer to read from.
2268  * @param align Whether @p ptr is aligned.
2269  * @pre
2270  *   If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
2271  *   aligned.
2272  * @return The 32-bit little endian integer from the bytes at @p ptr.
2273  */
2274 
2275 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2276 /*
2277  * Manual byteshift. Best for old compilers which don't inline memcpy.
2278  * We actually directly use XXH_readLE32 and XXH_readBE32.
2279  */
2280 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
2281 
2282 /*
2283  * Force direct memory access. Only works on CPU which support unaligned memory
2284  * access in hardware.
2285  */
2286 static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
2287 
2288 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
2289 
2290 /*
2291  * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
2292  * documentation claimed that it only increased the alignment, but actually it
2293  * can decrease it on gcc, clang, and icc:
2294  * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
2295  * https://gcc.godbolt.org/z/xYez1j67Y.
2296  */
2297 #ifdef XXH_OLD_NAMES
2298 typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
2299 #endif
2300 static xxh_u32 XXH_read32(const void* ptr)
2301 {
2302     typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32;
2303     return *((const xxh_unalign32*)ptr);
2304 }
2305 
2306 #else
2307 
2308 /*
2309  * Portable and safe solution. Generally efficient.
2310  * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
2311  */
2312 static xxh_u32 XXH_read32(const void* memPtr)
2313 {
2314     xxh_u32 val;
2315     XXH_memcpy(&val, memPtr, sizeof(val));
2316     return val;
2317 }
2318 
2319 #endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
2320 
2321 
2322 /* ***   Endianness   *** */
2323 
2324 /*!
2325  * @ingroup tuning
2326  * @def XXH_CPU_LITTLE_ENDIAN
2327  * @brief Whether the target is little endian.
2328  *
2329  * Defined to 1 if the target is little endian, or 0 if it is big endian.
2330  * It can be defined externally, for example on the compiler command line.
2331  *
2332  * If it is not defined,
2333  * a runtime check (which is usually constant folded) is used instead.
2334  *
2335  * @note
2336  *   This is not necessarily defined to an integer constant.
2337  *
2338  * @see XXH_isLittleEndian() for the runtime check.
2339  */
2340 #ifndef XXH_CPU_LITTLE_ENDIAN
2341 /*
2342  * Try to detect endianness automatically, to avoid the nonstandard behavior
2343  * in `XXH_isLittleEndian()`
2344  */
2345 #  if defined(_WIN32) /* Windows is always little endian */ \
2346      || defined(__LITTLE_ENDIAN__) \
2347      || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
2348 #    define XXH_CPU_LITTLE_ENDIAN 1
2349 #  elif defined(__BIG_ENDIAN__) \
2350      || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2351 #    define XXH_CPU_LITTLE_ENDIAN 0
2352 #  else
2353 /*!
2354  * @internal
2355  * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
2356  *
2357  * Most compilers will constant fold this.
2358  */
2359 static int XXH_isLittleEndian(void)
2360 {
2361     /*
2362      * Portable and well-defined behavior.
2363      * Don't use static: it is detrimental to performance.
2364      */
2365     const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
2366     return one.c[0];
2367 }
2368 #   define XXH_CPU_LITTLE_ENDIAN   XXH_isLittleEndian()
2369 #  endif
2370 #endif
2371 
2372 
2373 
2374 
2375 /* ****************************************
2376 *  Compiler-specific Functions and Macros
2377 ******************************************/
2378 #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
2379 
2380 #ifdef __has_builtin
2381 #  define XXH_HAS_BUILTIN(x) __has_builtin(x)
2382 #else
2383 #  define XXH_HAS_BUILTIN(x) 0
2384 #endif
2385 
2386 
2387 
2388 /*
2389  * C23 and future versions have standard "unreachable()".
2390  * Once it has been implemented reliably we can add it as an
2391  * additional case:
2392  *
2393  * ```
2394  * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN)
2395  * #  include <stddef.h>
2396  * #  ifdef unreachable
2397  * #    define XXH_UNREACHABLE() unreachable()
2398  * #  endif
2399  * #endif
2400  * ```
2401  *
2402  * Note C++23 also has std::unreachable() which can be detected
2403  * as follows:
2404  * ```
2405  * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L)
2406  * #  include <utility>
2407  * #  define XXH_UNREACHABLE() std::unreachable()
2408  * #endif
2409  * ```
2410  * NB: `__cpp_lib_unreachable` is defined in the `<version>` header.
2411  * We don't use that as including `<utility>` in `extern "C"` blocks
2412  * doesn't work on GCC12
2413  */
2414 
2415 #if XXH_HAS_BUILTIN(__builtin_unreachable)
2416 #  define XXH_UNREACHABLE() __builtin_unreachable()
2417 
2418 #elif defined(_MSC_VER)
2419 #  define XXH_UNREACHABLE() __assume(0)
2420 
2421 #else
2422 #  define XXH_UNREACHABLE()
2423 #endif
2424 
2425 #if XXH_HAS_BUILTIN(__builtin_assume)
2426 #  define XXH_ASSUME(c) __builtin_assume(c)
2427 #else
2428 #  define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); }
2429 #endif
2430 
2431 /*!
2432  * @internal
2433  * @def XXH_rotl32(x,r)
2434  * @brief 32-bit rotate left.
2435  *
2436  * @param x The 32-bit integer to be rotated.
2437  * @param r The number of bits to rotate.
2438  * @pre
2439  *   @p r > 0 && @p r < 32
2440  * @note
2441  *   @p x and @p r may be evaluated multiple times.
2442  * @return The rotated result.
2443  */
2444 #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
2445                                && XXH_HAS_BUILTIN(__builtin_rotateleft64)
2446 #  define XXH_rotl32 __builtin_rotateleft32
2447 #  define XXH_rotl64 __builtin_rotateleft64
2448 /* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
2449 #elif defined(_MSC_VER)
2450 #  define XXH_rotl32(x,r) _rotl(x,r)
2451 #  define XXH_rotl64(x,r) _rotl64(x,r)
2452 #else
2453 #  define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
2454 #  define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
2455 #endif
2456 
2457 /*!
2458  * @internal
2459  * @fn xxh_u32 XXH_swap32(xxh_u32 x)
2460  * @brief A 32-bit byteswap.
2461  *
2462  * @param x The 32-bit integer to byteswap.
2463  * @return @p x, byteswapped.
2464  */
2465 #if defined(_MSC_VER)     /* Visual Studio */
2466 #  define XXH_swap32 _byteswap_ulong
2467 #elif XXH_GCC_VERSION >= 403
2468 #  define XXH_swap32 __builtin_bswap32
2469 #else
2470 static xxh_u32 XXH_swap32 (xxh_u32 x)
2471 {
2472     return  ((x << 24) & 0xff000000 ) |
2473             ((x <<  8) & 0x00ff0000 ) |
2474             ((x >>  8) & 0x0000ff00 ) |
2475             ((x >> 24) & 0x000000ff );
2476 }
2477 #endif
2478 
2479 
2480 /* ***************************
2481 *  Memory reads
2482 *****************************/
2483 
2484 /*!
2485  * @internal
2486  * @brief Enum to indicate whether a pointer is aligned.
2487  */
2488 typedef enum {
2489     XXH_aligned,  /*!< Aligned */
2490     XXH_unaligned /*!< Possibly unaligned */
2491 } XXH_alignment;
2492 
2493 /*
2494  * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
2495  *
2496  * This is ideal for older compilers which don't inline memcpy.
2497  */
2498 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2499 
2500 XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
2501 {
2502     const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2503     return bytePtr[0]
2504          | ((xxh_u32)bytePtr[1] << 8)
2505          | ((xxh_u32)bytePtr[2] << 16)
2506          | ((xxh_u32)bytePtr[3] << 24);
2507 }
2508 
2509 XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
2510 {
2511     const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2512     return bytePtr[3]
2513          | ((xxh_u32)bytePtr[2] << 8)
2514          | ((xxh_u32)bytePtr[1] << 16)
2515          | ((xxh_u32)bytePtr[0] << 24);
2516 }
2517 
2518 #else
2519 XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
2520 {
2521     return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
2522 }
2523 
2524 static xxh_u32 XXH_readBE32(const void* ptr)
2525 {
2526     return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
2527 }
2528 #endif
2529 
2530 XXH_FORCE_INLINE xxh_u32
2531 XXH_readLE32_align(const void* ptr, XXH_alignment align)
2532 {
2533     if (align==XXH_unaligned) {
2534         return XXH_readLE32(ptr);
2535     } else {
2536         return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
2537     }
2538 }
2539 
2540 
2541 /* *************************************
2542 *  Misc
2543 ***************************************/
2544 /*! @ingroup public */
2545 XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
2546 
2547 
2548 /* *******************************************************************
2549 *  32-bit hash functions
2550 *********************************************************************/
2551 /*!
2552  * @}
2553  * @defgroup XXH32_impl XXH32 implementation
2554  * @ingroup impl
2555  *
2556  * Details on the XXH32 implementation.
2557  * @{
2558  */
2559  /* #define instead of static const, to be used as initializers */
2560 #define XXH_PRIME32_1  0x9E3779B1U  /*!< 0b10011110001101110111100110110001 */
2561 #define XXH_PRIME32_2  0x85EBCA77U  /*!< 0b10000101111010111100101001110111 */
2562 #define XXH_PRIME32_3  0xC2B2AE3DU  /*!< 0b11000010101100101010111000111101 */
2563 #define XXH_PRIME32_4  0x27D4EB2FU  /*!< 0b00100111110101001110101100101111 */
2564 #define XXH_PRIME32_5  0x165667B1U  /*!< 0b00010110010101100110011110110001 */
2565 
2566 #ifdef XXH_OLD_NAMES
2567 #  define PRIME32_1 XXH_PRIME32_1
2568 #  define PRIME32_2 XXH_PRIME32_2
2569 #  define PRIME32_3 XXH_PRIME32_3
2570 #  define PRIME32_4 XXH_PRIME32_4
2571 #  define PRIME32_5 XXH_PRIME32_5
2572 #endif
2573 
2574 /*!
2575  * @internal
2576  * @brief Normal stripe processing routine.
2577  *
2578  * This shuffles the bits so that any bit from @p input impacts several bits in
2579  * @p acc.
2580  *
2581  * @param acc The accumulator lane.
2582  * @param input The stripe of input to mix.
2583  * @return The mixed accumulator lane.
2584  */
2585 static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
2586 {
2587     acc += input * XXH_PRIME32_2;
2588     acc  = XXH_rotl32(acc, 13);
2589     acc *= XXH_PRIME32_1;
2590 #if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
2591     /*
2592      * UGLY HACK:
2593      * A compiler fence is the only thing that prevents GCC and Clang from
2594      * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
2595      * reason) without globally disabling SSE4.1.
2596      *
2597      * The reason we want to avoid vectorization is because despite working on
2598      * 4 integers at a time, there are multiple factors slowing XXH32 down on
2599      * SSE4:
2600      * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
2601      *   newer chips!) making it slightly slower to multiply four integers at
2602      *   once compared to four integers independently. Even when pmulld was
2603      *   fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
2604      *   just to multiply unless doing a long operation.
2605      *
2606      * - Four instructions are required to rotate,
2607      *      movqda tmp,  v // not required with VEX encoding
2608      *      pslld  tmp, 13 // tmp <<= 13
2609      *      psrld  v,   19 // x >>= 19
2610      *      por    v,  tmp // x |= tmp
2611      *   compared to one for scalar:
2612      *      roll   v, 13    // reliably fast across the board
2613      *      shldl  v, v, 13 // Sandy Bridge and later prefer this for some reason
2614      *
2615      * - Instruction level parallelism is actually more beneficial here because
2616      *   the SIMD actually serializes this operation: While v1 is rotating, v2
2617      *   can load data, while v3 can multiply. SSE forces them to operate
2618      *   together.
2619      *
2620      * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing
2621      * the loop. NEON is only faster on the A53, and with the newer cores, it is less
2622      * than half the speed.
2623      *
2624      * Additionally, this is used on WASM SIMD128 because it JITs to the same
2625      * SIMD instructions and has the same issue.
2626      */
2627     XXH_COMPILER_GUARD(acc);
2628 #endif
2629     return acc;
2630 }
2631 
2632 /*!
2633  * @internal
2634  * @brief Mixes all bits to finalize the hash.
2635  *
2636  * The final mix ensures that all input bits have a chance to impact any bit in
2637  * the output digest, resulting in an unbiased distribution.
2638  *
2639  * @param hash The hash to avalanche.
2640  * @return The avalanched hash.
2641  */
2642 static xxh_u32 XXH32_avalanche(xxh_u32 hash)
2643 {
2644     hash ^= hash >> 15;
2645     hash *= XXH_PRIME32_2;
2646     hash ^= hash >> 13;
2647     hash *= XXH_PRIME32_3;
2648     hash ^= hash >> 16;
2649     return hash;
2650 }
2651 
2652 #define XXH_get32bits(p) XXH_readLE32_align(p, align)
2653 
2654 /*!
2655  * @internal
2656  * @brief Processes the last 0-15 bytes of @p ptr.
2657  *
2658  * There may be up to 15 bytes remaining to consume from the input.
2659  * This final stage will digest them to ensure that all input bytes are present
2660  * in the final mix.
2661  *
2662  * @param hash The hash to finalize.
2663  * @param ptr The pointer to the remaining input.
2664  * @param len The remaining length, modulo 16.
2665  * @param align Whether @p ptr is aligned.
2666  * @return The finalized hash.
2667  * @see XXH64_finalize().
2668  */
2669 static XXH_PUREF xxh_u32
2670 XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
2671 {
2672 #define XXH_PROCESS1 do {                             \
2673     hash += (*ptr++) * XXH_PRIME32_5;                 \
2674     hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1;      \
2675 } while (0)
2676 
2677 #define XXH_PROCESS4 do {                             \
2678     hash += XXH_get32bits(ptr) * XXH_PRIME32_3;       \
2679     ptr += 4;                                         \
2680     hash  = XXH_rotl32(hash, 17) * XXH_PRIME32_4;     \
2681 } while (0)
2682 
2683     if (ptr==NULL) XXH_ASSERT(len == 0);
2684 
2685     /* Compact rerolled version; generally faster */
2686     if (!XXH32_ENDJMP) {
2687         len &= 15;
2688         while (len >= 4) {
2689             XXH_PROCESS4;
2690             len -= 4;
2691         }
2692         while (len > 0) {
2693             XXH_PROCESS1;
2694             --len;
2695         }
2696         return XXH32_avalanche(hash);
2697     } else {
2698          switch(len&15) /* or switch(bEnd - p) */ {
2699            case 12:      XXH_PROCESS4;
2700                          XXH_FALLTHROUGH;  /* fallthrough */
2701            case 8:       XXH_PROCESS4;
2702                          XXH_FALLTHROUGH;  /* fallthrough */
2703            case 4:       XXH_PROCESS4;
2704                          return XXH32_avalanche(hash);
2705 
2706            case 13:      XXH_PROCESS4;
2707                          XXH_FALLTHROUGH;  /* fallthrough */
2708            case 9:       XXH_PROCESS4;
2709                          XXH_FALLTHROUGH;  /* fallthrough */
2710            case 5:       XXH_PROCESS4;
2711                          XXH_PROCESS1;
2712                          return XXH32_avalanche(hash);
2713 
2714            case 14:      XXH_PROCESS4;
2715                          XXH_FALLTHROUGH;  /* fallthrough */
2716            case 10:      XXH_PROCESS4;
2717                          XXH_FALLTHROUGH;  /* fallthrough */
2718            case 6:       XXH_PROCESS4;
2719                          XXH_PROCESS1;
2720                          XXH_PROCESS1;
2721                          return XXH32_avalanche(hash);
2722 
2723            case 15:      XXH_PROCESS4;
2724                          XXH_FALLTHROUGH;  /* fallthrough */
2725            case 11:      XXH_PROCESS4;
2726                          XXH_FALLTHROUGH;  /* fallthrough */
2727            case 7:       XXH_PROCESS4;
2728                          XXH_FALLTHROUGH;  /* fallthrough */
2729            case 3:       XXH_PROCESS1;
2730                          XXH_FALLTHROUGH;  /* fallthrough */
2731            case 2:       XXH_PROCESS1;
2732                          XXH_FALLTHROUGH;  /* fallthrough */
2733            case 1:       XXH_PROCESS1;
2734                          XXH_FALLTHROUGH;  /* fallthrough */
2735            case 0:       return XXH32_avalanche(hash);
2736         }
2737         XXH_ASSERT(0);
2738         return hash;   /* reaching this point is deemed impossible */
2739     }
2740 }
2741 
2742 #ifdef XXH_OLD_NAMES
2743 #  define PROCESS1 XXH_PROCESS1
2744 #  define PROCESS4 XXH_PROCESS4
2745 #else
2746 #  undef XXH_PROCESS1
2747 #  undef XXH_PROCESS4
2748 #endif
2749 
2750 /*!
2751  * @internal
2752  * @brief The implementation for @ref XXH32().
2753  *
2754  * @param input , len , seed Directly passed from @ref XXH32().
2755  * @param align Whether @p input is aligned.
2756  * @return The calculated hash.
2757  */
2758 XXH_FORCE_INLINE XXH_PUREF xxh_u32
2759 XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
2760 {
2761     xxh_u32 h32;
2762 
2763     if (input==NULL) XXH_ASSERT(len == 0);
2764 
2765     if (len>=16) {
2766         const xxh_u8* const bEnd = input + len;
2767         const xxh_u8* const limit = bEnd - 15;
2768         xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
2769         xxh_u32 v2 = seed + XXH_PRIME32_2;
2770         xxh_u32 v3 = seed + 0;
2771         xxh_u32 v4 = seed - XXH_PRIME32_1;
2772 
2773         do {
2774             v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
2775             v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
2776             v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
2777             v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
2778         } while (input < limit);
2779 
2780         h32 = XXH_rotl32(v1, 1)  + XXH_rotl32(v2, 7)
2781             + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
2782     } else {
2783         h32  = seed + XXH_PRIME32_5;
2784     }
2785 
2786     h32 += (xxh_u32)len;
2787 
2788     return XXH32_finalize(h32, input, len&15, align);
2789 }
2790 
2791 /*! @ingroup XXH32_family */
2792 XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
2793 {
2794 #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
2795     /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
2796     XXH32_state_t state;
2797     XXH32_reset(&state, seed);
2798     XXH32_update(&state, (const xxh_u8*)input, len);
2799     return XXH32_digest(&state);
2800 #else
2801     if (XXH_FORCE_ALIGN_CHECK) {
2802         if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
2803             return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
2804     }   }
2805 
2806     return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
2807 #endif
2808 }
2809 
2810 
2811 
2812 /*******   Hash streaming   *******/
2813 #ifndef XXH_NO_STREAM
2814 /*! @ingroup XXH32_family */
2815 XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
2816 {
2817     return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
2818 }
2819 /*! @ingroup XXH32_family */
2820 XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
2821 {
2822     XXH_free(statePtr);
2823     return XXH_OK;
2824 }
2825 
2826 /*! @ingroup XXH32_family */
2827 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
2828 {
2829     XXH_memcpy(dstState, srcState, sizeof(*dstState));
2830 }
2831 
2832 /*! @ingroup XXH32_family */
2833 XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
2834 {
2835     XXH_ASSERT(statePtr != NULL);
2836     memset(statePtr, 0, sizeof(*statePtr));
2837     statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
2838     statePtr->v[1] = seed + XXH_PRIME32_2;
2839     statePtr->v[2] = seed + 0;
2840     statePtr->v[3] = seed - XXH_PRIME32_1;
2841     return XXH_OK;
2842 }
2843 
2844 
2845 /*! @ingroup XXH32_family */
2846 XXH_PUBLIC_API XXH_errorcode
2847 XXH32_update(XXH32_state_t* state, const void* input, size_t len)
2848 {
2849     if (input==NULL) {
2850         XXH_ASSERT(len == 0);
2851         return XXH_OK;
2852     }
2853 
2854     {   const xxh_u8* p = (const xxh_u8*)input;
2855         const xxh_u8* const bEnd = p + len;
2856 
2857         state->total_len_32 += (XXH32_hash_t)len;
2858         state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
2859 
2860         if (state->memsize + len < 16)  {   /* fill in tmp buffer */
2861             XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
2862             state->memsize += (XXH32_hash_t)len;
2863             return XXH_OK;
2864         }
2865 
2866         if (state->memsize) {   /* some data left from previous update */
2867             XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
2868             {   const xxh_u32* p32 = state->mem32;
2869                 state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
2870                 state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
2871                 state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
2872                 state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
2873             }
2874             p += 16-state->memsize;
2875             state->memsize = 0;
2876         }
2877 
2878         if (p <= bEnd-16) {
2879             const xxh_u8* const limit = bEnd - 16;
2880 
2881             do {
2882                 state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
2883                 state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
2884                 state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
2885                 state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
2886             } while (p<=limit);
2887 
2888         }
2889 
2890         if (p < bEnd) {
2891             XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
2892             state->memsize = (unsigned)(bEnd-p);
2893         }
2894     }
2895 
2896     return XXH_OK;
2897 }
2898 
2899 
2900 /*! @ingroup XXH32_family */
2901 XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
2902 {
2903     xxh_u32 h32;
2904 
2905     if (state->large_len) {
2906         h32 = XXH_rotl32(state->v[0], 1)
2907             + XXH_rotl32(state->v[1], 7)
2908             + XXH_rotl32(state->v[2], 12)
2909             + XXH_rotl32(state->v[3], 18);
2910     } else {
2911         h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
2912     }
2913 
2914     h32 += state->total_len_32;
2915 
2916     return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
2917 }
2918 #endif /* !XXH_NO_STREAM */
2919 
2920 /*******   Canonical representation   *******/
2921 
2922 /*!
2923  * @ingroup XXH32_family
2924  * The default return values from XXH functions are unsigned 32 and 64 bit
2925  * integers.
2926  *
2927  * The canonical representation uses big endian convention, the same convention
2928  * as human-readable numbers (large digits first).
2929  *
2930  * This way, hash values can be written into a file or buffer, remaining
2931  * comparable across different systems.
2932  *
2933  * The following functions allow transformation of hash values to and from their
2934  * canonical format.
2935  */
2936 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
2937 {
2938     XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
2939     if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
2940     XXH_memcpy(dst, &hash, sizeof(*dst));
2941 }
2942 /*! @ingroup XXH32_family */
2943 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
2944 {
2945     return XXH_readBE32(src);
2946 }
2947 
2948 
2949 #ifndef XXH_NO_LONG_LONG
2950 
2951 /* *******************************************************************
2952 *  64-bit hash functions
2953 *********************************************************************/
2954 /*!
2955  * @}
2956  * @ingroup impl
2957  * @{
2958  */
2959 /*******   Memory access   *******/
2960 
2961 typedef XXH64_hash_t xxh_u64;
2962 
2963 #ifdef XXH_OLD_NAMES
2964 #  define U64 xxh_u64
2965 #endif
2966 
2967 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2968 /*
2969  * Manual byteshift. Best for old compilers which don't inline memcpy.
2970  * We actually directly use XXH_readLE64 and XXH_readBE64.
2971  */
2972 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
2973 
2974 /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
2975 static xxh_u64 XXH_read64(const void* memPtr)
2976 {
2977     return *(const xxh_u64*) memPtr;
2978 }
2979 
2980 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
2981 
2982 /*
2983  * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
2984  * documentation claimed that it only increased the alignment, but actually it
2985  * can decrease it on gcc, clang, and icc:
2986  * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
2987  * https://gcc.godbolt.org/z/xYez1j67Y.
2988  */
2989 #ifdef XXH_OLD_NAMES
2990 typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
2991 #endif
2992 static xxh_u64 XXH_read64(const void* ptr)
2993 {
2994     typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64;
2995     return *((const xxh_unalign64*)ptr);
2996 }
2997 
2998 #else
2999 
3000 /*
3001  * Portable and safe solution. Generally efficient.
3002  * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
3003  */
3004 static xxh_u64 XXH_read64(const void* memPtr)
3005 {
3006     xxh_u64 val;
3007     XXH_memcpy(&val, memPtr, sizeof(val));
3008     return val;
3009 }
3010 
3011 #endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
3012 
3013 #if defined(_MSC_VER)     /* Visual Studio */
3014 #  define XXH_swap64 _byteswap_uint64
3015 #elif XXH_GCC_VERSION >= 403
3016 #  define XXH_swap64 __builtin_bswap64
3017 #else
3018 static xxh_u64 XXH_swap64(xxh_u64 x)
3019 {
3020     return  ((x << 56) & 0xff00000000000000ULL) |
3021             ((x << 40) & 0x00ff000000000000ULL) |
3022             ((x << 24) & 0x0000ff0000000000ULL) |
3023             ((x << 8)  & 0x000000ff00000000ULL) |
3024             ((x >> 8)  & 0x00000000ff000000ULL) |
3025             ((x >> 24) & 0x0000000000ff0000ULL) |
3026             ((x >> 40) & 0x000000000000ff00ULL) |
3027             ((x >> 56) & 0x00000000000000ffULL);
3028 }
3029 #endif
3030 
3031 
3032 /* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
3033 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
3034 
3035 XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
3036 {
3037     const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3038     return bytePtr[0]
3039          | ((xxh_u64)bytePtr[1] << 8)
3040          | ((xxh_u64)bytePtr[2] << 16)
3041          | ((xxh_u64)bytePtr[3] << 24)
3042          | ((xxh_u64)bytePtr[4] << 32)
3043          | ((xxh_u64)bytePtr[5] << 40)
3044          | ((xxh_u64)bytePtr[6] << 48)
3045          | ((xxh_u64)bytePtr[7] << 56);
3046 }
3047 
3048 XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
3049 {
3050     const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3051     return bytePtr[7]
3052          | ((xxh_u64)bytePtr[6] << 8)
3053          | ((xxh_u64)bytePtr[5] << 16)
3054          | ((xxh_u64)bytePtr[4] << 24)
3055          | ((xxh_u64)bytePtr[3] << 32)
3056          | ((xxh_u64)bytePtr[2] << 40)
3057          | ((xxh_u64)bytePtr[1] << 48)
3058          | ((xxh_u64)bytePtr[0] << 56);
3059 }
3060 
3061 #else
3062 XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
3063 {
3064     return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
3065 }
3066 
3067 static xxh_u64 XXH_readBE64(const void* ptr)
3068 {
3069     return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
3070 }
3071 #endif
3072 
3073 XXH_FORCE_INLINE xxh_u64
3074 XXH_readLE64_align(const void* ptr, XXH_alignment align)
3075 {
3076     if (align==XXH_unaligned)
3077         return XXH_readLE64(ptr);
3078     else
3079         return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
3080 }
3081 
3082 
3083 /*******   xxh64   *******/
3084 /*!
3085  * @}
3086  * @defgroup XXH64_impl XXH64 implementation
3087  * @ingroup impl
3088  *
3089  * Details on the XXH64 implementation.
3090  * @{
3091  */
3092 /* #define rather that static const, to be used as initializers */
3093 #define XXH_PRIME64_1  0x9E3779B185EBCA87ULL  /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
3094 #define XXH_PRIME64_2  0xC2B2AE3D27D4EB4FULL  /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
3095 #define XXH_PRIME64_3  0x165667B19E3779F9ULL  /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
3096 #define XXH_PRIME64_4  0x85EBCA77C2B2AE63ULL  /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
3097 #define XXH_PRIME64_5  0x27D4EB2F165667C5ULL  /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
3098 
3099 #ifdef XXH_OLD_NAMES
3100 #  define PRIME64_1 XXH_PRIME64_1
3101 #  define PRIME64_2 XXH_PRIME64_2
3102 #  define PRIME64_3 XXH_PRIME64_3
3103 #  define PRIME64_4 XXH_PRIME64_4
3104 #  define PRIME64_5 XXH_PRIME64_5
3105 #endif
3106 
3107 /*! @copydoc XXH32_round */
3108 static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
3109 {
3110     acc += input * XXH_PRIME64_2;
3111     acc  = XXH_rotl64(acc, 31);
3112     acc *= XXH_PRIME64_1;
3113     return acc;
3114 }
3115 
3116 static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
3117 {
3118     val  = XXH64_round(0, val);
3119     acc ^= val;
3120     acc  = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
3121     return acc;
3122 }
3123 
3124 /*! @copydoc XXH32_avalanche */
3125 static xxh_u64 XXH64_avalanche(xxh_u64 hash)
3126 {
3127     hash ^= hash >> 33;
3128     hash *= XXH_PRIME64_2;
3129     hash ^= hash >> 29;
3130     hash *= XXH_PRIME64_3;
3131     hash ^= hash >> 32;
3132     return hash;
3133 }
3134 
3135 
3136 #define XXH_get64bits(p) XXH_readLE64_align(p, align)
3137 
3138 /*!
3139  * @internal
3140  * @brief Processes the last 0-31 bytes of @p ptr.
3141  *
3142  * There may be up to 31 bytes remaining to consume from the input.
3143  * This final stage will digest them to ensure that all input bytes are present
3144  * in the final mix.
3145  *
3146  * @param hash The hash to finalize.
3147  * @param ptr The pointer to the remaining input.
3148  * @param len The remaining length, modulo 32.
3149  * @param align Whether @p ptr is aligned.
3150  * @return The finalized hash
3151  * @see XXH32_finalize().
3152  */
3153 static XXH_PUREF xxh_u64
3154 XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
3155 {
3156     if (ptr==NULL) XXH_ASSERT(len == 0);
3157     len &= 31;
3158     while (len >= 8) {
3159         xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
3160         ptr += 8;
3161         hash ^= k1;
3162         hash  = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
3163         len -= 8;
3164     }
3165     if (len >= 4) {
3166         hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
3167         ptr += 4;
3168         hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
3169         len -= 4;
3170     }
3171     while (len > 0) {
3172         hash ^= (*ptr++) * XXH_PRIME64_5;
3173         hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1;
3174         --len;
3175     }
3176     return  XXH64_avalanche(hash);
3177 }
3178 
3179 #ifdef XXH_OLD_NAMES
3180 #  define PROCESS1_64 XXH_PROCESS1_64
3181 #  define PROCESS4_64 XXH_PROCESS4_64
3182 #  define PROCESS8_64 XXH_PROCESS8_64
3183 #else
3184 #  undef XXH_PROCESS1_64
3185 #  undef XXH_PROCESS4_64
3186 #  undef XXH_PROCESS8_64
3187 #endif
3188 
3189 /*!
3190  * @internal
3191  * @brief The implementation for @ref XXH64().
3192  *
3193  * @param input , len , seed Directly passed from @ref XXH64().
3194  * @param align Whether @p input is aligned.
3195  * @return The calculated hash.
3196  */
3197 XXH_FORCE_INLINE XXH_PUREF xxh_u64
3198 XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
3199 {
3200     xxh_u64 h64;
3201     if (input==NULL) XXH_ASSERT(len == 0);
3202 
3203     if (len>=32) {
3204         const xxh_u8* const bEnd = input + len;
3205         const xxh_u8* const limit = bEnd - 31;
3206         xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3207         xxh_u64 v2 = seed + XXH_PRIME64_2;
3208         xxh_u64 v3 = seed + 0;
3209         xxh_u64 v4 = seed - XXH_PRIME64_1;
3210 
3211         do {
3212             v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
3213             v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
3214             v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
3215             v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
3216         } while (input<limit);
3217 
3218         h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
3219         h64 = XXH64_mergeRound(h64, v1);
3220         h64 = XXH64_mergeRound(h64, v2);
3221         h64 = XXH64_mergeRound(h64, v3);
3222         h64 = XXH64_mergeRound(h64, v4);
3223 
3224     } else {
3225         h64  = seed + XXH_PRIME64_5;
3226     }
3227 
3228     h64 += (xxh_u64) len;
3229 
3230     return XXH64_finalize(h64, input, len, align);
3231 }
3232 
3233 
3234 /*! @ingroup XXH64_family */
3235 XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
3236 {
3237 #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
3238     /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
3239     XXH64_state_t state;
3240     XXH64_reset(&state, seed);
3241     XXH64_update(&state, (const xxh_u8*)input, len);
3242     return XXH64_digest(&state);
3243 #else
3244     if (XXH_FORCE_ALIGN_CHECK) {
3245         if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
3246             return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
3247     }   }
3248 
3249     return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
3250 
3251 #endif
3252 }
3253 
3254 /*******   Hash Streaming   *******/
3255 #ifndef XXH_NO_STREAM
3256 /*! @ingroup XXH64_family*/
3257 XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
3258 {
3259     return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
3260 }
3261 /*! @ingroup XXH64_family */
3262 XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
3263 {
3264     XXH_free(statePtr);
3265     return XXH_OK;
3266 }
3267 
3268 /*! @ingroup XXH64_family */
3269 XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState)
3270 {
3271     XXH_memcpy(dstState, srcState, sizeof(*dstState));
3272 }
3273 
3274 /*! @ingroup XXH64_family */
3275 XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed)
3276 {
3277     XXH_ASSERT(statePtr != NULL);
3278     memset(statePtr, 0, sizeof(*statePtr));
3279     statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3280     statePtr->v[1] = seed + XXH_PRIME64_2;
3281     statePtr->v[2] = seed + 0;
3282     statePtr->v[3] = seed - XXH_PRIME64_1;
3283     return XXH_OK;
3284 }
3285 
3286 /*! @ingroup XXH64_family */
3287 XXH_PUBLIC_API XXH_errorcode
3288 XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len)
3289 {
3290     if (input==NULL) {
3291         XXH_ASSERT(len == 0);
3292         return XXH_OK;
3293     }
3294 
3295     {   const xxh_u8* p = (const xxh_u8*)input;
3296         const xxh_u8* const bEnd = p + len;
3297 
3298         state->total_len += len;
3299 
3300         if (state->memsize + len < 32) {  /* fill in tmp buffer */
3301             XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
3302             state->memsize += (xxh_u32)len;
3303             return XXH_OK;
3304         }
3305 
3306         if (state->memsize) {   /* tmp buffer is full */
3307             XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
3308             state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
3309             state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
3310             state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
3311             state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
3312             p += 32 - state->memsize;
3313             state->memsize = 0;
3314         }
3315 
3316         if (p+32 <= bEnd) {
3317             const xxh_u8* const limit = bEnd - 32;
3318 
3319             do {
3320                 state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
3321                 state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
3322                 state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
3323                 state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
3324             } while (p<=limit);
3325 
3326         }
3327 
3328         if (p < bEnd) {
3329             XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
3330             state->memsize = (unsigned)(bEnd-p);
3331         }
3332     }
3333 
3334     return XXH_OK;
3335 }
3336 
3337 
3338 /*! @ingroup XXH64_family */
3339 XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state)
3340 {
3341     xxh_u64 h64;
3342 
3343     if (state->total_len >= 32) {
3344         h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
3345         h64 = XXH64_mergeRound(h64, state->v[0]);
3346         h64 = XXH64_mergeRound(h64, state->v[1]);
3347         h64 = XXH64_mergeRound(h64, state->v[2]);
3348         h64 = XXH64_mergeRound(h64, state->v[3]);
3349     } else {
3350         h64  = state->v[2] /*seed*/ + XXH_PRIME64_5;
3351     }
3352 
3353     h64 += (xxh_u64) state->total_len;
3354 
3355     return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
3356 }
3357 #endif /* !XXH_NO_STREAM */
3358 
3359 /******* Canonical representation   *******/
3360 
3361 /*! @ingroup XXH64_family */
3362 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash)
3363 {
3364     XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
3365     if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
3366     XXH_memcpy(dst, &hash, sizeof(*dst));
3367 }
3368 
3369 /*! @ingroup XXH64_family */
3370 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src)
3371 {
3372     return XXH_readBE64(src);
3373 }
3374 
3375 #ifndef XXH_NO_XXH3
3376 
3377 /* *********************************************************************
3378 *  XXH3
3379 *  New generation hash designed for speed on small keys and vectorization
3380 ************************************************************************ */
3381 /*!
3382  * @}
3383  * @defgroup XXH3_impl XXH3 implementation
3384  * @ingroup impl
3385  * @{
3386  */
3387 
3388 /* ===   Compiler specifics   === */
3389 
3390 #if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
3391 #  define XXH_RESTRICT   /* disable */
3392 #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* >= C99 */
3393 #  define XXH_RESTRICT   restrict
3394 #elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
3395    || (defined (__clang__)) \
3396    || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
3397    || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
3398 /*
3399  * There are a LOT more compilers that recognize __restrict but this
3400  * covers the major ones.
3401  */
3402 #  define XXH_RESTRICT   __restrict
3403 #else
3404 #  define XXH_RESTRICT   /* disable */
3405 #endif
3406 
3407 #if (defined(__GNUC__) && (__GNUC__ >= 3))  \
3408   || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
3409   || defined(__clang__)
3410 #    define XXH_likely(x) __builtin_expect(x, 1)
3411 #    define XXH_unlikely(x) __builtin_expect(x, 0)
3412 #else
3413 #    define XXH_likely(x) (x)
3414 #    define XXH_unlikely(x) (x)
3415 #endif
3416 
3417 #ifndef XXH_HAS_INCLUDE
3418 #  ifdef __has_include
3419 #    define XXH_HAS_INCLUDE(x) __has_include(x)
3420 #  else
3421 #    define XXH_HAS_INCLUDE(x) 0
3422 #  endif
3423 #endif
3424 
3425 #if defined(__GNUC__) || defined(__clang__)
3426 #  if defined(__ARM_FEATURE_SVE)
3427 #    include <arm_sve.h>
3428 #  endif
3429 #  if defined(__ARM_NEON__) || defined(__ARM_NEON) \
3430    || (defined(_M_ARM) && _M_ARM >= 7) \
3431    || defined(_M_ARM64) || defined(_M_ARM64EC) \
3432    || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */
3433 #    define inline __inline__  /* circumvent a clang bug */
3434 #    include <arm_neon.h>
3435 #    undef inline
3436 #  elif defined(__AVX2__)
3437 #    include <immintrin.h>
3438 #  elif defined(__SSE2__)
3439 #    include <emmintrin.h>
3440 #  endif
3441 #endif
3442 
3443 #if defined(_MSC_VER)
3444 #  include <intrin.h>
3445 #endif
3446 
3447 /*
3448  * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
3449  * remaining a true 64-bit/128-bit hash function.
3450  *
3451  * This is done by prioritizing a subset of 64-bit operations that can be
3452  * emulated without too many steps on the average 32-bit machine.
3453  *
3454  * For example, these two lines seem similar, and run equally fast on 64-bit:
3455  *
3456  *   xxh_u64 x;
3457  *   x ^= (x >> 47); // good
3458  *   x ^= (x >> 13); // bad
3459  *
3460  * However, to a 32-bit machine, there is a major difference.
3461  *
3462  * x ^= (x >> 47) looks like this:
3463  *
3464  *   x.lo ^= (x.hi >> (47 - 32));
3465  *
3466  * while x ^= (x >> 13) looks like this:
3467  *
3468  *   // note: funnel shifts are not usually cheap.
3469  *   x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
3470  *   x.hi ^= (x.hi >> 13);
3471  *
3472  * The first one is significantly faster than the second, simply because the
3473  * shift is larger than 32. This means:
3474  *  - All the bits we need are in the upper 32 bits, so we can ignore the lower
3475  *    32 bits in the shift.
3476  *  - The shift result will always fit in the lower 32 bits, and therefore,
3477  *    we can ignore the upper 32 bits in the xor.
3478  *
3479  * Thanks to this optimization, XXH3 only requires these features to be efficient:
3480  *
3481  *  - Usable unaligned access
3482  *  - A 32-bit or 64-bit ALU
3483  *      - If 32-bit, a decent ADC instruction
3484  *  - A 32 or 64-bit multiply with a 64-bit result
3485  *  - For the 128-bit variant, a decent byteswap helps short inputs.
3486  *
3487  * The first two are already required by XXH32, and almost all 32-bit and 64-bit
3488  * platforms which can run XXH32 can run XXH3 efficiently.
3489  *
3490  * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
3491  * notable exception.
3492  *
3493  * First of all, Thumb-1 lacks support for the UMULL instruction which
3494  * performs the important long multiply. This means numerous __aeabi_lmul
3495  * calls.
3496  *
3497  * Second of all, the 8 functional registers are just not enough.
3498  * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
3499  * Lo registers, and this shuffling results in thousands more MOVs than A32.
3500  *
3501  * A32 and T32 don't have this limitation. They can access all 14 registers,
3502  * do a 32->64 multiply with UMULL, and the flexible operand allowing free
3503  * shifts is helpful, too.
3504  *
3505  * Therefore, we do a quick sanity check.
3506  *
3507  * If compiling Thumb-1 for a target which supports ARM instructions, we will
3508  * emit a warning, as it is not a "sane" platform to compile for.
3509  *
3510  * Usually, if this happens, it is because of an accident and you probably need
3511  * to specify -march, as you likely meant to compile for a newer architecture.
3512  *
3513  * Credit: large sections of the vectorial and asm source code paths
3514  *         have been contributed by @easyaspi314
3515  */
3516 #if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
3517 #   warning "XXH3 is highly inefficient without ARM or Thumb-2."
3518 #endif
3519 
3520 /* ==========================================
3521  * Vectorization detection
3522  * ========================================== */
3523 
3524 #ifdef XXH_DOXYGEN
3525 /*!
3526  * @ingroup tuning
3527  * @brief Overrides the vectorization implementation chosen for XXH3.
3528  *
3529  * Can be defined to 0 to disable SIMD or any of the values mentioned in
3530  * @ref XXH_VECTOR_TYPE.
3531  *
3532  * If this is not defined, it uses predefined macros to determine the best
3533  * implementation.
3534  */
3535 #  define XXH_VECTOR XXH_SCALAR
3536 /*!
3537  * @ingroup tuning
3538  * @brief Possible values for @ref XXH_VECTOR.
3539  *
3540  * Note that these are actually implemented as macros.
3541  *
3542  * If this is not defined, it is detected automatically.
3543  * internal macro XXH_X86DISPATCH overrides this.
3544  */
3545 enum XXH_VECTOR_TYPE /* fake enum */ {
3546     XXH_SCALAR = 0,  /*!< Portable scalar version */
3547     XXH_SSE2   = 1,  /*!<
3548                       * SSE2 for Pentium 4, Opteron, all x86_64.
3549                       *
3550                       * @note SSE2 is also guaranteed on Windows 10, macOS, and
3551                       * Android x86.
3552                       */
3553     XXH_AVX2   = 2,  /*!< AVX2 for Haswell and Bulldozer */
3554     XXH_AVX512 = 3,  /*!< AVX512 for Skylake and Icelake */
3555     XXH_NEON   = 4,  /*!<
3556                        * NEON for most ARMv7-A, all AArch64, and WASM SIMD128
3557                        * via the SIMDeverywhere polyfill provided with the
3558                        * Emscripten SDK.
3559                        */
3560     XXH_VSX    = 5,  /*!< VSX and ZVector for POWER8/z13 (64-bit) */
3561     XXH_SVE    = 6,  /*!< SVE for some ARMv8-A and ARMv9-A */
3562 };
3563 /*!
3564  * @ingroup tuning
3565  * @brief Selects the minimum alignment for XXH3's accumulators.
3566  *
3567  * When using SIMD, this should match the alignment required for said vector
3568  * type, so, for example, 32 for AVX2.
3569  *
3570  * Default: Auto detected.
3571  */
3572 #  define XXH_ACC_ALIGN 8
3573 #endif
3574 
3575 /* Actual definition */
3576 #ifndef XXH_DOXYGEN
3577 #  define XXH_SCALAR 0
3578 #  define XXH_SSE2   1
3579 #  define XXH_AVX2   2
3580 #  define XXH_AVX512 3
3581 #  define XXH_NEON   4
3582 #  define XXH_VSX    5
3583 #  define XXH_SVE    6
3584 #endif
3585 
3586 #ifndef XXH_VECTOR    /* can be defined on command line */
3587 #  if defined(__ARM_FEATURE_SVE)
3588 #    define XXH_VECTOR XXH_SVE
3589 #  elif ( \
3590         defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
3591      || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \
3592      || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \
3593    ) && ( \
3594         defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
3595     || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
3596    )
3597 #    define XXH_VECTOR XXH_NEON
3598 #  elif defined(__AVX512F__)
3599 #    define XXH_VECTOR XXH_AVX512
3600 #  elif defined(__AVX2__)
3601 #    define XXH_VECTOR XXH_AVX2
3602 #  elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
3603 #    define XXH_VECTOR XXH_SSE2
3604 #  elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
3605      || (defined(__s390x__) && defined(__VEC__)) \
3606      && defined(__GNUC__) /* TODO: IBM XL */
3607 #    define XXH_VECTOR XXH_VSX
3608 #  else
3609 #    define XXH_VECTOR XXH_SCALAR
3610 #  endif
3611 #endif
3612 
3613 /* __ARM_FEATURE_SVE is only supported by GCC & Clang. */
3614 #if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE)
3615 #  ifdef _MSC_VER
3616 #    pragma warning(once : 4606)
3617 #  else
3618 #    warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead."
3619 #  endif
3620 #  undef XXH_VECTOR
3621 #  define XXH_VECTOR XXH_SCALAR
3622 #endif
3623 
3624 /*
3625  * Controls the alignment of the accumulator,
3626  * for compatibility with aligned vector loads, which are usually faster.
3627  */
3628 #ifndef XXH_ACC_ALIGN
3629 #  if defined(XXH_X86DISPATCH)
3630 #     define XXH_ACC_ALIGN 64  /* for compatibility with avx512 */
3631 #  elif XXH_VECTOR == XXH_SCALAR  /* scalar */
3632 #     define XXH_ACC_ALIGN 8
3633 #  elif XXH_VECTOR == XXH_SSE2  /* sse2 */
3634 #     define XXH_ACC_ALIGN 16
3635 #  elif XXH_VECTOR == XXH_AVX2  /* avx2 */
3636 #     define XXH_ACC_ALIGN 32
3637 #  elif XXH_VECTOR == XXH_NEON  /* neon */
3638 #     define XXH_ACC_ALIGN 16
3639 #  elif XXH_VECTOR == XXH_VSX   /* vsx */
3640 #     define XXH_ACC_ALIGN 16
3641 #  elif XXH_VECTOR == XXH_AVX512  /* avx512 */
3642 #     define XXH_ACC_ALIGN 64
3643 #  elif XXH_VECTOR == XXH_SVE   /* sve */
3644 #     define XXH_ACC_ALIGN 64
3645 #  endif
3646 #endif
3647 
3648 #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
3649     || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
3650 #  define XXH_SEC_ALIGN XXH_ACC_ALIGN
3651 #elif XXH_VECTOR == XXH_SVE
3652 #  define XXH_SEC_ALIGN XXH_ACC_ALIGN
3653 #else
3654 #  define XXH_SEC_ALIGN 8
3655 #endif
3656 
3657 #if defined(__GNUC__) || defined(__clang__)
3658 #  define XXH_ALIASING __attribute__((may_alias))
3659 #else
3660 #  define XXH_ALIASING /* nothing */
3661 #endif
3662 
3663 /*
3664  * UGLY HACK:
3665  * GCC usually generates the best code with -O3 for xxHash.
3666  *
3667  * However, when targeting AVX2, it is overzealous in its unrolling resulting
3668  * in code roughly 3/4 the speed of Clang.
3669  *
3670  * There are other issues, such as GCC splitting _mm256_loadu_si256 into
3671  * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
3672  * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
3673  *
3674  * That is why when compiling the AVX2 version, it is recommended to use either
3675  *   -O2 -mavx2 -march=haswell
3676  * or
3677  *   -O2 -mavx2 -mno-avx256-split-unaligned-load
3678  * for decent performance, or to use Clang instead.
3679  *
3680  * Fortunately, we can control the first one with a pragma that forces GCC into
3681  * -O2, but the other one we can't control without "failed to inline always
3682  * inline function due to target mismatch" warnings.
3683  */
3684 #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
3685   && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
3686   && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
3687 #  pragma GCC push_options
3688 #  pragma GCC optimize("-O2")
3689 #endif
3690 
3691 #if XXH_VECTOR == XXH_NEON
3692 
3693 /*
3694  * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3
3695  * optimizes out the entire hashLong loop because of the aliasing violation.
3696  *
3697  * However, GCC is also inefficient at load-store optimization with vld1q/vst1q,
3698  * so the only option is to mark it as aliasing.
3699  */
3700 typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING;
3701 
3702 /*!
3703  * @internal
3704  * @brief `vld1q_u64` but faster and alignment-safe.
3705  *
3706  * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only
3707  * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86).
3708  *
3709  * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it
3710  * prohibits load-store optimizations. Therefore, a direct dereference is used.
3711  *
3712  * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe
3713  * unaligned load.
3714  */
3715 #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)
3716 XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */
3717 {
3718     return *(xxh_aliasing_uint64x2_t const *)ptr;
3719 }
3720 #else
3721 XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr)
3722 {
3723     return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr));
3724 }
3725 #endif
3726 
3727 /*!
3728  * @internal
3729  * @brief `vmlal_u32` on low and high halves of a vector.
3730  *
3731  * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with
3732  * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32`
3733  * with `vmlal_u32`.
3734  */
3735 #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11
3736 XXH_FORCE_INLINE uint64x2_t
3737 XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3738 {
3739     /* Inline assembly is the only way */
3740     __asm__("umlal   %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs));
3741     return acc;
3742 }
3743 XXH_FORCE_INLINE uint64x2_t
3744 XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3745 {
3746     /* This intrinsic works as expected */
3747     return vmlal_high_u32(acc, lhs, rhs);
3748 }
3749 #else
3750 /* Portable intrinsic versions */
3751 XXH_FORCE_INLINE uint64x2_t
3752 XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3753 {
3754     return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
3755 }
3756 /*! @copydoc XXH_vmlal_low_u32
3757  * Assume the compiler converts this to vmlal_high_u32 on aarch64 */
3758 XXH_FORCE_INLINE uint64x2_t
3759 XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3760 {
3761     return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
3762 }
3763 #endif
3764 
3765 /*!
3766  * @ingroup tuning
3767  * @brief Controls the NEON to scalar ratio for XXH3
3768  *
3769  * This can be set to 2, 4, 6, or 8.
3770  *
3771  * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used.
3772  *
3773  * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those
3774  * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU
3775  * bandwidth.
3776  *
3777  * This is even more noticeable on the more advanced cores like the Cortex-A76 which
3778  * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
3779  *
3780  * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes
3781  * and 2 scalar lanes, which is chosen by default.
3782  *
3783  * This does not apply to Apple processors or 32-bit processors, which run better with
3784  * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes.
3785  *
3786  * This change benefits CPUs with large micro-op buffers without negatively affecting
3787  * most other CPUs:
3788  *
3789  *  | Chipset               | Dispatch type       | NEON only | 6:2 hybrid | Diff. |
3790  *  |:----------------------|:--------------------|----------:|-----------:|------:|
3791  *  | Snapdragon 730 (A76)  | 2 NEON/8 micro-ops  |  8.8 GB/s |  10.1 GB/s |  ~16% |
3792  *  | Snapdragon 835 (A73)  | 2 NEON/3 micro-ops  |  5.1 GB/s |   5.3 GB/s |   ~5% |
3793  *  | Marvell PXA1928 (A53) | In-order dual-issue |  1.9 GB/s |   1.9 GB/s |    0% |
3794  *  | Apple M1              | 4 NEON/8 micro-ops  | 37.3 GB/s |  36.1 GB/s |  ~-3% |
3795  *
3796  * It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
3797  *
3798  * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning
3799  * it effectively becomes worse 4.
3800  *
3801  * @see XXH3_accumulate_512_neon()
3802  */
3803 # ifndef XXH3_NEON_LANES
3804 #  if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \
3805    && !defined(__APPLE__) && XXH_SIZE_OPT <= 0
3806 #   define XXH3_NEON_LANES 6
3807 #  else
3808 #   define XXH3_NEON_LANES XXH_ACC_NB
3809 #  endif
3810 # endif
3811 #endif  /* XXH_VECTOR == XXH_NEON */
3812 
3813 /*
3814  * VSX and Z Vector helpers.
3815  *
3816  * This is very messy, and any pull requests to clean this up are welcome.
3817  *
3818  * There are a lot of problems with supporting VSX and s390x, due to
3819  * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
3820  */
3821 #if XXH_VECTOR == XXH_VSX
3822 /* Annoyingly, these headers _may_ define three macros: `bool`, `vector`,
3823  * and `pixel`. This is a problem for obvious reasons.
3824  *
3825  * These keywords are unnecessary; the spec literally says they are
3826  * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd
3827  * after including the header.
3828  *
3829  * We use pragma push_macro/pop_macro to keep the namespace clean. */
3830 #  pragma push_macro("bool")
3831 #  pragma push_macro("vector")
3832 #  pragma push_macro("pixel")
3833 /* silence potential macro redefined warnings */
3834 #  undef bool
3835 #  undef vector
3836 #  undef pixel
3837 
3838 #  if defined(__s390x__)
3839 #    include <s390intrin.h>
3840 #  else
3841 #    include <altivec.h>
3842 #  endif
3843 
3844 /* Restore the original macro values, if applicable. */
3845 #  pragma pop_macro("pixel")
3846 #  pragma pop_macro("vector")
3847 #  pragma pop_macro("bool")
3848 
3849 typedef __vector unsigned long long xxh_u64x2;
3850 typedef __vector unsigned char xxh_u8x16;
3851 typedef __vector unsigned xxh_u32x4;
3852 
3853 /*
3854  * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue.
3855  */
3856 typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;
3857 
3858 # ifndef XXH_VSX_BE
3859 #  if defined(__BIG_ENDIAN__) \
3860   || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
3861 #    define XXH_VSX_BE 1
3862 #  elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
3863 #    warning "-maltivec=be is not recommended. Please use native endianness."
3864 #    define XXH_VSX_BE 1
3865 #  else
3866 #    define XXH_VSX_BE 0
3867 #  endif
3868 # endif /* !defined(XXH_VSX_BE) */
3869 
3870 # if XXH_VSX_BE
3871 #  if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
3872 #    define XXH_vec_revb vec_revb
3873 #  else
3874 /*!
3875  * A polyfill for POWER9's vec_revb().
3876  */
3877 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
3878 {
3879     xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
3880                                   0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
3881     return vec_perm(val, val, vByteSwap);
3882 }
3883 #  endif
3884 # endif /* XXH_VSX_BE */
3885 
3886 /*!
3887  * Performs an unaligned vector load and byte swaps it on big endian.
3888  */
3889 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
3890 {
3891     xxh_u64x2 ret;
3892     XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
3893 # if XXH_VSX_BE
3894     ret = XXH_vec_revb(ret);
3895 # endif
3896     return ret;
3897 }
3898 
3899 /*
3900  * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
3901  *
3902  * These intrinsics weren't added until GCC 8, despite existing for a while,
3903  * and they are endian dependent. Also, their meaning swap depending on version.
3904  * */
3905 # if defined(__s390x__)
3906  /* s390x is always big endian, no issue on this platform */
3907 #  define XXH_vec_mulo vec_mulo
3908 #  define XXH_vec_mule vec_mule
3909 # elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__)
3910 /* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
3911  /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */
3912 #  define XXH_vec_mulo __builtin_altivec_vmulouw
3913 #  define XXH_vec_mule __builtin_altivec_vmuleuw
3914 # else
3915 /* gcc needs inline assembly */
3916 /* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
3917 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
3918 {
3919     xxh_u64x2 result;
3920     __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
3921     return result;
3922 }
3923 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
3924 {
3925     xxh_u64x2 result;
3926     __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
3927     return result;
3928 }
3929 # endif /* XXH_vec_mulo, XXH_vec_mule */
3930 #endif /* XXH_VECTOR == XXH_VSX */
3931 
3932 #if XXH_VECTOR == XXH_SVE
3933 #define ACCRND(acc, offset) \
3934 do { \
3935     svuint64_t input_vec = svld1_u64(mask, xinput + offset);         \
3936     svuint64_t secret_vec = svld1_u64(mask, xsecret + offset);       \
3937     svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec);     \
3938     svuint64_t swapped = svtbl_u64(input_vec, kSwap);                \
3939     svuint64_t mixed_lo = svextw_u64_x(mask, mixed);                 \
3940     svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32);            \
3941     svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \
3942     acc = svadd_u64_x(mask, acc, mul);                               \
3943 } while (0)
3944 #endif /* XXH_VECTOR == XXH_SVE */
3945 
3946 /* prefetch
3947  * can be disabled, by declaring XXH_NO_PREFETCH build macro */
3948 #if defined(XXH_NO_PREFETCH)
3949 #  define XXH_PREFETCH(ptr)  (void)(ptr)  /* disabled */
3950 #else
3951 #  if XXH_SIZE_OPT >= 1
3952 #    define XXH_PREFETCH(ptr) (void)(ptr)
3953 #  elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))  /* _mm_prefetch() not defined outside of x86/x64 */
3954 #    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
3955 #    define XXH_PREFETCH(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
3956 #  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
3957 #    define XXH_PREFETCH(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
3958 #  else
3959 #    define XXH_PREFETCH(ptr) (void)(ptr)  /* disabled */
3960 #  endif
3961 #endif  /* XXH_NO_PREFETCH */
3962 
3963 
3964 /* ==========================================
3965  * XXH3 default settings
3966  * ========================================== */
3967 
3968 #define XXH_SECRET_DEFAULT_SIZE 192   /* minimum XXH3_SECRET_SIZE_MIN */
3969 
3970 #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
3971 #  error "default keyset is not large enough"
3972 #endif
3973 
3974 /*! Pseudorandom secret taken directly from FARSH. */
3975 XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
3976     0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
3977     0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
3978     0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
3979     0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
3980     0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
3981     0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
3982     0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
3983     0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
3984     0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
3985     0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
3986     0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
3987     0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
3988 };
3989 
3990 static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL;  /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */
3991 static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL;  /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */
3992 
3993 #ifdef XXH_OLD_NAMES
3994 #  define kSecret XXH3_kSecret
3995 #endif
3996 
3997 #ifdef XXH_DOXYGEN
3998 /*!
3999  * @brief Calculates a 32-bit to 64-bit long multiply.
4000  *
4001  * Implemented as a macro.
4002  *
4003  * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
4004  * need to (but it shouldn't need to anyways, it is about 7 instructions to do
4005  * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
4006  * use that instead of the normal method.
4007  *
4008  * If you are compiling for platforms like Thumb-1 and don't have a better option,
4009  * you may also want to write your own long multiply routine here.
4010  *
4011  * @param x, y Numbers to be multiplied
4012  * @return 64-bit product of the low 32 bits of @p x and @p y.
4013  */
4014 XXH_FORCE_INLINE xxh_u64
4015 XXH_mult32to64(xxh_u64 x, xxh_u64 y)
4016 {
4017    return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
4018 }
4019 #elif defined(_MSC_VER) && defined(_M_IX86)
4020 #    define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
4021 #else
4022 /*
4023  * Downcast + upcast is usually better than masking on older compilers like
4024  * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
4025  *
4026  * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
4027  * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
4028  */
4029 #    define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
4030 #endif
4031 
4032 /*!
4033  * @brief Calculates a 64->128-bit long multiply.
4034  *
4035  * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
4036  * version.
4037  *
4038  * @param lhs , rhs The 64-bit integers to be multiplied
4039  * @return The 128-bit result represented in an @ref XXH128_hash_t.
4040  */
4041 static XXH128_hash_t
4042 XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
4043 {
4044     /*
4045      * GCC/Clang __uint128_t method.
4046      *
4047      * On most 64-bit targets, GCC and Clang define a __uint128_t type.
4048      * This is usually the best way as it usually uses a native long 64-bit
4049      * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
4050      *
4051      * Usually.
4052      *
4053      * Despite being a 32-bit platform, Clang (and emscripten) define this type
4054      * despite not having the arithmetic for it. This results in a laggy
4055      * compiler builtin call which calculates a full 128-bit multiply.
4056      * In that case it is best to use the portable one.
4057      * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
4058      */
4059 #if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \
4060     && defined(__SIZEOF_INT128__) \
4061     || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
4062 
4063     __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
4064     XXH128_hash_t r128;
4065     r128.low64  = (xxh_u64)(product);
4066     r128.high64 = (xxh_u64)(product >> 64);
4067     return r128;
4068 
4069     /*
4070      * MSVC for x64's _umul128 method.
4071      *
4072      * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
4073      *
4074      * This compiles to single operand MUL on x64.
4075      */
4076 #elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC)
4077 
4078 #ifndef _MSC_VER
4079 #   pragma intrinsic(_umul128)
4080 #endif
4081     xxh_u64 product_high;
4082     xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
4083     XXH128_hash_t r128;
4084     r128.low64  = product_low;
4085     r128.high64 = product_high;
4086     return r128;
4087 
4088     /*
4089      * MSVC for ARM64's __umulh method.
4090      *
4091      * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
4092      */
4093 #elif defined(_M_ARM64) || defined(_M_ARM64EC)
4094 
4095 #ifndef _MSC_VER
4096 #   pragma intrinsic(__umulh)
4097 #endif
4098     XXH128_hash_t r128;
4099     r128.low64  = lhs * rhs;
4100     r128.high64 = __umulh(lhs, rhs);
4101     return r128;
4102 
4103 #else
4104     /*
4105      * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
4106      *
4107      * This is a fast and simple grade school multiply, which is shown below
4108      * with base 10 arithmetic instead of base 0x100000000.
4109      *
4110      *           9 3 // D2 lhs = 93
4111      *         x 7 5 // D2 rhs = 75
4112      *     ----------
4113      *           1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
4114      *         4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
4115      *         2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
4116      *     + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
4117      *     ---------
4118      *         2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
4119      *     + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
4120      *     ---------
4121      *       6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
4122      *
4123      * The reasons for adding the products like this are:
4124      *  1. It avoids manual carry tracking. Just like how
4125      *     (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
4126      *     This avoids a lot of complexity.
4127      *
4128      *  2. It hints for, and on Clang, compiles to, the powerful UMAAL
4129      *     instruction available in ARM's Digital Signal Processing extension
4130      *     in 32-bit ARMv6 and later, which is shown below:
4131      *
4132      *         void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
4133      *         {
4134      *             xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
4135      *             *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
4136      *             *RdHi = (xxh_u32)(product >> 32);
4137      *         }
4138      *
4139      *     This instruction was designed for efficient long multiplication, and
4140      *     allows this to be calculated in only 4 instructions at speeds
4141      *     comparable to some 64-bit ALUs.
4142      *
4143      *  3. It isn't terrible on other platforms. Usually this will be a couple
4144      *     of 32-bit ADD/ADCs.
4145      */
4146 
4147     /* First calculate all of the cross products. */
4148     xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
4149     xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32,        rhs & 0xFFFFFFFF);
4150     xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
4151     xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32,        rhs >> 32);
4152 
4153     /* Now add the products together. These will never overflow. */
4154     xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
4155     xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32)        + hi_hi;
4156     xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
4157 
4158     XXH128_hash_t r128;
4159     r128.low64  = lower;
4160     r128.high64 = upper;
4161     return r128;
4162 #endif
4163 }
4164 
4165 /*!
4166  * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
4167  *
4168  * The reason for the separate function is to prevent passing too many structs
4169  * around by value. This will hopefully inline the multiply, but we don't force it.
4170  *
4171  * @param lhs , rhs The 64-bit integers to multiply
4172  * @return The low 64 bits of the product XOR'd by the high 64 bits.
4173  * @see XXH_mult64to128()
4174  */
4175 static xxh_u64
4176 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
4177 {
4178     XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
4179     return product.low64 ^ product.high64;
4180 }
4181 
4182 /*! Seems to produce slightly better code on GCC for some reason. */
4183 XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
4184 {
4185     XXH_ASSERT(0 <= shift && shift < 64);
4186     return v64 ^ (v64 >> shift);
4187 }
4188 
4189 /*
4190  * This is a fast avalanche stage,
4191  * suitable when input bits are already partially mixed
4192  */
4193 static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
4194 {
4195     h64 = XXH_xorshift64(h64, 37);
4196     h64 *= PRIME_MX1;
4197     h64 = XXH_xorshift64(h64, 32);
4198     return h64;
4199 }
4200 
4201 /*
4202  * This is a stronger avalanche,
4203  * inspired by Pelle Evensen's rrmxmx
4204  * preferable when input has not been previously mixed
4205  */
4206 static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
4207 {
4208     /* this mix is inspired by Pelle Evensen's rrmxmx */
4209     h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
4210     h64 *= PRIME_MX2;
4211     h64 ^= (h64 >> 35) + len ;
4212     h64 *= PRIME_MX2;
4213     return XXH_xorshift64(h64, 28);
4214 }
4215 
4216 
4217 /* ==========================================
4218  * Short keys
4219  * ==========================================
4220  * One of the shortcomings of XXH32 and XXH64 was that their performance was
4221  * sub-optimal on short lengths. It used an iterative algorithm which strongly
4222  * favored lengths that were a multiple of 4 or 8.
4223  *
4224  * Instead of iterating over individual inputs, we use a set of single shot
4225  * functions which piece together a range of lengths and operate in constant time.
4226  *
4227  * Additionally, the number of multiplies has been significantly reduced. This
4228  * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
4229  *
4230  * Depending on the platform, this may or may not be faster than XXH32, but it
4231  * is almost guaranteed to be faster than XXH64.
4232  */
4233 
4234 /*
4235  * At very short lengths, there isn't enough input to fully hide secrets, or use
4236  * the entire secret.
4237  *
4238  * There is also only a limited amount of mixing we can do before significantly
4239  * impacting performance.
4240  *
4241  * Therefore, we use different sections of the secret and always mix two secret
4242  * samples with an XOR. This should have no effect on performance on the
4243  * seedless or withSeed variants because everything _should_ be constant folded
4244  * by modern compilers.
4245  *
4246  * The XOR mixing hides individual parts of the secret and increases entropy.
4247  *
4248  * This adds an extra layer of strength for custom secrets.
4249  */
4250 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4251 XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4252 {
4253     XXH_ASSERT(input != NULL);
4254     XXH_ASSERT(1 <= len && len <= 3);
4255     XXH_ASSERT(secret != NULL);
4256     /*
4257      * len = 1: combined = { input[0], 0x01, input[0], input[0] }
4258      * len = 2: combined = { input[1], 0x02, input[0], input[1] }
4259      * len = 3: combined = { input[2], 0x03, input[0], input[1] }
4260      */
4261     {   xxh_u8  const c1 = input[0];
4262         xxh_u8  const c2 = input[len >> 1];
4263         xxh_u8  const c3 = input[len - 1];
4264         xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2  << 24)
4265                                | ((xxh_u32)c3 <<  0) | ((xxh_u32)len << 8);
4266         xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
4267         xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
4268         return XXH64_avalanche(keyed);
4269     }
4270 }
4271 
4272 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4273 XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4274 {
4275     XXH_ASSERT(input != NULL);
4276     XXH_ASSERT(secret != NULL);
4277     XXH_ASSERT(4 <= len && len <= 8);
4278     seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
4279     {   xxh_u32 const input1 = XXH_readLE32(input);
4280         xxh_u32 const input2 = XXH_readLE32(input + len - 4);
4281         xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
4282         xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
4283         xxh_u64 const keyed = input64 ^ bitflip;
4284         return XXH3_rrmxmx(keyed, len);
4285     }
4286 }
4287 
4288 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4289 XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4290 {
4291     XXH_ASSERT(input != NULL);
4292     XXH_ASSERT(secret != NULL);
4293     XXH_ASSERT(9 <= len && len <= 16);
4294     {   xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
4295         xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
4296         xxh_u64 const input_lo = XXH_readLE64(input)           ^ bitflip1;
4297         xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
4298         xxh_u64 const acc = len
4299                           + XXH_swap64(input_lo) + input_hi
4300                           + XXH3_mul128_fold64(input_lo, input_hi);
4301         return XXH3_avalanche(acc);
4302     }
4303 }
4304 
4305 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4306 XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4307 {
4308     XXH_ASSERT(len <= 16);
4309     {   if (XXH_likely(len >  8)) return XXH3_len_9to16_64b(input, len, secret, seed);
4310         if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
4311         if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
4312         return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
4313     }
4314 }
4315 
4316 /*
4317  * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
4318  * multiplication by zero, affecting hashes of lengths 17 to 240.
4319  *
4320  * However, they are very unlikely.
4321  *
4322  * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
4323  * unseeded non-cryptographic hashes, it does not attempt to defend itself
4324  * against specially crafted inputs, only random inputs.
4325  *
4326  * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
4327  * cancelling out the secret is taken an arbitrary number of times (addressed
4328  * in XXH3_accumulate_512), this collision is very unlikely with random inputs
4329  * and/or proper seeding:
4330  *
4331  * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
4332  * function that is only called up to 16 times per hash with up to 240 bytes of
4333  * input.
4334  *
4335  * This is not too bad for a non-cryptographic hash function, especially with
4336  * only 64 bit outputs.
4337  *
4338  * The 128-bit variant (which trades some speed for strength) is NOT affected
4339  * by this, although it is always a good idea to use a proper seed if you care
4340  * about strength.
4341  */
4342 XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
4343                                      const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
4344 {
4345 #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
4346   && defined(__i386__) && defined(__SSE2__)  /* x86 + SSE2 */ \
4347   && !defined(XXH_ENABLE_AUTOVECTORIZE)      /* Define to disable like XXH32 hack */
4348     /*
4349      * UGLY HACK:
4350      * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
4351      * slower code.
4352      *
4353      * By forcing seed64 into a register, we disrupt the cost model and
4354      * cause it to scalarize. See `XXH32_round()`
4355      *
4356      * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
4357      * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
4358      * GCC 9.2, despite both emitting scalar code.
4359      *
4360      * GCC generates much better scalar code than Clang for the rest of XXH3,
4361      * which is why finding a more optimal codepath is an interest.
4362      */
4363     XXH_COMPILER_GUARD(seed64);
4364 #endif
4365     {   xxh_u64 const input_lo = XXH_readLE64(input);
4366         xxh_u64 const input_hi = XXH_readLE64(input+8);
4367         return XXH3_mul128_fold64(
4368             input_lo ^ (XXH_readLE64(secret)   + seed64),
4369             input_hi ^ (XXH_readLE64(secret+8) - seed64)
4370         );
4371     }
4372 }
4373 
4374 /* For mid range keys, XXH3 uses a Mum-hash variant. */
4375 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4376 XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4377                      const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4378                      XXH64_hash_t seed)
4379 {
4380     XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4381     XXH_ASSERT(16 < len && len <= 128);
4382 
4383     {   xxh_u64 acc = len * XXH_PRIME64_1;
4384 #if XXH_SIZE_OPT >= 1
4385         /* Smaller and cleaner, but slightly slower. */
4386         unsigned int i = (unsigned int)(len - 1) / 32;
4387         do {
4388             acc += XXH3_mix16B(input+16 * i, secret+32*i, seed);
4389             acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed);
4390         } while (i-- != 0);
4391 #else
4392         if (len > 32) {
4393             if (len > 64) {
4394                 if (len > 96) {
4395                     acc += XXH3_mix16B(input+48, secret+96, seed);
4396                     acc += XXH3_mix16B(input+len-64, secret+112, seed);
4397                 }
4398                 acc += XXH3_mix16B(input+32, secret+64, seed);
4399                 acc += XXH3_mix16B(input+len-48, secret+80, seed);
4400             }
4401             acc += XXH3_mix16B(input+16, secret+32, seed);
4402             acc += XXH3_mix16B(input+len-32, secret+48, seed);
4403         }
4404         acc += XXH3_mix16B(input+0, secret+0, seed);
4405         acc += XXH3_mix16B(input+len-16, secret+16, seed);
4406 #endif
4407         return XXH3_avalanche(acc);
4408     }
4409 }
4410 
4411 #define XXH3_MIDSIZE_MAX 240
4412 
4413 XXH_NO_INLINE XXH_PUREF XXH64_hash_t
4414 XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4415                       const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4416                       XXH64_hash_t seed)
4417 {
4418     XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4419     XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4420 
4421     #define XXH3_MIDSIZE_STARTOFFSET 3
4422     #define XXH3_MIDSIZE_LASTOFFSET  17
4423 
4424     {   xxh_u64 acc = len * XXH_PRIME64_1;
4425         xxh_u64 acc_end;
4426         unsigned int const nbRounds = (unsigned int)len / 16;
4427         unsigned int i;
4428         XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4429         for (i=0; i<8; i++) {
4430             acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
4431         }
4432         /* last bytes */
4433         acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
4434         XXH_ASSERT(nbRounds >= 8);
4435         acc = XXH3_avalanche(acc);
4436 #if defined(__clang__)                                /* Clang */ \
4437     && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
4438     && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
4439         /*
4440          * UGLY HACK:
4441          * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
4442          * In everywhere else, it uses scalar code.
4443          *
4444          * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
4445          * would still be slower than UMAAL (see XXH_mult64to128).
4446          *
4447          * Unfortunately, Clang doesn't handle the long multiplies properly and
4448          * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
4449          * scalarized into an ugly mess of VMOV.32 instructions.
4450          *
4451          * This mess is difficult to avoid without turning autovectorization
4452          * off completely, but they are usually relatively minor and/or not
4453          * worth it to fix.
4454          *
4455          * This loop is the easiest to fix, as unlike XXH32, this pragma
4456          * _actually works_ because it is a loop vectorization instead of an
4457          * SLP vectorization.
4458          */
4459         #pragma clang loop vectorize(disable)
4460 #endif
4461         for (i=8 ; i < nbRounds; i++) {
4462             /*
4463              * Prevents clang for unrolling the acc loop and interleaving with this one.
4464              */
4465             XXH_COMPILER_GUARD(acc);
4466             acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
4467         }
4468         return XXH3_avalanche(acc + acc_end);
4469     }
4470 }
4471 
4472 
4473 /* =======     Long Keys     ======= */
4474 
4475 #define XXH_STRIPE_LEN 64
4476 #define XXH_SECRET_CONSUME_RATE 8   /* nb of secret bytes consumed at each accumulation */
4477 #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
4478 
4479 #ifdef XXH_OLD_NAMES
4480 #  define STRIPE_LEN XXH_STRIPE_LEN
4481 #  define ACC_NB XXH_ACC_NB
4482 #endif
4483 
4484 #ifndef XXH_PREFETCH_DIST
4485 #  ifdef __clang__
4486 #    define XXH_PREFETCH_DIST 320
4487 #  else
4488 #    if (XXH_VECTOR == XXH_AVX512)
4489 #      define XXH_PREFETCH_DIST 512
4490 #    else
4491 #      define XXH_PREFETCH_DIST 384
4492 #    endif
4493 #  endif  /* __clang__ */
4494 #endif  /* XXH_PREFETCH_DIST */
4495 
4496 /*
4497  * These macros are to generate an XXH3_accumulate() function.
4498  * The two arguments select the name suffix and target attribute.
4499  *
4500  * The name of this symbol is XXH3_accumulate_<name>() and it calls
4501  * XXH3_accumulate_512_<name>().
4502  *
4503  * It may be useful to hand implement this function if the compiler fails to
4504  * optimize the inline function.
4505  */
4506 #define XXH3_ACCUMULATE_TEMPLATE(name)                      \
4507 void                                                        \
4508 XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc,           \
4509                        const xxh_u8* XXH_RESTRICT input,    \
4510                        const xxh_u8* XXH_RESTRICT secret,   \
4511                        size_t nbStripes)                    \
4512 {                                                           \
4513     size_t n;                                               \
4514     for (n = 0; n < nbStripes; n++ ) {                      \
4515         const xxh_u8* const in = input + n*XXH_STRIPE_LEN;  \
4516         XXH_PREFETCH(in + XXH_PREFETCH_DIST);               \
4517         XXH3_accumulate_512_##name(                         \
4518                  acc,                                       \
4519                  in,                                        \
4520                  secret + n*XXH_SECRET_CONSUME_RATE);       \
4521     }                                                       \
4522 }
4523 
4524 
4525 XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
4526 {
4527     if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
4528     XXH_memcpy(dst, &v64, sizeof(v64));
4529 }
4530 
4531 /* Several intrinsic functions below are supposed to accept __int64 as argument,
4532  * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
4533  * However, several environments do not define __int64 type,
4534  * requiring a workaround.
4535  */
4536 #if !defined (__VMS) \
4537   && (defined (__cplusplus) \
4538   || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
4539     typedef int64_t xxh_i64;
4540 #else
4541     /* the following type must have a width of 64-bit */
4542     typedef long long xxh_i64;
4543 #endif
4544 
4545 
4546 /*
4547  * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
4548  *
4549  * It is a hardened version of UMAC, based off of FARSH's implementation.
4550  *
4551  * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
4552  * implementations, and it is ridiculously fast.
4553  *
4554  * We harden it by mixing the original input to the accumulators as well as the product.
4555  *
4556  * This means that in the (relatively likely) case of a multiply by zero, the
4557  * original input is preserved.
4558  *
4559  * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
4560  * cross-pollination, as otherwise the upper and lower halves would be
4561  * essentially independent.
4562  *
4563  * This doesn't matter on 64-bit hashes since they all get merged together in
4564  * the end, so we skip the extra step.
4565  *
4566  * Both XXH3_64bits and XXH3_128bits use this subroutine.
4567  */
4568 
4569 #if (XXH_VECTOR == XXH_AVX512) \
4570      || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
4571 
4572 #ifndef XXH_TARGET_AVX512
4573 # define XXH_TARGET_AVX512  /* disable attribute target */
4574 #endif
4575 
4576 XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4577 XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
4578                      const void* XXH_RESTRICT input,
4579                      const void* XXH_RESTRICT secret)
4580 {
4581     __m512i* const xacc = (__m512i *) acc;
4582     XXH_ASSERT((((size_t)acc) & 63) == 0);
4583     XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4584 
4585     {
4586         /* data_vec    = input[0]; */
4587         __m512i const data_vec    = _mm512_loadu_si512   (input);
4588         /* key_vec     = secret[0]; */
4589         __m512i const key_vec     = _mm512_loadu_si512   (secret);
4590         /* data_key    = data_vec ^ key_vec; */
4591         __m512i const data_key    = _mm512_xor_si512     (data_vec, key_vec);
4592         /* data_key_lo = data_key >> 32; */
4593         __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32);
4594         /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4595         __m512i const product     = _mm512_mul_epu32     (data_key, data_key_lo);
4596         /* xacc[0] += swap(data_vec); */
4597         __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
4598         __m512i const sum       = _mm512_add_epi64(*xacc, data_swap);
4599         /* xacc[0] += product; */
4600         *xacc = _mm512_add_epi64(product, sum);
4601     }
4602 }
4603 XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512)
4604 
4605 /*
4606  * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
4607  *
4608  * Multiplication isn't perfect, as explained by Google in HighwayHash:
4609  *
4610  *  // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
4611  *  // varying degrees. In descending order of goodness, bytes
4612  *  // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
4613  *  // As expected, the upper and lower bytes are much worse.
4614  *
4615  * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
4616  *
4617  * Since our algorithm uses a pseudorandom secret to add some variance into the
4618  * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
4619  *
4620  * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
4621  * extraction.
4622  *
4623  * Both XXH3_64bits and XXH3_128bits use this subroutine.
4624  */
4625 
4626 XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4627 XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4628 {
4629     XXH_ASSERT((((size_t)acc) & 63) == 0);
4630     XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4631     {   __m512i* const xacc = (__m512i*) acc;
4632         const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
4633 
4634         /* xacc[0] ^= (xacc[0] >> 47) */
4635         __m512i const acc_vec     = *xacc;
4636         __m512i const shifted     = _mm512_srli_epi64    (acc_vec, 47);
4637         /* xacc[0] ^= secret; */
4638         __m512i const key_vec     = _mm512_loadu_si512   (secret);
4639         __m512i const data_key    = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */);
4640 
4641         /* xacc[0] *= XXH_PRIME32_1; */
4642         __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32);
4643         __m512i const prod_lo     = _mm512_mul_epu32     (data_key, prime32);
4644         __m512i const prod_hi     = _mm512_mul_epu32     (data_key_hi, prime32);
4645         *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
4646     }
4647 }
4648 
4649 XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4650 XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4651 {
4652     XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
4653     XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
4654     XXH_ASSERT(((size_t)customSecret & 63) == 0);
4655     (void)(&XXH_writeLE64);
4656     {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
4657         __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64);
4658         __m512i const seed     = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos);
4659 
4660         const __m512i* const src  = (const __m512i*) ((const void*) XXH3_kSecret);
4661               __m512i* const dest = (      __m512i*) customSecret;
4662         int i;
4663         XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
4664         XXH_ASSERT(((size_t)dest & 63) == 0);
4665         for (i=0; i < nbRounds; ++i) {
4666             dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed);
4667     }   }
4668 }
4669 
4670 #endif
4671 
4672 #if (XXH_VECTOR == XXH_AVX2) \
4673     || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
4674 
4675 #ifndef XXH_TARGET_AVX2
4676 # define XXH_TARGET_AVX2  /* disable attribute target */
4677 #endif
4678 
4679 XXH_FORCE_INLINE XXH_TARGET_AVX2 void
4680 XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
4681                     const void* XXH_RESTRICT input,
4682                     const void* XXH_RESTRICT secret)
4683 {
4684     XXH_ASSERT((((size_t)acc) & 31) == 0);
4685     {   __m256i* const xacc    =       (__m256i *) acc;
4686         /* Unaligned. This is mainly for pointer arithmetic, and because
4687          * _mm256_loadu_si256 requires  a const __m256i * pointer for some reason. */
4688         const         __m256i* const xinput  = (const __m256i *) input;
4689         /* Unaligned. This is mainly for pointer arithmetic, and because
4690          * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
4691         const         __m256i* const xsecret = (const __m256i *) secret;
4692 
4693         size_t i;
4694         for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
4695             /* data_vec    = xinput[i]; */
4696             __m256i const data_vec    = _mm256_loadu_si256    (xinput+i);
4697             /* key_vec     = xsecret[i]; */
4698             __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
4699             /* data_key    = data_vec ^ key_vec; */
4700             __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);
4701             /* data_key_lo = data_key >> 32; */
4702             __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32);
4703             /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4704             __m256i const product     = _mm256_mul_epu32     (data_key, data_key_lo);
4705             /* xacc[i] += swap(data_vec); */
4706             __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
4707             __m256i const sum       = _mm256_add_epi64(xacc[i], data_swap);
4708             /* xacc[i] += product; */
4709             xacc[i] = _mm256_add_epi64(product, sum);
4710     }   }
4711 }
4712 XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2)
4713 
4714 XXH_FORCE_INLINE XXH_TARGET_AVX2 void
4715 XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4716 {
4717     XXH_ASSERT((((size_t)acc) & 31) == 0);
4718     {   __m256i* const xacc = (__m256i*) acc;
4719         /* Unaligned. This is mainly for pointer arithmetic, and because
4720          * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
4721         const         __m256i* const xsecret = (const __m256i *) secret;
4722         const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
4723 
4724         size_t i;
4725         for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
4726             /* xacc[i] ^= (xacc[i] >> 47) */
4727             __m256i const acc_vec     = xacc[i];
4728             __m256i const shifted     = _mm256_srli_epi64    (acc_vec, 47);
4729             __m256i const data_vec    = _mm256_xor_si256     (acc_vec, shifted);
4730             /* xacc[i] ^= xsecret; */
4731             __m256i const key_vec     = _mm256_loadu_si256   (xsecret+i);
4732             __m256i const data_key    = _mm256_xor_si256     (data_vec, key_vec);
4733 
4734             /* xacc[i] *= XXH_PRIME32_1; */
4735             __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32);
4736             __m256i const prod_lo     = _mm256_mul_epu32     (data_key, prime32);
4737             __m256i const prod_hi     = _mm256_mul_epu32     (data_key_hi, prime32);
4738             xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
4739         }
4740     }
4741 }
4742 
4743 XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4744 {
4745     XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
4746     XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
4747     XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
4748     (void)(&XXH_writeLE64);
4749     XXH_PREFETCH(customSecret);
4750     {   __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
4751 
4752         const __m256i* const src  = (const __m256i*) ((const void*) XXH3_kSecret);
4753               __m256i*       dest = (      __m256i*) customSecret;
4754 
4755 #       if defined(__GNUC__) || defined(__clang__)
4756         /*
4757          * On GCC & Clang, marking 'dest' as modified will cause the compiler:
4758          *   - do not extract the secret from sse registers in the internal loop
4759          *   - use less common registers, and avoid pushing these reg into stack
4760          */
4761         XXH_COMPILER_GUARD(dest);
4762 #       endif
4763         XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
4764         XXH_ASSERT(((size_t)dest & 31) == 0);
4765 
4766         /* GCC -O2 need unroll loop manually */
4767         dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed);
4768         dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed);
4769         dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed);
4770         dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed);
4771         dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed);
4772         dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed);
4773     }
4774 }
4775 
4776 #endif
4777 
4778 /* x86dispatch always generates SSE2 */
4779 #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
4780 
4781 #ifndef XXH_TARGET_SSE2
4782 # define XXH_TARGET_SSE2  /* disable attribute target */
4783 #endif
4784 
4785 XXH_FORCE_INLINE XXH_TARGET_SSE2 void
4786 XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
4787                     const void* XXH_RESTRICT input,
4788                     const void* XXH_RESTRICT secret)
4789 {
4790     /* SSE2 is just a half-scale version of the AVX2 version. */
4791     XXH_ASSERT((((size_t)acc) & 15) == 0);
4792     {   __m128i* const xacc    =       (__m128i *) acc;
4793         /* Unaligned. This is mainly for pointer arithmetic, and because
4794          * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
4795         const         __m128i* const xinput  = (const __m128i *) input;
4796         /* Unaligned. This is mainly for pointer arithmetic, and because
4797          * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
4798         const         __m128i* const xsecret = (const __m128i *) secret;
4799 
4800         size_t i;
4801         for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
4802             /* data_vec    = xinput[i]; */
4803             __m128i const data_vec    = _mm_loadu_si128   (xinput+i);
4804             /* key_vec     = xsecret[i]; */
4805             __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
4806             /* data_key    = data_vec ^ key_vec; */
4807             __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);
4808             /* data_key_lo = data_key >> 32; */
4809             __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
4810             /* product     = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4811             __m128i const product     = _mm_mul_epu32     (data_key, data_key_lo);
4812             /* xacc[i] += swap(data_vec); */
4813             __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
4814             __m128i const sum       = _mm_add_epi64(xacc[i], data_swap);
4815             /* xacc[i] += product; */
4816             xacc[i] = _mm_add_epi64(product, sum);
4817     }   }
4818 }
4819 XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2)
4820 
4821 XXH_FORCE_INLINE XXH_TARGET_SSE2 void
4822 XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4823 {
4824     XXH_ASSERT((((size_t)acc) & 15) == 0);
4825     {   __m128i* const xacc = (__m128i*) acc;
4826         /* Unaligned. This is mainly for pointer arithmetic, and because
4827          * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
4828         const         __m128i* const xsecret = (const __m128i *) secret;
4829         const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
4830 
4831         size_t i;
4832         for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
4833             /* xacc[i] ^= (xacc[i] >> 47) */
4834             __m128i const acc_vec     = xacc[i];
4835             __m128i const shifted     = _mm_srli_epi64    (acc_vec, 47);
4836             __m128i const data_vec    = _mm_xor_si128     (acc_vec, shifted);
4837             /* xacc[i] ^= xsecret[i]; */
4838             __m128i const key_vec     = _mm_loadu_si128   (xsecret+i);
4839             __m128i const data_key    = _mm_xor_si128     (data_vec, key_vec);
4840 
4841             /* xacc[i] *= XXH_PRIME32_1; */
4842             __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
4843             __m128i const prod_lo     = _mm_mul_epu32     (data_key, prime32);
4844             __m128i const prod_hi     = _mm_mul_epu32     (data_key_hi, prime32);
4845             xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
4846         }
4847     }
4848 }
4849 
4850 XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4851 {
4852     XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
4853     (void)(&XXH_writeLE64);
4854     {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
4855 
4856 #       if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
4857         /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
4858         XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
4859         __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
4860 #       else
4861         __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
4862 #       endif
4863         int i;
4864 
4865         const void* const src16 = XXH3_kSecret;
4866         __m128i* dst16 = (__m128i*) customSecret;
4867 #       if defined(__GNUC__) || defined(__clang__)
4868         /*
4869          * On GCC & Clang, marking 'dest' as modified will cause the compiler:
4870          *   - do not extract the secret from sse registers in the internal loop
4871          *   - use less common registers, and avoid pushing these reg into stack
4872          */
4873         XXH_COMPILER_GUARD(dst16);
4874 #       endif
4875         XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
4876         XXH_ASSERT(((size_t)dst16 & 15) == 0);
4877 
4878         for (i=0; i < nbRounds; ++i) {
4879             dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
4880     }   }
4881 }
4882 
4883 #endif
4884 
4885 #if (XXH_VECTOR == XXH_NEON)
4886 
4887 /* forward declarations for the scalar routines */
4888 XXH_FORCE_INLINE void
4889 XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
4890                  void const* XXH_RESTRICT secret, size_t lane);
4891 
4892 XXH_FORCE_INLINE void
4893 XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
4894                          void const* XXH_RESTRICT secret, size_t lane);
4895 
4896 /*!
4897  * @internal
4898  * @brief The bulk processing loop for NEON and WASM SIMD128.
4899  *
4900  * The NEON code path is actually partially scalar when running on AArch64. This
4901  * is to optimize the pipelining and can have up to 15% speedup depending on the
4902  * CPU, and it also mitigates some GCC codegen issues.
4903  *
4904  * @see XXH3_NEON_LANES for configuring this and details about this optimization.
4905  *
4906  * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit
4907  * integers instead of the other platforms which mask full 64-bit vectors,
4908  * so the setup is more complicated than just shifting right.
4909  *
4910  * Additionally, there is an optimization for 4 lanes at once noted below.
4911  *
4912  * Since, as stated, the most optimal amount of lanes for Cortexes is 6,
4913  * there needs to be *three* versions of the accumulate operation used
4914  * for the remaining 2 lanes.
4915  *
4916  * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap
4917  * nearly perfectly.
4918  */
4919 
4920 XXH_FORCE_INLINE void
4921 XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
4922                     const void* XXH_RESTRICT input,
4923                     const void* XXH_RESTRICT secret)
4924 {
4925     XXH_ASSERT((((size_t)acc) & 15) == 0);
4926     XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
4927     {   /* GCC for darwin arm64 does not like aliasing here */
4928         xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc;
4929         /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
4930         uint8_t const* xinput = (const uint8_t *) input;
4931         uint8_t const* xsecret  = (const uint8_t *) secret;
4932 
4933         size_t i;
4934 #ifdef __wasm_simd128__
4935         /*
4936          * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret
4937          * is constant propagated, which results in it converting it to this
4938          * inside the loop:
4939          *
4940          *    a = v128.load(XXH3_kSecret +  0 + $secret_offset, offset = 0)
4941          *    b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0)
4942          *    ...
4943          *
4944          * This requires a full 32-bit address immediate (and therefore a 6 byte
4945          * instruction) as well as an add for each offset.
4946          *
4947          * Putting an asm guard prevents it from folding (at the cost of losing
4948          * the alignment hint), and uses the free offset in `v128.load` instead
4949          * of adding secret_offset each time which overall reduces code size by
4950          * about a kilobyte and improves performance.
4951          */
4952         XXH_COMPILER_GUARD(xsecret);
4953 #endif
4954         /* Scalar lanes use the normal scalarRound routine */
4955         for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
4956             XXH3_scalarRound(acc, input, secret, i);
4957         }
4958         i = 0;
4959         /* 4 NEON lanes at a time. */
4960         for (; i+1 < XXH3_NEON_LANES / 2; i+=2) {
4961             /* data_vec = xinput[i]; */
4962             uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput  + (i * 16));
4963             uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput  + ((i+1) * 16));
4964             /* key_vec  = xsecret[i];  */
4965             uint64x2_t key_vec_1  = XXH_vld1q_u64(xsecret + (i * 16));
4966             uint64x2_t key_vec_2  = XXH_vld1q_u64(xsecret + ((i+1) * 16));
4967             /* data_swap = swap(data_vec) */
4968             uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
4969             uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
4970             /* data_key = data_vec ^ key_vec; */
4971             uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
4972             uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
4973 
4974             /*
4975              * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a
4976              * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to
4977              * get one vector with the low 32 bits of each lane, and one vector
4978              * with the high 32 bits of each lane.
4979              *
4980              * The intrinsic returns a double vector because the original ARMv7-a
4981              * instruction modified both arguments in place. AArch64 and SIMD128 emit
4982              * two instructions from this intrinsic.
4983              *
4984              *  [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ]
4985              *  [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ]
4986              */
4987             uint32x4x2_t unzipped = vuzpq_u32(
4988                 vreinterpretq_u32_u64(data_key_1),
4989                 vreinterpretq_u32_u64(data_key_2)
4990             );
4991             /* data_key_lo = data_key & 0xFFFFFFFF */
4992             uint32x4_t data_key_lo = unzipped.val[0];
4993             /* data_key_hi = data_key >> 32 */
4994             uint32x4_t data_key_hi = unzipped.val[1];
4995             /*
4996              * Then, we can split the vectors horizontally and multiply which, as for most
4997              * widening intrinsics, have a variant that works on both high half vectors
4998              * for free on AArch64. A similar instruction is available on SIMD128.
4999              *
5000              * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi
5001              */
5002             uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi);
5003             uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi);
5004             /*
5005              * Clang reorders
5006              *    a += b * c;     // umlal   swap.2d, dkl.2s, dkh.2s
5007              *    c += a;         // add     acc.2d, acc.2d, swap.2d
5008              * to
5009              *    c += a;         // add     acc.2d, acc.2d, swap.2d
5010              *    c += b * c;     // umlal   acc.2d, dkl.2s, dkh.2s
5011              *
5012              * While it would make sense in theory since the addition is faster,
5013              * for reasons likely related to umlal being limited to certain NEON
5014              * pipelines, this is worse. A compiler guard fixes this.
5015              */
5016             XXH_COMPILER_GUARD_CLANG_NEON(sum_1);
5017             XXH_COMPILER_GUARD_CLANG_NEON(sum_2);
5018             /* xacc[i] = acc_vec + sum; */
5019             xacc[i]   = vaddq_u64(xacc[i], sum_1);
5020             xacc[i+1] = vaddq_u64(xacc[i+1], sum_2);
5021         }
5022         /* Operate on the remaining NEON lanes 2 at a time. */
5023         for (; i < XXH3_NEON_LANES / 2; i++) {
5024             /* data_vec = xinput[i]; */
5025             uint64x2_t data_vec = XXH_vld1q_u64(xinput  + (i * 16));
5026             /* key_vec  = xsecret[i];  */
5027             uint64x2_t key_vec  = XXH_vld1q_u64(xsecret + (i * 16));
5028             /* acc_vec_2 = swap(data_vec) */
5029             uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1);
5030             /* data_key = data_vec ^ key_vec; */
5031             uint64x2_t data_key = veorq_u64(data_vec, key_vec);
5032             /* For two lanes, just use VMOVN and VSHRN. */
5033             /* data_key_lo = data_key & 0xFFFFFFFF; */
5034             uint32x2_t data_key_lo = vmovn_u64(data_key);
5035             /* data_key_hi = data_key >> 32; */
5036             uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32);
5037             /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */
5038             uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi);
5039             /* Same Clang workaround as before */
5040             XXH_COMPILER_GUARD_CLANG_NEON(sum);
5041             /* xacc[i] = acc_vec + sum; */
5042             xacc[i] = vaddq_u64 (xacc[i], sum);
5043         }
5044     }
5045 }
5046 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon)
5047 
5048 XXH_FORCE_INLINE void
5049 XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5050 {
5051     XXH_ASSERT((((size_t)acc) & 15) == 0);
5052 
5053     {   xxh_aliasing_uint64x2_t* xacc       = (xxh_aliasing_uint64x2_t*) acc;
5054         uint8_t const* xsecret = (uint8_t const*) secret;
5055 
5056         size_t i;
5057         /* WASM uses operator overloads and doesn't need these. */
5058 #ifndef __wasm_simd128__
5059         /* { prime32_1, prime32_1 } */
5060         uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1);
5061         /* { 0, prime32_1, 0, prime32_1 } */
5062         uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32));
5063 #endif
5064 
5065         /* AArch64 uses both scalar and neon at the same time */
5066         for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
5067             XXH3_scalarScrambleRound(acc, secret, i);
5068         }
5069         for (i=0; i < XXH3_NEON_LANES / 2; i++) {
5070             /* xacc[i] ^= (xacc[i] >> 47); */
5071             uint64x2_t acc_vec  = xacc[i];
5072             uint64x2_t shifted  = vshrq_n_u64(acc_vec, 47);
5073             uint64x2_t data_vec = veorq_u64(acc_vec, shifted);
5074 
5075             /* xacc[i] ^= xsecret[i]; */
5076             uint64x2_t key_vec  = XXH_vld1q_u64(xsecret + (i * 16));
5077             uint64x2_t data_key = veorq_u64(data_vec, key_vec);
5078             /* xacc[i] *= XXH_PRIME32_1 */
5079 #ifdef __wasm_simd128__
5080             /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */
5081             xacc[i] = data_key * XXH_PRIME32_1;
5082 #else
5083             /*
5084              * Expanded version with portable NEON intrinsics
5085              *
5086              *    lo(x) * lo(y) + (hi(x) * lo(y) << 32)
5087              *
5088              * prod_hi = hi(data_key) * lo(prime) << 32
5089              *
5090              * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector
5091              * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits
5092              * and avoid the shift.
5093              */
5094             uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi);
5095             /* Extract low bits for vmlal_u32  */
5096             uint32x2_t data_key_lo = vmovn_u64(data_key);
5097             /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */
5098             xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo);
5099 #endif
5100         }
5101     }
5102 }
5103 #endif
5104 
5105 #if (XXH_VECTOR == XXH_VSX)
5106 
5107 XXH_FORCE_INLINE void
5108 XXH3_accumulate_512_vsx(  void* XXH_RESTRICT acc,
5109                     const void* XXH_RESTRICT input,
5110                     const void* XXH_RESTRICT secret)
5111 {
5112     /* presumed aligned */
5113     xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
5114     xxh_u8 const* const xinput   = (xxh_u8 const*) input;   /* no alignment restriction */
5115     xxh_u8 const* const xsecret  = (xxh_u8 const*) secret;    /* no alignment restriction */
5116     xxh_u64x2 const v32 = { 32, 32 };
5117     size_t i;
5118     for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
5119         /* data_vec = xinput[i]; */
5120         xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i);
5121         /* key_vec = xsecret[i]; */
5122         xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + 16*i);
5123         xxh_u64x2 const data_key = data_vec ^ key_vec;
5124         /* shuffled = (data_key << 32) | (data_key >> 32); */
5125         xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
5126         /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
5127         xxh_u64x2 const product  = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
5128         /* acc_vec = xacc[i]; */
5129         xxh_u64x2 acc_vec        = xacc[i];
5130         acc_vec += product;
5131 
5132         /* swap high and low halves */
5133 #ifdef __s390x__
5134         acc_vec += vec_permi(data_vec, data_vec, 2);
5135 #else
5136         acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
5137 #endif
5138         xacc[i] = acc_vec;
5139     }
5140 }
5141 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx)
5142 
5143 XXH_FORCE_INLINE void
5144 XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5145 {
5146     XXH_ASSERT((((size_t)acc) & 15) == 0);
5147 
5148     {   xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
5149         const xxh_u8* const xsecret = (const xxh_u8*) secret;
5150         /* constants */
5151         xxh_u64x2 const v32  = { 32, 32 };
5152         xxh_u64x2 const v47 = { 47, 47 };
5153         xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
5154         size_t i;
5155         for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
5156             /* xacc[i] ^= (xacc[i] >> 47); */
5157             xxh_u64x2 const acc_vec  = xacc[i];
5158             xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
5159 
5160             /* xacc[i] ^= xsecret[i]; */
5161             xxh_u64x2 const key_vec  = XXH_vec_loadu(xsecret + 16*i);
5162             xxh_u64x2 const data_key = data_vec ^ key_vec;
5163 
5164             /* xacc[i] *= XXH_PRIME32_1 */
5165             /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF);  */
5166             xxh_u64x2 const prod_even  = XXH_vec_mule((xxh_u32x4)data_key, prime);
5167             /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32);  */
5168             xxh_u64x2 const prod_odd  = XXH_vec_mulo((xxh_u32x4)data_key, prime);
5169             xacc[i] = prod_odd + (prod_even << v32);
5170     }   }
5171 }
5172 
5173 #endif
5174 
5175 #if (XXH_VECTOR == XXH_SVE)
5176 
5177 XXH_FORCE_INLINE void
5178 XXH3_accumulate_512_sve( void* XXH_RESTRICT acc,
5179                    const void* XXH_RESTRICT input,
5180                    const void* XXH_RESTRICT secret)
5181 {
5182     uint64_t *xacc = (uint64_t *)acc;
5183     const uint64_t *xinput = (const uint64_t *)(const void *)input;
5184     const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
5185     svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
5186     uint64_t element_count = svcntd();
5187     if (element_count >= 8) {
5188         svbool_t mask = svptrue_pat_b64(SV_VL8);
5189         svuint64_t vacc = svld1_u64(mask, xacc);
5190         ACCRND(vacc, 0);
5191         svst1_u64(mask, xacc, vacc);
5192     } else if (element_count == 2) {   /* sve128 */
5193         svbool_t mask = svptrue_pat_b64(SV_VL2);
5194         svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5195         svuint64_t acc1 = svld1_u64(mask, xacc + 2);
5196         svuint64_t acc2 = svld1_u64(mask, xacc + 4);
5197         svuint64_t acc3 = svld1_u64(mask, xacc + 6);
5198         ACCRND(acc0, 0);
5199         ACCRND(acc1, 2);
5200         ACCRND(acc2, 4);
5201         ACCRND(acc3, 6);
5202         svst1_u64(mask, xacc + 0, acc0);
5203         svst1_u64(mask, xacc + 2, acc1);
5204         svst1_u64(mask, xacc + 4, acc2);
5205         svst1_u64(mask, xacc + 6, acc3);
5206     } else {
5207         svbool_t mask = svptrue_pat_b64(SV_VL4);
5208         svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5209         svuint64_t acc1 = svld1_u64(mask, xacc + 4);
5210         ACCRND(acc0, 0);
5211         ACCRND(acc1, 4);
5212         svst1_u64(mask, xacc + 0, acc0);
5213         svst1_u64(mask, xacc + 4, acc1);
5214     }
5215 }
5216 
5217 XXH_FORCE_INLINE void
5218 XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc,
5219                const xxh_u8* XXH_RESTRICT input,
5220                const xxh_u8* XXH_RESTRICT secret,
5221                size_t nbStripes)
5222 {
5223     if (nbStripes != 0) {
5224         uint64_t *xacc = (uint64_t *)acc;
5225         const uint64_t *xinput = (const uint64_t *)(const void *)input;
5226         const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
5227         svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
5228         uint64_t element_count = svcntd();
5229         if (element_count >= 8) {
5230             svbool_t mask = svptrue_pat_b64(SV_VL8);
5231             svuint64_t vacc = svld1_u64(mask, xacc + 0);
5232             do {
5233                 /* svprfd(svbool_t, void *, enum svfprop); */
5234                 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5235                 ACCRND(vacc, 0);
5236                 xinput += 8;
5237                 xsecret += 1;
5238                 nbStripes--;
5239            } while (nbStripes != 0);
5240 
5241            svst1_u64(mask, xacc + 0, vacc);
5242         } else if (element_count == 2) { /* sve128 */
5243             svbool_t mask = svptrue_pat_b64(SV_VL2);
5244             svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5245             svuint64_t acc1 = svld1_u64(mask, xacc + 2);
5246             svuint64_t acc2 = svld1_u64(mask, xacc + 4);
5247             svuint64_t acc3 = svld1_u64(mask, xacc + 6);
5248             do {
5249                 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5250                 ACCRND(acc0, 0);
5251                 ACCRND(acc1, 2);
5252                 ACCRND(acc2, 4);
5253                 ACCRND(acc3, 6);
5254                 xinput += 8;
5255                 xsecret += 1;
5256                 nbStripes--;
5257            } while (nbStripes != 0);
5258 
5259            svst1_u64(mask, xacc + 0, acc0);
5260            svst1_u64(mask, xacc + 2, acc1);
5261            svst1_u64(mask, xacc + 4, acc2);
5262            svst1_u64(mask, xacc + 6, acc3);
5263         } else {
5264             svbool_t mask = svptrue_pat_b64(SV_VL4);
5265             svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5266             svuint64_t acc1 = svld1_u64(mask, xacc + 4);
5267             do {
5268                 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5269                 ACCRND(acc0, 0);
5270                 ACCRND(acc1, 4);
5271                 xinput += 8;
5272                 xsecret += 1;
5273                 nbStripes--;
5274            } while (nbStripes != 0);
5275 
5276            svst1_u64(mask, xacc + 0, acc0);
5277            svst1_u64(mask, xacc + 4, acc1);
5278        }
5279     }
5280 }
5281 
5282 #endif
5283 
5284 /* scalar variants - universal */
5285 
5286 #if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__))
5287 /*
5288  * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they
5289  * emit an excess mask and a full 64-bit multiply-add (MADD X-form).
5290  *
5291  * While this might not seem like much, as AArch64 is a 64-bit architecture, only
5292  * big Cortex designs have a full 64-bit multiplier.
5293  *
5294  * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit
5295  * multiplies expand to 2-3 multiplies in microcode. This has a major penalty
5296  * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline.
5297  *
5298  * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does
5299  * not have this penalty and does the mask automatically.
5300  */
5301 XXH_FORCE_INLINE xxh_u64
5302 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
5303 {
5304     xxh_u64 ret;
5305     /* note: %x = 64-bit register, %w = 32-bit register */
5306     __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc));
5307     return ret;
5308 }
5309 #else
5310 XXH_FORCE_INLINE xxh_u64
5311 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
5312 {
5313     return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc;
5314 }
5315 #endif
5316 
5317 /*!
5318  * @internal
5319  * @brief Scalar round for @ref XXH3_accumulate_512_scalar().
5320  *
5321  * This is extracted to its own function because the NEON path uses a combination
5322  * of NEON and scalar.
5323  */
5324 XXH_FORCE_INLINE void
5325 XXH3_scalarRound(void* XXH_RESTRICT acc,
5326                  void const* XXH_RESTRICT input,
5327                  void const* XXH_RESTRICT secret,
5328                  size_t lane)
5329 {
5330     xxh_u64* xacc = (xxh_u64*) acc;
5331     xxh_u8 const* xinput  = (xxh_u8 const*) input;
5332     xxh_u8 const* xsecret = (xxh_u8 const*) secret;
5333     XXH_ASSERT(lane < XXH_ACC_NB);
5334     XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
5335     {
5336         xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8);
5337         xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8);
5338         xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
5339         xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]);
5340     }
5341 }
5342 
5343 /*!
5344  * @internal
5345  * @brief Processes a 64 byte block of data using the scalar path.
5346  */
5347 XXH_FORCE_INLINE void
5348 XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
5349                      const void* XXH_RESTRICT input,
5350                      const void* XXH_RESTRICT secret)
5351 {
5352     size_t i;
5353     /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */
5354 #if defined(__GNUC__) && !defined(__clang__) \
5355   && (defined(__arm__) || defined(__thumb2__)) \
5356   && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \
5357   && XXH_SIZE_OPT <= 0
5358 #  pragma GCC unroll 8
5359 #endif
5360     for (i=0; i < XXH_ACC_NB; i++) {
5361         XXH3_scalarRound(acc, input, secret, i);
5362     }
5363 }
5364 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar)
5365 
5366 /*!
5367  * @internal
5368  * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar().
5369  *
5370  * This is extracted to its own function because the NEON path uses a combination
5371  * of NEON and scalar.
5372  */
5373 XXH_FORCE_INLINE void
5374 XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
5375                          void const* XXH_RESTRICT secret,
5376                          size_t lane)
5377 {
5378     xxh_u64* const xacc = (xxh_u64*) acc;   /* presumed aligned */
5379     const xxh_u8* const xsecret = (const xxh_u8*) secret;   /* no alignment restriction */
5380     XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
5381     XXH_ASSERT(lane < XXH_ACC_NB);
5382     {
5383         xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8);
5384         xxh_u64 acc64 = xacc[lane];
5385         acc64 = XXH_xorshift64(acc64, 47);
5386         acc64 ^= key64;
5387         acc64 *= XXH_PRIME32_1;
5388         xacc[lane] = acc64;
5389     }
5390 }
5391 
5392 /*!
5393  * @internal
5394  * @brief Scrambles the accumulators after a large chunk has been read
5395  */
5396 XXH_FORCE_INLINE void
5397 XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5398 {
5399     size_t i;
5400     for (i=0; i < XXH_ACC_NB; i++) {
5401         XXH3_scalarScrambleRound(acc, secret, i);
5402     }
5403 }
5404 
5405 XXH_FORCE_INLINE void
5406 XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
5407 {
5408     /*
5409      * We need a separate pointer for the hack below,
5410      * which requires a non-const pointer.
5411      * Any decent compiler will optimize this out otherwise.
5412      */
5413     const xxh_u8* kSecretPtr = XXH3_kSecret;
5414     XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
5415 
5416 #if defined(__GNUC__) && defined(__aarch64__)
5417     /*
5418      * UGLY HACK:
5419      * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are
5420      * placed sequentially, in order, at the top of the unrolled loop.
5421      *
5422      * While MOVK is great for generating constants (2 cycles for a 64-bit
5423      * constant compared to 4 cycles for LDR), it fights for bandwidth with
5424      * the arithmetic instructions.
5425      *
5426      *   I   L   S
5427      * MOVK
5428      * MOVK
5429      * MOVK
5430      * MOVK
5431      * ADD
5432      * SUB      STR
5433      *          STR
5434      * By forcing loads from memory (as the asm line causes the compiler to assume
5435      * that XXH3_kSecretPtr has been changed), the pipelines are used more
5436      * efficiently:
5437      *   I   L   S
5438      *      LDR
5439      *  ADD LDR
5440      *  SUB     STR
5441      *          STR
5442      *
5443      * See XXH3_NEON_LANES for details on the pipsline.
5444      *
5445      * XXH3_64bits_withSeed, len == 256, Snapdragon 835
5446      *   without hack: 2654.4 MB/s
5447      *   with hack:    3202.9 MB/s
5448      */
5449     XXH_COMPILER_GUARD(kSecretPtr);
5450 #endif
5451     {   int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
5452         int i;
5453         for (i=0; i < nbRounds; i++) {
5454             /*
5455              * The asm hack causes the compiler to assume that kSecretPtr aliases with
5456              * customSecret, and on aarch64, this prevented LDP from merging two
5457              * loads together for free. Putting the loads together before the stores
5458              * properly generates LDP.
5459              */
5460             xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i)     + seed64;
5461             xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
5462             XXH_writeLE64((xxh_u8*)customSecret + 16*i,     lo);
5463             XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
5464     }   }
5465 }
5466 
5467 
5468 typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t);
5469 typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
5470 typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
5471 
5472 
5473 #if (XXH_VECTOR == XXH_AVX512)
5474 
5475 #define XXH3_accumulate_512 XXH3_accumulate_512_avx512
5476 #define XXH3_accumulate     XXH3_accumulate_avx512
5477 #define XXH3_scrambleAcc    XXH3_scrambleAcc_avx512
5478 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
5479 
5480 #elif (XXH_VECTOR == XXH_AVX2)
5481 
5482 #define XXH3_accumulate_512 XXH3_accumulate_512_avx2
5483 #define XXH3_accumulate     XXH3_accumulate_avx2
5484 #define XXH3_scrambleAcc    XXH3_scrambleAcc_avx2
5485 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
5486 
5487 #elif (XXH_VECTOR == XXH_SSE2)
5488 
5489 #define XXH3_accumulate_512 XXH3_accumulate_512_sse2
5490 #define XXH3_accumulate     XXH3_accumulate_sse2
5491 #define XXH3_scrambleAcc    XXH3_scrambleAcc_sse2
5492 #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
5493 
5494 #elif (XXH_VECTOR == XXH_NEON)
5495 
5496 #define XXH3_accumulate_512 XXH3_accumulate_512_neon
5497 #define XXH3_accumulate     XXH3_accumulate_neon
5498 #define XXH3_scrambleAcc    XXH3_scrambleAcc_neon
5499 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5500 
5501 #elif (XXH_VECTOR == XXH_VSX)
5502 
5503 #define XXH3_accumulate_512 XXH3_accumulate_512_vsx
5504 #define XXH3_accumulate     XXH3_accumulate_vsx
5505 #define XXH3_scrambleAcc    XXH3_scrambleAcc_vsx
5506 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5507 
5508 #elif (XXH_VECTOR == XXH_SVE)
5509 #define XXH3_accumulate_512 XXH3_accumulate_512_sve
5510 #define XXH3_accumulate     XXH3_accumulate_sve
5511 #define XXH3_scrambleAcc    XXH3_scrambleAcc_scalar
5512 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5513 
5514 #else /* scalar */
5515 
5516 #define XXH3_accumulate_512 XXH3_accumulate_512_scalar
5517 #define XXH3_accumulate     XXH3_accumulate_scalar
5518 #define XXH3_scrambleAcc    XXH3_scrambleAcc_scalar
5519 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5520 
5521 #endif
5522 
5523 #if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */
5524 #  undef XXH3_initCustomSecret
5525 #  define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5526 #endif
5527 
5528 XXH_FORCE_INLINE void
5529 XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
5530                       const xxh_u8* XXH_RESTRICT input, size_t len,
5531                       const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
5532                             XXH3_f_accumulate f_acc,
5533                             XXH3_f_scrambleAcc f_scramble)
5534 {
5535     size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
5536     size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
5537     size_t const nb_blocks = (len - 1) / block_len;
5538 
5539     size_t n;
5540 
5541     XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
5542 
5543     for (n = 0; n < nb_blocks; n++) {
5544         f_acc(acc, input + n*block_len, secret, nbStripesPerBlock);
5545         f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
5546     }
5547 
5548     /* last partial block */
5549     XXH_ASSERT(len > XXH_STRIPE_LEN);
5550     {   size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
5551         XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
5552         f_acc(acc, input + nb_blocks*block_len, secret, nbStripes);
5553 
5554         /* last stripe */
5555         {   const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
5556 #define XXH_SECRET_LASTACC_START 7  /* not aligned on 8, last secret is different from acc & scrambler */
5557             XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
5558     }   }
5559 }
5560 
5561 XXH_FORCE_INLINE xxh_u64
5562 XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
5563 {
5564     return XXH3_mul128_fold64(
5565                acc[0] ^ XXH_readLE64(secret),
5566                acc[1] ^ XXH_readLE64(secret+8) );
5567 }
5568 
5569 static XXH64_hash_t
5570 XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
5571 {
5572     xxh_u64 result64 = start;
5573     size_t i = 0;
5574 
5575     for (i = 0; i < 4; i++) {
5576         result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
5577 #if defined(__clang__)                                /* Clang */ \
5578     && (defined(__arm__) || defined(__thumb__))       /* ARMv7 */ \
5579     && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */  \
5580     && !defined(XXH_ENABLE_AUTOVECTORIZE)             /* Define to disable */
5581         /*
5582          * UGLY HACK:
5583          * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
5584          * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
5585          * XXH3_64bits, len == 256, Snapdragon 835:
5586          *   without hack: 2063.7 MB/s
5587          *   with hack:    2560.7 MB/s
5588          */
5589         XXH_COMPILER_GUARD(result64);
5590 #endif
5591     }
5592 
5593     return XXH3_avalanche(result64);
5594 }
5595 
5596 #define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
5597                         XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
5598 
5599 XXH_FORCE_INLINE XXH64_hash_t
5600 XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
5601                            const void* XXH_RESTRICT secret, size_t secretSize,
5602                            XXH3_f_accumulate f_acc,
5603                            XXH3_f_scrambleAcc f_scramble)
5604 {
5605     XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
5606 
5607     XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble);
5608 
5609     /* converge into final hash */
5610     XXH_STATIC_ASSERT(sizeof(acc) == 64);
5611     /* do not align on 8, so that the secret is different from the accumulator */
5612 #define XXH_SECRET_MERGEACCS_START 11
5613     XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
5614     return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
5615 }
5616 
5617 /*
5618  * It's important for performance to transmit secret's size (when it's static)
5619  * so that the compiler can properly optimize the vectorized loop.
5620  * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
5621  * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
5622  * breaks -Og, this is XXH_NO_INLINE.
5623  */
5624 XXH3_WITH_SECRET_INLINE XXH64_hash_t
5625 XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
5626                              XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5627 {
5628     (void)seed64;
5629     return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc);
5630 }
5631 
5632 /*
5633  * It's preferable for performance that XXH3_hashLong is not inlined,
5634  * as it results in a smaller function for small data, easier to the instruction cache.
5635  * Note that inside this no_inline function, we do inline the internal loop,
5636  * and provide a statically defined secret size to allow optimization of vector loop.
5637  */
5638 XXH_NO_INLINE XXH_PUREF XXH64_hash_t
5639 XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
5640                           XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5641 {
5642     (void)seed64; (void)secret; (void)secretLen;
5643     return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc);
5644 }
5645 
5646 /*
5647  * XXH3_hashLong_64b_withSeed():
5648  * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
5649  * and then use this key for long mode hashing.
5650  *
5651  * This operation is decently fast but nonetheless costs a little bit of time.
5652  * Try to avoid it whenever possible (typically when seed==0).
5653  *
5654  * It's important for performance that XXH3_hashLong is not inlined. Not sure
5655  * why (uop cache maybe?), but the difference is large and easily measurable.
5656  */
5657 XXH_FORCE_INLINE XXH64_hash_t
5658 XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
5659                                     XXH64_hash_t seed,
5660                                     XXH3_f_accumulate f_acc,
5661                                     XXH3_f_scrambleAcc f_scramble,
5662                                     XXH3_f_initCustomSecret f_initSec)
5663 {
5664 #if XXH_SIZE_OPT <= 0
5665     if (seed == 0)
5666         return XXH3_hashLong_64b_internal(input, len,
5667                                           XXH3_kSecret, sizeof(XXH3_kSecret),
5668                                           f_acc, f_scramble);
5669 #endif
5670     {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
5671         f_initSec(secret, seed);
5672         return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
5673                                           f_acc, f_scramble);
5674     }
5675 }
5676 
5677 /*
5678  * It's important for performance that XXH3_hashLong is not inlined.
5679  */
5680 XXH_NO_INLINE XXH64_hash_t
5681 XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len,
5682                            XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5683 {
5684     (void)secret; (void)secretLen;
5685     return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
5686                 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
5687 }
5688 
5689 
5690 typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
5691                                           XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
5692 
5693 XXH_FORCE_INLINE XXH64_hash_t
5694 XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
5695                      XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
5696                      XXH3_hashLong64_f f_hashLong)
5697 {
5698     XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
5699     /*
5700      * If an action is to be taken if `secretLen` condition is not respected,
5701      * it should be done here.
5702      * For now, it's a contract pre-condition.
5703      * Adding a check and a branch here would cost performance at every hash.
5704      * Also, note that function signature doesn't offer room to return an error.
5705      */
5706     if (len <= 16)
5707         return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
5708     if (len <= 128)
5709         return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
5710     if (len <= XXH3_MIDSIZE_MAX)
5711         return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
5712     return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
5713 }
5714 
5715 
5716 /* ===   Public entry point   === */
5717 
5718 /*! @ingroup XXH3_family */
5719 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length)
5720 {
5721     return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
5722 }
5723 
5724 /*! @ingroup XXH3_family */
5725 XXH_PUBLIC_API XXH64_hash_t
5726 XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize)
5727 {
5728     return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
5729 }
5730 
5731 /*! @ingroup XXH3_family */
5732 XXH_PUBLIC_API XXH64_hash_t
5733 XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed)
5734 {
5735     return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
5736 }
5737 
5738 XXH_PUBLIC_API XXH64_hash_t
5739 XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
5740 {
5741     if (length <= XXH3_MIDSIZE_MAX)
5742         return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
5743     return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize);
5744 }
5745 
5746 
5747 /* ===   XXH3 streaming   === */
5748 #ifndef XXH_NO_STREAM
5749 /*
5750  * Malloc's a pointer that is always aligned to align.
5751  *
5752  * This must be freed with `XXH_alignedFree()`.
5753  *
5754  * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
5755  * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
5756  * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
5757  *
5758  * This underalignment previously caused a rather obvious crash which went
5759  * completely unnoticed due to XXH3_createState() not actually being tested.
5760  * Credit to RedSpah for noticing this bug.
5761  *
5762  * The alignment is done manually: Functions like posix_memalign or _mm_malloc
5763  * are avoided: To maintain portability, we would have to write a fallback
5764  * like this anyways, and besides, testing for the existence of library
5765  * functions without relying on external build tools is impossible.
5766  *
5767  * The method is simple: Overallocate, manually align, and store the offset
5768  * to the original behind the returned pointer.
5769  *
5770  * Align must be a power of 2 and 8 <= align <= 128.
5771  */
5772 static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align)
5773 {
5774     XXH_ASSERT(align <= 128 && align >= 8); /* range check */
5775     XXH_ASSERT((align & (align-1)) == 0);   /* power of 2 */
5776     XXH_ASSERT(s != 0 && s < (s + align));  /* empty/overflow */
5777     {   /* Overallocate to make room for manual realignment and an offset byte */
5778         xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
5779         if (base != NULL) {
5780             /*
5781              * Get the offset needed to align this pointer.
5782              *
5783              * Even if the returned pointer is aligned, there will always be
5784              * at least one byte to store the offset to the original pointer.
5785              */
5786             size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
5787             /* Add the offset for the now-aligned pointer */
5788             xxh_u8* ptr = base + offset;
5789 
5790             XXH_ASSERT((size_t)ptr % align == 0);
5791 
5792             /* Store the offset immediately before the returned pointer. */
5793             ptr[-1] = (xxh_u8)offset;
5794             return ptr;
5795         }
5796         return NULL;
5797     }
5798 }
5799 /*
5800  * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
5801  * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
5802  */
5803 static void XXH_alignedFree(void* p)
5804 {
5805     if (p != NULL) {
5806         xxh_u8* ptr = (xxh_u8*)p;
5807         /* Get the offset byte we added in XXH_malloc. */
5808         xxh_u8 offset = ptr[-1];
5809         /* Free the original malloc'd pointer */
5810         xxh_u8* base = ptr - offset;
5811         XXH_free(base);
5812     }
5813 }
5814 /*! @ingroup XXH3_family */
5815 /*!
5816  * @brief Allocate an @ref XXH3_state_t.
5817  *
5818  * Must be freed with XXH3_freeState().
5819  * @return An allocated XXH3_state_t on success, `NULL` on failure.
5820  */
5821 XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
5822 {
5823     XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
5824     if (state==NULL) return NULL;
5825     XXH3_INITSTATE(state);
5826     return state;
5827 }
5828 
5829 /*! @ingroup XXH3_family */
5830 /*!
5831  * @brief Frees an @ref XXH3_state_t.
5832  *
5833  * Must be allocated with XXH3_createState().
5834  * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
5835  * @return XXH_OK.
5836  */
5837 XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
5838 {
5839     XXH_alignedFree(statePtr);
5840     return XXH_OK;
5841 }
5842 
5843 /*! @ingroup XXH3_family */
5844 XXH_PUBLIC_API void
5845 XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state)
5846 {
5847     XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
5848 }
5849 
5850 static void
5851 XXH3_reset_internal(XXH3_state_t* statePtr,
5852                     XXH64_hash_t seed,
5853                     const void* secret, size_t secretSize)
5854 {
5855     size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
5856     size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
5857     XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
5858     XXH_ASSERT(statePtr != NULL);
5859     /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
5860     memset((char*)statePtr + initStart, 0, initLength);
5861     statePtr->acc[0] = XXH_PRIME32_3;
5862     statePtr->acc[1] = XXH_PRIME64_1;
5863     statePtr->acc[2] = XXH_PRIME64_2;
5864     statePtr->acc[3] = XXH_PRIME64_3;
5865     statePtr->acc[4] = XXH_PRIME64_4;
5866     statePtr->acc[5] = XXH_PRIME32_2;
5867     statePtr->acc[6] = XXH_PRIME64_5;
5868     statePtr->acc[7] = XXH_PRIME32_1;
5869     statePtr->seed = seed;
5870     statePtr->useSeed = (seed != 0);
5871     statePtr->extSecret = (const unsigned char*)secret;
5872     XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
5873     statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
5874     statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
5875 }
5876 
5877 /*! @ingroup XXH3_family */
5878 XXH_PUBLIC_API XXH_errorcode
5879 XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
5880 {
5881     if (statePtr == NULL) return XXH_ERROR;
5882     XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
5883     return XXH_OK;
5884 }
5885 
5886 /*! @ingroup XXH3_family */
5887 XXH_PUBLIC_API XXH_errorcode
5888 XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
5889 {
5890     if (statePtr == NULL) return XXH_ERROR;
5891     XXH3_reset_internal(statePtr, 0, secret, secretSize);
5892     if (secret == NULL) return XXH_ERROR;
5893     if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
5894     return XXH_OK;
5895 }
5896 
5897 /*! @ingroup XXH3_family */
5898 XXH_PUBLIC_API XXH_errorcode
5899 XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
5900 {
5901     if (statePtr == NULL) return XXH_ERROR;
5902     if (seed==0) return XXH3_64bits_reset(statePtr);
5903     if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
5904         XXH3_initCustomSecret(statePtr->customSecret, seed);
5905     XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
5906     return XXH_OK;
5907 }
5908 
5909 /*! @ingroup XXH3_family */
5910 XXH_PUBLIC_API XXH_errorcode
5911 XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64)
5912 {
5913     if (statePtr == NULL) return XXH_ERROR;
5914     if (secret == NULL) return XXH_ERROR;
5915     if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
5916     XXH3_reset_internal(statePtr, seed64, secret, secretSize);
5917     statePtr->useSeed = 1; /* always, even if seed64==0 */
5918     return XXH_OK;
5919 }
5920 
5921 /*!
5922  * @internal
5923  * @brief Processes a large input for XXH3_update() and XXH3_digest_long().
5924  *
5925  * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block.
5926  *
5927  * @param acc                Pointer to the 8 accumulator lanes
5928  * @param nbStripesSoFarPtr  In/out pointer to the number of leftover stripes in the block*
5929  * @param nbStripesPerBlock  Number of stripes in a block
5930  * @param input              Input pointer
5931  * @param nbStripes          Number of stripes to process
5932  * @param secret             Secret pointer
5933  * @param secretLimit        Offset of the last block in @p secret
5934  * @param f_acc              Pointer to an XXH3_accumulate implementation
5935  * @param f_scramble         Pointer to an XXH3_scrambleAcc implementation
5936  * @return                   Pointer past the end of @p input after processing
5937  */
5938 XXH_FORCE_INLINE const xxh_u8 *
5939 XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
5940                     size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
5941                     const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
5942                     const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
5943                     XXH3_f_accumulate f_acc,
5944                     XXH3_f_scrambleAcc f_scramble)
5945 {
5946     const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE;
5947     /* Process full blocks */
5948     if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) {
5949         /* Process the initial partial block... */
5950         size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr;
5951 
5952         do {
5953             /* Accumulate and scramble */
5954             f_acc(acc, input, initialSecret, nbStripesThisIter);
5955             f_scramble(acc, secret + secretLimit);
5956             input += nbStripesThisIter * XXH_STRIPE_LEN;
5957             nbStripes -= nbStripesThisIter;
5958             /* Then continue the loop with the full block size */
5959             nbStripesThisIter = nbStripesPerBlock;
5960             initialSecret = secret;
5961         } while (nbStripes >= nbStripesPerBlock);
5962         *nbStripesSoFarPtr = 0;
5963     }
5964     /* Process a partial block */
5965     if (nbStripes > 0) {
5966         f_acc(acc, input, initialSecret, nbStripes);
5967         input += nbStripes * XXH_STRIPE_LEN;
5968         *nbStripesSoFarPtr += nbStripes;
5969     }
5970     /* Return end pointer */
5971     return input;
5972 }
5973 
5974 #ifndef XXH3_STREAM_USE_STACK
5975 # if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */
5976 #   define XXH3_STREAM_USE_STACK 1
5977 # endif
5978 #endif
5979 /*
5980  * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
5981  */
5982 XXH_FORCE_INLINE XXH_errorcode
5983 XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
5984             const xxh_u8* XXH_RESTRICT input, size_t len,
5985             XXH3_f_accumulate f_acc,
5986             XXH3_f_scrambleAcc f_scramble)
5987 {
5988     if (input==NULL) {
5989         XXH_ASSERT(len == 0);
5990         return XXH_OK;
5991     }
5992 
5993     XXH_ASSERT(state != NULL);
5994     {   const xxh_u8* const bEnd = input + len;
5995         const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
5996 #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
5997         /* For some reason, gcc and MSVC seem to suffer greatly
5998          * when operating accumulators directly into state.
5999          * Operating into stack space seems to enable proper optimization.
6000          * clang, on the other hand, doesn't seem to need this trick */
6001         XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8];
6002         XXH_memcpy(acc, state->acc, sizeof(acc));
6003 #else
6004         xxh_u64* XXH_RESTRICT const acc = state->acc;
6005 #endif
6006         state->totalLen += len;
6007         XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
6008 
6009         /* small input : just fill in tmp buffer */
6010         if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) {
6011             XXH_memcpy(state->buffer + state->bufferedSize, input, len);
6012             state->bufferedSize += (XXH32_hash_t)len;
6013             return XXH_OK;
6014         }
6015 
6016         /* total input is now > XXH3_INTERNALBUFFER_SIZE */
6017         #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
6018         XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0);   /* clean multiple */
6019 
6020         /*
6021          * Internal buffer is partially filled (always, except at beginning)
6022          * Complete it, then consume it.
6023          */
6024         if (state->bufferedSize) {
6025             size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
6026             XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
6027             input += loadSize;
6028             XXH3_consumeStripes(acc,
6029                                &state->nbStripesSoFar, state->nbStripesPerBlock,
6030                                 state->buffer, XXH3_INTERNALBUFFER_STRIPES,
6031                                 secret, state->secretLimit,
6032                                 f_acc, f_scramble);
6033             state->bufferedSize = 0;
6034         }
6035         XXH_ASSERT(input < bEnd);
6036         if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
6037             size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
6038             input = XXH3_consumeStripes(acc,
6039                                        &state->nbStripesSoFar, state->nbStripesPerBlock,
6040                                        input, nbStripes,
6041                                        secret, state->secretLimit,
6042                                        f_acc, f_scramble);
6043             XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
6044 
6045         }
6046         /* Some remaining input (always) : buffer it */
6047         XXH_ASSERT(input < bEnd);
6048         XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
6049         XXH_ASSERT(state->bufferedSize == 0);
6050         XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
6051         state->bufferedSize = (XXH32_hash_t)(bEnd-input);
6052 #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
6053         /* save stack accumulators into state */
6054         XXH_memcpy(state->acc, acc, sizeof(acc));
6055 #endif
6056     }
6057 
6058     return XXH_OK;
6059 }
6060 
6061 /*! @ingroup XXH3_family */
6062 XXH_PUBLIC_API XXH_errorcode
6063 XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
6064 {
6065     return XXH3_update(state, (const xxh_u8*)input, len,
6066                        XXH3_accumulate, XXH3_scrambleAcc);
6067 }
6068 
6069 
6070 XXH_FORCE_INLINE void
6071 XXH3_digest_long (XXH64_hash_t* acc,
6072                   const XXH3_state_t* state,
6073                   const unsigned char* secret)
6074 {
6075     xxh_u8 lastStripe[XXH_STRIPE_LEN];
6076     const xxh_u8* lastStripePtr;
6077 
6078     /*
6079      * Digest on a local copy. This way, the state remains unaltered, and it can
6080      * continue ingesting more input afterwards.
6081      */
6082     XXH_memcpy(acc, state->acc, sizeof(state->acc));
6083     if (state->bufferedSize >= XXH_STRIPE_LEN) {
6084         /* Consume remaining stripes then point to remaining data in buffer */
6085         size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
6086         size_t nbStripesSoFar = state->nbStripesSoFar;
6087         XXH3_consumeStripes(acc,
6088                            &nbStripesSoFar, state->nbStripesPerBlock,
6089                             state->buffer, nbStripes,
6090                             secret, state->secretLimit,
6091                             XXH3_accumulate, XXH3_scrambleAcc);
6092         lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN;
6093     } else {  /* bufferedSize < XXH_STRIPE_LEN */
6094         /* Copy to temp buffer */
6095         size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
6096         XXH_ASSERT(state->bufferedSize > 0);  /* there is always some input buffered */
6097         XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
6098         XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
6099         lastStripePtr = lastStripe;
6100     }
6101     /* Last stripe */
6102     XXH3_accumulate_512(acc,
6103                         lastStripePtr,
6104                         secret + state->secretLimit - XXH_SECRET_LASTACC_START);
6105 }
6106 
6107 /*! @ingroup XXH3_family */
6108 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
6109 {
6110     const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6111     if (state->totalLen > XXH3_MIDSIZE_MAX) {
6112         XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
6113         XXH3_digest_long(acc, state, secret);
6114         return XXH3_mergeAccs(acc,
6115                               secret + XXH_SECRET_MERGEACCS_START,
6116                               (xxh_u64)state->totalLen * XXH_PRIME64_1);
6117     }
6118     /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
6119     if (state->useSeed)
6120         return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
6121     return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
6122                                   secret, state->secretLimit + XXH_STRIPE_LEN);
6123 }
6124 #endif /* !XXH_NO_STREAM */
6125 
6126 
6127 /* ==========================================
6128  * XXH3 128 bits (a.k.a XXH128)
6129  * ==========================================
6130  * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
6131  * even without counting the significantly larger output size.
6132  *
6133  * For example, extra steps are taken to avoid the seed-dependent collisions
6134  * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
6135  *
6136  * This strength naturally comes at the cost of some speed, especially on short
6137  * lengths. Note that longer hashes are about as fast as the 64-bit version
6138  * due to it using only a slight modification of the 64-bit loop.
6139  *
6140  * XXH128 is also more oriented towards 64-bit machines. It is still extremely
6141  * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
6142  */
6143 
6144 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6145 XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6146 {
6147     /* A doubled version of 1to3_64b with different constants. */
6148     XXH_ASSERT(input != NULL);
6149     XXH_ASSERT(1 <= len && len <= 3);
6150     XXH_ASSERT(secret != NULL);
6151     /*
6152      * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
6153      * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
6154      * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
6155      */
6156     {   xxh_u8 const c1 = input[0];
6157         xxh_u8 const c2 = input[len >> 1];
6158         xxh_u8 const c3 = input[len - 1];
6159         xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
6160                                 | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
6161         xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
6162         xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
6163         xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
6164         xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
6165         xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
6166         XXH128_hash_t h128;
6167         h128.low64  = XXH64_avalanche(keyed_lo);
6168         h128.high64 = XXH64_avalanche(keyed_hi);
6169         return h128;
6170     }
6171 }
6172 
6173 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6174 XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6175 {
6176     XXH_ASSERT(input != NULL);
6177     XXH_ASSERT(secret != NULL);
6178     XXH_ASSERT(4 <= len && len <= 8);
6179     seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
6180     {   xxh_u32 const input_lo = XXH_readLE32(input);
6181         xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
6182         xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
6183         xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
6184         xxh_u64 const keyed = input_64 ^ bitflip;
6185 
6186         /* Shift len to the left to ensure it is even, this avoids even multiplies. */
6187         XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
6188 
6189         m128.high64 += (m128.low64 << 1);
6190         m128.low64  ^= (m128.high64 >> 3);
6191 
6192         m128.low64   = XXH_xorshift64(m128.low64, 35);
6193         m128.low64  *= PRIME_MX2;
6194         m128.low64   = XXH_xorshift64(m128.low64, 28);
6195         m128.high64  = XXH3_avalanche(m128.high64);
6196         return m128;
6197     }
6198 }
6199 
6200 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6201 XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6202 {
6203     XXH_ASSERT(input != NULL);
6204     XXH_ASSERT(secret != NULL);
6205     XXH_ASSERT(9 <= len && len <= 16);
6206     {   xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
6207         xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
6208         xxh_u64 const input_lo = XXH_readLE64(input);
6209         xxh_u64       input_hi = XXH_readLE64(input + len - 8);
6210         XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
6211         /*
6212          * Put len in the middle of m128 to ensure that the length gets mixed to
6213          * both the low and high bits in the 128x64 multiply below.
6214          */
6215         m128.low64 += (xxh_u64)(len - 1) << 54;
6216         input_hi   ^= bitfliph;
6217         /*
6218          * Add the high 32 bits of input_hi to the high 32 bits of m128, then
6219          * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
6220          * the high 64 bits of m128.
6221          *
6222          * The best approach to this operation is different on 32-bit and 64-bit.
6223          */
6224         if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
6225             /*
6226              * 32-bit optimized version, which is more readable.
6227              *
6228              * On 32-bit, it removes an ADC and delays a dependency between the two
6229              * halves of m128.high64, but it generates an extra mask on 64-bit.
6230              */
6231             m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
6232         } else {
6233             /*
6234              * 64-bit optimized (albeit more confusing) version.
6235              *
6236              * Uses some properties of addition and multiplication to remove the mask:
6237              *
6238              * Let:
6239              *    a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
6240              *    b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
6241              *    c = XXH_PRIME32_2
6242              *
6243              *    a + (b * c)
6244              * Inverse Property: x + y - x == y
6245              *    a + (b * (1 + c - 1))
6246              * Distributive Property: x * (y + z) == (x * y) + (x * z)
6247              *    a + (b * 1) + (b * (c - 1))
6248              * Identity Property: x * 1 == x
6249              *    a + b + (b * (c - 1))
6250              *
6251              * Substitute a, b, and c:
6252              *    input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
6253              *
6254              * Since input_hi.hi + input_hi.lo == input_hi, we get this:
6255              *    input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
6256              */
6257             m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
6258         }
6259         /* m128 ^= XXH_swap64(m128 >> 64); */
6260         m128.low64  ^= XXH_swap64(m128.high64);
6261 
6262         {   /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
6263             XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
6264             h128.high64 += m128.high64 * XXH_PRIME64_2;
6265 
6266             h128.low64   = XXH3_avalanche(h128.low64);
6267             h128.high64  = XXH3_avalanche(h128.high64);
6268             return h128;
6269     }   }
6270 }
6271 
6272 /*
6273  * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
6274  */
6275 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6276 XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6277 {
6278     XXH_ASSERT(len <= 16);
6279     {   if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
6280         if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
6281         if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
6282         {   XXH128_hash_t h128;
6283             xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
6284             xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
6285             h128.low64 = XXH64_avalanche(seed ^ bitflipl);
6286             h128.high64 = XXH64_avalanche( seed ^ bitfliph);
6287             return h128;
6288     }   }
6289 }
6290 
6291 /*
6292  * A bit slower than XXH3_mix16B, but handles multiply by zero better.
6293  */
6294 XXH_FORCE_INLINE XXH128_hash_t
6295 XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
6296               const xxh_u8* secret, XXH64_hash_t seed)
6297 {
6298     acc.low64  += XXH3_mix16B (input_1, secret+0, seed);
6299     acc.low64  ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
6300     acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
6301     acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
6302     return acc;
6303 }
6304 
6305 
6306 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6307 XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
6308                       const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6309                       XXH64_hash_t seed)
6310 {
6311     XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
6312     XXH_ASSERT(16 < len && len <= 128);
6313 
6314     {   XXH128_hash_t acc;
6315         acc.low64 = len * XXH_PRIME64_1;
6316         acc.high64 = 0;
6317 
6318 #if XXH_SIZE_OPT >= 1
6319         {
6320             /* Smaller, but slightly slower. */
6321             unsigned int i = (unsigned int)(len - 1) / 32;
6322             do {
6323                 acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed);
6324             } while (i-- != 0);
6325         }
6326 #else
6327         if (len > 32) {
6328             if (len > 64) {
6329                 if (len > 96) {
6330                     acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
6331                 }
6332                 acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
6333             }
6334             acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
6335         }
6336         acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
6337 #endif
6338         {   XXH128_hash_t h128;
6339             h128.low64  = acc.low64 + acc.high64;
6340             h128.high64 = (acc.low64    * XXH_PRIME64_1)
6341                         + (acc.high64   * XXH_PRIME64_4)
6342                         + ((len - seed) * XXH_PRIME64_2);
6343             h128.low64  = XXH3_avalanche(h128.low64);
6344             h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
6345             return h128;
6346         }
6347     }
6348 }
6349 
6350 XXH_NO_INLINE XXH_PUREF XXH128_hash_t
6351 XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
6352                        const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6353                        XXH64_hash_t seed)
6354 {
6355     XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
6356     XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
6357 
6358     {   XXH128_hash_t acc;
6359         unsigned i;
6360         acc.low64 = len * XXH_PRIME64_1;
6361         acc.high64 = 0;
6362         /*
6363          *  We set as `i` as offset + 32. We do this so that unchanged
6364          * `len` can be used as upper bound. This reaches a sweet spot
6365          * where both x86 and aarch64 get simple agen and good codegen
6366          * for the loop.
6367          */
6368         for (i = 32; i < 160; i += 32) {
6369             acc = XXH128_mix32B(acc,
6370                                 input  + i - 32,
6371                                 input  + i - 16,
6372                                 secret + i - 32,
6373                                 seed);
6374         }
6375         acc.low64 = XXH3_avalanche(acc.low64);
6376         acc.high64 = XXH3_avalanche(acc.high64);
6377         /*
6378          * NB: `i <= len` will duplicate the last 32-bytes if
6379          * len % 32 was zero. This is an unfortunate necessity to keep
6380          * the hash result stable.
6381          */
6382         for (i=160; i <= len; i += 32) {
6383             acc = XXH128_mix32B(acc,
6384                                 input + i - 32,
6385                                 input + i - 16,
6386                                 secret + XXH3_MIDSIZE_STARTOFFSET + i - 160,
6387                                 seed);
6388         }
6389         /* last bytes */
6390         acc = XXH128_mix32B(acc,
6391                             input + len - 16,
6392                             input + len - 32,
6393                             secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
6394                             (XXH64_hash_t)0 - seed);
6395 
6396         {   XXH128_hash_t h128;
6397             h128.low64  = acc.low64 + acc.high64;
6398             h128.high64 = (acc.low64    * XXH_PRIME64_1)
6399                         + (acc.high64   * XXH_PRIME64_4)
6400                         + ((len - seed) * XXH_PRIME64_2);
6401             h128.low64  = XXH3_avalanche(h128.low64);
6402             h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
6403             return h128;
6404         }
6405     }
6406 }
6407 
6408 XXH_FORCE_INLINE XXH128_hash_t
6409 XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
6410                             const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6411                             XXH3_f_accumulate f_acc,
6412                             XXH3_f_scrambleAcc f_scramble)
6413 {
6414     XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
6415 
6416     XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble);
6417 
6418     /* converge into final hash */
6419     XXH_STATIC_ASSERT(sizeof(acc) == 64);
6420     XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
6421     {   XXH128_hash_t h128;
6422         h128.low64  = XXH3_mergeAccs(acc,
6423                                      secret + XXH_SECRET_MERGEACCS_START,
6424                                      (xxh_u64)len * XXH_PRIME64_1);
6425         h128.high64 = XXH3_mergeAccs(acc,
6426                                      secret + secretSize
6427                                             - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
6428                                      ~((xxh_u64)len * XXH_PRIME64_2));
6429         return h128;
6430     }
6431 }
6432 
6433 /*
6434  * It's important for performance that XXH3_hashLong() is not inlined.
6435  */
6436 XXH_NO_INLINE XXH_PUREF XXH128_hash_t
6437 XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
6438                            XXH64_hash_t seed64,
6439                            const void* XXH_RESTRICT secret, size_t secretLen)
6440 {
6441     (void)seed64; (void)secret; (void)secretLen;
6442     return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
6443                                        XXH3_accumulate, XXH3_scrambleAcc);
6444 }
6445 
6446 /*
6447  * It's important for performance to pass @p secretLen (when it's static)
6448  * to the compiler, so that it can properly optimize the vectorized loop.
6449  *
6450  * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
6451  * breaks -Og, this is XXH_NO_INLINE.
6452  */
6453 XXH3_WITH_SECRET_INLINE XXH128_hash_t
6454 XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
6455                               XXH64_hash_t seed64,
6456                               const void* XXH_RESTRICT secret, size_t secretLen)
6457 {
6458     (void)seed64;
6459     return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
6460                                        XXH3_accumulate, XXH3_scrambleAcc);
6461 }
6462 
6463 XXH_FORCE_INLINE XXH128_hash_t
6464 XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
6465                                 XXH64_hash_t seed64,
6466                                 XXH3_f_accumulate f_acc,
6467                                 XXH3_f_scrambleAcc f_scramble,
6468                                 XXH3_f_initCustomSecret f_initSec)
6469 {
6470     if (seed64 == 0)
6471         return XXH3_hashLong_128b_internal(input, len,
6472                                            XXH3_kSecret, sizeof(XXH3_kSecret),
6473                                            f_acc, f_scramble);
6474     {   XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
6475         f_initSec(secret, seed64);
6476         return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
6477                                            f_acc, f_scramble);
6478     }
6479 }
6480 
6481 /*
6482  * It's important for performance that XXH3_hashLong is not inlined.
6483  */
6484 XXH_NO_INLINE XXH128_hash_t
6485 XXH3_hashLong_128b_withSeed(const void* input, size_t len,
6486                             XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
6487 {
6488     (void)secret; (void)secretLen;
6489     return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
6490                 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
6491 }
6492 
6493 typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
6494                                             XXH64_hash_t, const void* XXH_RESTRICT, size_t);
6495 
6496 XXH_FORCE_INLINE XXH128_hash_t
6497 XXH3_128bits_internal(const void* input, size_t len,
6498                       XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
6499                       XXH3_hashLong128_f f_hl128)
6500 {
6501     XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
6502     /*
6503      * If an action is to be taken if `secret` conditions are not respected,
6504      * it should be done here.
6505      * For now, it's a contract pre-condition.
6506      * Adding a check and a branch here would cost performance at every hash.
6507      */
6508     if (len <= 16)
6509         return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
6510     if (len <= 128)
6511         return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6512     if (len <= XXH3_MIDSIZE_MAX)
6513         return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6514     return f_hl128(input, len, seed64, secret, secretLen);
6515 }
6516 
6517 
6518 /* ===   Public XXH128 API   === */
6519 
6520 /*! @ingroup XXH3_family */
6521 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len)
6522 {
6523     return XXH3_128bits_internal(input, len, 0,
6524                                  XXH3_kSecret, sizeof(XXH3_kSecret),
6525                                  XXH3_hashLong_128b_default);
6526 }
6527 
6528 /*! @ingroup XXH3_family */
6529 XXH_PUBLIC_API XXH128_hash_t
6530 XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize)
6531 {
6532     return XXH3_128bits_internal(input, len, 0,
6533                                  (const xxh_u8*)secret, secretSize,
6534                                  XXH3_hashLong_128b_withSecret);
6535 }
6536 
6537 /*! @ingroup XXH3_family */
6538 XXH_PUBLIC_API XXH128_hash_t
6539 XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
6540 {
6541     return XXH3_128bits_internal(input, len, seed,
6542                                  XXH3_kSecret, sizeof(XXH3_kSecret),
6543                                  XXH3_hashLong_128b_withSeed);
6544 }
6545 
6546 /*! @ingroup XXH3_family */
6547 XXH_PUBLIC_API XXH128_hash_t
6548 XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6549 {
6550     if (len <= XXH3_MIDSIZE_MAX)
6551         return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
6552     return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
6553 }
6554 
6555 /*! @ingroup XXH3_family */
6556 XXH_PUBLIC_API XXH128_hash_t
6557 XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
6558 {
6559     return XXH3_128bits_withSeed(input, len, seed);
6560 }
6561 
6562 
6563 /* ===   XXH3 128-bit streaming   === */
6564 #ifndef XXH_NO_STREAM
6565 /*
6566  * All initialization and update functions are identical to 64-bit streaming variant.
6567  * The only difference is the finalization routine.
6568  */
6569 
6570 /*! @ingroup XXH3_family */
6571 XXH_PUBLIC_API XXH_errorcode
6572 XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
6573 {
6574     return XXH3_64bits_reset(statePtr);
6575 }
6576 
6577 /*! @ingroup XXH3_family */
6578 XXH_PUBLIC_API XXH_errorcode
6579 XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
6580 {
6581     return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
6582 }
6583 
6584 /*! @ingroup XXH3_family */
6585 XXH_PUBLIC_API XXH_errorcode
6586 XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
6587 {
6588     return XXH3_64bits_reset_withSeed(statePtr, seed);
6589 }
6590 
6591 /*! @ingroup XXH3_family */
6592 XXH_PUBLIC_API XXH_errorcode
6593 XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6594 {
6595     return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
6596 }
6597 
6598 /*! @ingroup XXH3_family */
6599 XXH_PUBLIC_API XXH_errorcode
6600 XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
6601 {
6602     return XXH3_64bits_update(state, input, len);
6603 }
6604 
6605 /*! @ingroup XXH3_family */
6606 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
6607 {
6608     const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6609     if (state->totalLen > XXH3_MIDSIZE_MAX) {
6610         XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
6611         XXH3_digest_long(acc, state, secret);
6612         XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
6613         {   XXH128_hash_t h128;
6614             h128.low64  = XXH3_mergeAccs(acc,
6615                                          secret + XXH_SECRET_MERGEACCS_START,
6616                                          (xxh_u64)state->totalLen * XXH_PRIME64_1);
6617             h128.high64 = XXH3_mergeAccs(acc,
6618                                          secret + state->secretLimit + XXH_STRIPE_LEN
6619                                                 - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
6620                                          ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
6621             return h128;
6622         }
6623     }
6624     /* len <= XXH3_MIDSIZE_MAX : short code */
6625     if (state->seed)
6626         return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
6627     return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
6628                                    secret, state->secretLimit + XXH_STRIPE_LEN);
6629 }
6630 #endif /* !XXH_NO_STREAM */
6631 /* 128-bit utility functions */
6632 
6633 #include <string.h>   /* memcmp, memcpy */
6634 
6635 /* return : 1 is equal, 0 if different */
6636 /*! @ingroup XXH3_family */
6637 XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
6638 {
6639     /* note : XXH128_hash_t is compact, it has no padding byte */
6640     return !(memcmp(&h1, &h2, sizeof(h1)));
6641 }
6642 
6643 /* This prototype is compatible with stdlib's qsort().
6644  * @return : >0 if *h128_1  > *h128_2
6645  *           <0 if *h128_1  < *h128_2
6646  *           =0 if *h128_1 == *h128_2  */
6647 /*! @ingroup XXH3_family */
6648 XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2)
6649 {
6650     XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
6651     XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
6652     int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
6653     /* note : bets that, in most cases, hash values are different */
6654     if (hcmp) return hcmp;
6655     return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
6656 }
6657 
6658 
6659 /*======   Canonical representation   ======*/
6660 /*! @ingroup XXH3_family */
6661 XXH_PUBLIC_API void
6662 XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash)
6663 {
6664     XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
6665     if (XXH_CPU_LITTLE_ENDIAN) {
6666         hash.high64 = XXH_swap64(hash.high64);
6667         hash.low64  = XXH_swap64(hash.low64);
6668     }
6669     XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
6670     XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
6671 }
6672 
6673 /*! @ingroup XXH3_family */
6674 XXH_PUBLIC_API XXH128_hash_t
6675 XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src)
6676 {
6677     XXH128_hash_t h;
6678     h.high64 = XXH_readBE64(src);
6679     h.low64  = XXH_readBE64(src->digest + 8);
6680     return h;
6681 }
6682 
6683 
6684 
6685 /* ==========================================
6686  * Secret generators
6687  * ==========================================
6688  */
6689 #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
6690 
6691 XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128)
6692 {
6693     XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
6694     XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
6695 }
6696 
6697 /*! @ingroup XXH3_family */
6698 XXH_PUBLIC_API XXH_errorcode
6699 XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize)
6700 {
6701 #if (XXH_DEBUGLEVEL >= 1)
6702     XXH_ASSERT(secretBuffer != NULL);
6703     XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
6704 #else
6705     /* production mode, assert() are disabled */
6706     if (secretBuffer == NULL) return XXH_ERROR;
6707     if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
6708 #endif
6709 
6710     if (customSeedSize == 0) {
6711         customSeed = XXH3_kSecret;
6712         customSeedSize = XXH_SECRET_DEFAULT_SIZE;
6713     }
6714 #if (XXH_DEBUGLEVEL >= 1)
6715     XXH_ASSERT(customSeed != NULL);
6716 #else
6717     if (customSeed == NULL) return XXH_ERROR;
6718 #endif
6719 
6720     /* Fill secretBuffer with a copy of customSeed - repeat as needed */
6721     {   size_t pos = 0;
6722         while (pos < secretSize) {
6723             size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
6724             memcpy((char*)secretBuffer + pos, customSeed, toCopy);
6725             pos += toCopy;
6726     }   }
6727 
6728     {   size_t const nbSeg16 = secretSize / 16;
6729         size_t n;
6730         XXH128_canonical_t scrambler;
6731         XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
6732         for (n=0; n<nbSeg16; n++) {
6733             XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
6734             XXH3_combine16((char*)secretBuffer + n*16, h128);
6735         }
6736         /* last segment */
6737         XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
6738     }
6739     return XXH_OK;
6740 }
6741 
6742 /*! @ingroup XXH3_family */
6743 XXH_PUBLIC_API void
6744 XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
6745 {
6746     XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
6747     XXH3_initCustomSecret(secret, seed);
6748     XXH_ASSERT(secretBuffer != NULL);
6749     memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
6750 }
6751 
6752 
6753 
6754 /* Pop our optimization override from above */
6755 #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
6756   && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
6757   && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
6758 #  pragma GCC pop_options
6759 #endif
6760 
6761 #endif  /* XXH_NO_LONG_LONG */
6762 
6763 #endif  /* XXH_NO_XXH3 */
6764 
6765 /*!
6766  * @}
6767  */
6768 #endif  /* XXH_IMPLEMENTATION */
6769 
6770 
6771 #if defined (__cplusplus)
6772 } /* extern "C" */
6773 #endif