root/ROOT/RHistUtils.hxx

0001 /// \file
0002 /// \warning This file contains implementation details that will change without notice. User code should never include
0003 /// this header directly.
0004
0005 #ifndef ROOT_RHistUtils
0006 #define ROOT_RHistUtils
0007
0008 #include <atomic>
0009 #include <type_traits>
0010
0011 #ifdef _MSC_VER
0012 #include <intrin.h>
0013 #endif
0014
0015 namespace ROOT {
0016 namespace Experimental {
0017 namespace Internal {
0018
0019 template <typename T, typename... Ts>
0020 struct LastType : LastType<Ts...> {};
0021 template <typename T>
0022 struct LastType<T> {
0023    using type = T;
0024 };
0025
0026 #ifdef _MSC_VER
0027 namespace MSVC {
0028 template <std::size_t N>
0029 struct AtomicOps {};
0030
0031 template <>
0032 struct AtomicOps<1> {
0033    static void Load(const void *ptr, void *ret)
0034    {
0035       *static_cast<char *>(ret) = __iso_volatile_load8(static_cast<const char *>(ptr));
0036    }
0037    static void Store(void *ptr, void *val)
0038    {
0039       __iso_volatile_store8(static_cast<char *>(ptr), *static_cast<char *>(val));
0040    }
0041    static void Add(void *ptr, const void *val)
0042    {
0043       _InterlockedExchangeAdd8(static_cast<char *>(ptr), *static_cast<const char *>(val));
0044    }
0045    static bool CompareExchange(void *ptr, void *expected, const void *desired)
0046    {
0047       // MSVC functions have the arguments reversed...
0048       const char expectedVal = *static_cast<char *>(expected);
0049       const char desiredVal = *static_cast<const char *>(desired);
0050       const char previous = _InterlockedCompareExchange8(static_cast<char *>(ptr), desiredVal, expectedVal);
0051       if (previous == expectedVal) {
0052          return true;
0053       }
0054       *static_cast<char *>(expected) = previous;
0055       return false;
0056    }
0057 };
0058
0059 template <>
0060 struct AtomicOps<2> {
0061    static void Load(const void *ptr, void *ret)
0062    {
0063       *static_cast<short *>(ret) = __iso_volatile_load16(static_cast<const short *>(ptr));
0064    }
0065    static void Store(void *ptr, void *val)
0066    {
0067       __iso_volatile_store16(static_cast<short *>(ptr), *static_cast<short *>(val));
0068    }
0069    static void Add(void *ptr, const void *val)
0070    {
0071       _InterlockedExchangeAdd16(static_cast<short *>(ptr), *static_cast<const short *>(val));
0072    }
0073    static bool CompareExchange(void *ptr, void *expected, const void *desired)
0074    {
0075       // MSVC functions have the arguments reversed...
0076       const short expectedVal = *static_cast<short *>(expected);
0077       const short desiredVal = *static_cast<const short *>(desired);
0078       const short previous = _InterlockedCompareExchange16(static_cast<short *>(ptr), desiredVal, expectedVal);
0079       if (previous == expectedVal) {
0080          return true;
0081       }
0082       *static_cast<short *>(expected) = previous;
0083       return false;
0084    }
0085 };
0086
0087 template <>
0088 struct AtomicOps<4> {
0089    static void Load(const void *ptr, void *ret)
0090    {
0091       *static_cast<int *>(ret) = __iso_volatile_load32(static_cast<const int *>(ptr));
0092    }
0093    static void Store(void *ptr, void *val)
0094    {
0095       __iso_volatile_store32(static_cast<int *>(ptr), *static_cast<int *>(val));
0096    }
0097    static void Add(void *ptr, const void *val)
0098    {
0099       _InterlockedExchangeAdd(static_cast<long *>(ptr), *static_cast<const long *>(val));
0100    }
0101    static bool CompareExchange(void *ptr, void *expected, const void *desired)
0102    {
0103       // MSVC functions have the arguments reversed...
0104       const long expectedVal = *static_cast<long *>(expected);
0105       const long desiredVal = *static_cast<const long *>(desired);
0106       const long previous = _InterlockedCompareExchange(static_cast<long *>(ptr), desiredVal, expectedVal);
0107       if (previous == expectedVal) {
0108          return true;
0109       }
0110       *static_cast<long *>(expected) = previous;
0111       return false;
0112    }
0113 };
0114
0115 template <>
0116 struct AtomicOps<8> {
0117    static void Load(const void *ptr, void *ret)
0118    {
0119       *static_cast<__int64 *>(ret) = __iso_volatile_load64(static_cast<const __int64 *>(ptr));
0120    }
0121    static void Store(void *ptr, void *val)
0122    {
0123       __iso_volatile_store64(static_cast<__int64 *>(ptr), *static_cast<__int64 *>(val));
0124    }
0125    static void Add(void *ptr, const void *val);
0126    static bool CompareExchange(void *ptr, void *expected, const void *desired)
0127    {
0128       // MSVC functions have the arguments reversed...
0129       const __int64 expectedVal = *static_cast<__int64 *>(expected);
0130       const __int64 desiredVal = *static_cast<const __int64 *>(desired);
0131       const __int64 previous = _InterlockedCompareExchange64(static_cast<__int64 *>(ptr), desiredVal, expectedVal);
0132       if (previous == expectedVal) {
0133          return true;
0134       }
0135       *static_cast<__int64 *>(expected) = previous;
0136       return false;
0137    }
0138 };
0139 } // namespace MSVC
0140 #endif
0141
0142 template <typename T>
0143 void AtomicLoad(const T *ptr, T *ret)
0144 {
0145 #ifndef _MSC_VER
0146    __atomic_load(ptr, ret, __ATOMIC_RELAXED);
0147 #else
0148    MSVC::AtomicOps<sizeof(T)>::Load(ptr, ret);
0149 #endif
0150 }
0151
0152 template <typename T>
0153 void AtomicStoreRelease(T *ptr, T *val)
0154 {
0155 #ifndef _MSC_VER
0156    __atomic_store(ptr, val, __ATOMIC_RELEASE);
0157 #else
0158    // Cannot specify the memory order directly, use a fence.
0159    std::atomic_thread_fence(std::memory_order_release);
0160    MSVC::AtomicOps<sizeof(T)>::Store(ptr, val);
0161 #endif
0162 }
0163
0164 template <typename T>
0165 bool AtomicCompareExchange(T *ptr, T *expected, T *desired)
0166 {
0167 #ifndef _MSC_VER
0168    return __atomic_compare_exchange(ptr, expected, desired, /*weak=*/false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
0169 #else
0170    return MSVC::AtomicOps<sizeof(T)>::CompareExchange(ptr, expected, desired);
0171 #endif
0172 }
0173
0174 template <typename T>
0175 bool AtomicCompareExchangeAcquire(T *ptr, T *expected, T *desired)
0176 {
0177 #ifndef _MSC_VER
0178    return __atomic_compare_exchange(ptr, expected, desired, /*weak=*/false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
0179 #else
0180    bool success = MSVC::AtomicOps<sizeof(T)>::CompareExchange(ptr, expected, desired);
0181    // Cannot specify the memory order directly, use an unconditional fence to avoid branching code.
0182    std::atomic_thread_fence(std::memory_order_acquire);
0183    return success;
0184 #endif
0185 }
0186
0187 template <typename T>
0188 void AtomicAddCompareExchangeLoop(T *ptr, T val)
0189 {
0190    T expected;
0191    AtomicLoad(ptr, &expected);
0192    T desired = expected + val;
0193    while (!AtomicCompareExchange(ptr, &expected, &desired)) {
0194       // expected holds the new value; try again.
0195       desired = expected + val;
0196    }
0197 }
0198
0199 #ifdef _MSC_VER
0200 namespace MSVC {
0201 inline void AtomicOps<8>::Add(void *ptr, const void *val)
0202 {
0203 #if _WIN64
0204    _InterlockedExchangeAdd64(static_cast<__int64 *>(ptr), *static_cast<const __int64 *>(val));
0205 #else
0206    AtomicAddCompareExchangeLoop(static_cast<__int64 *>(ptr), *static_cast<const __int64 *>(val));
0207 #endif
0208 }
0209 } // namespace MSVC
0210 #endif
0211
0212 template <typename T>
0213 std::enable_if_t<std::is_integral_v<T>> AtomicAdd(T *ptr, T val)
0214 {
0215 #ifndef _MSC_VER
0216    __atomic_fetch_add(ptr, val, __ATOMIC_RELAXED);
0217 #else
0218    MSVC::AtomicOps<sizeof(T)>::Add(ptr, &val);
0219 #endif
0220 }
0221
0222 template <typename T>
0223 std::enable_if_t<std::is_floating_point_v<T>> AtomicAdd(T *ptr, T val)
0224 {
0225    AtomicAddCompareExchangeLoop(ptr, val);
0226 }
0227
0228 // For adding a double-precision weight to a single-precision bin content type, cast the argument once before the
0229 // compare-exchange loop.
0230 static inline void AtomicAdd(float *ptr, double val)
0231 {
0232    AtomicAdd(ptr, static_cast<float>(val));
0233 }
0234
0235 template <typename T>
0236 std::enable_if_t<std::is_arithmetic_v<T>> AtomicInc(T *ptr)
0237 {
0238    AtomicAdd(ptr, static_cast<T>(1));
0239 }
0240
0241 template <typename T, typename U>
0242 auto AtomicAdd(T *ptr, const U &add) -> decltype(ptr->AtomicAdd(add))
0243 {
0244    return ptr->AtomicAdd(add);
0245 }
0246
0247 template <typename T>
0248 auto AtomicInc(T *ptr) -> decltype(ptr->AtomicInc())
0249 {
0250    return ptr->AtomicInc();
0251 }
0252
0253 } // namespace Internal
0254 } // namespace Experimental
0255 } // namespace ROOT
0256
0257 #endif