Vc/common/x86_prefetches.h

0001 /*  This file is part of the Vc library. {{{
0002 Copyright © 2013-2015 Matthias Kretz <kretz@kde.org>
0003
0004 Redistribution and use in source and binary forms, with or without
0005 modification, are permitted provided that the following conditions are met:
0006     * Redistributions of source code must retain the above copyright
0007       notice, this list of conditions and the following disclaimer.
0008     * Redistributions in binary form must reproduce the above copyright
0009       notice, this list of conditions and the following disclaimer in the
0010       documentation and/or other materials provided with the distribution.
0011     * Neither the names of contributing organizations nor the
0012       names of its contributors may be used to endorse or promote products
0013       derived from this software without specific prior written permission.
0014
0015 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
0016 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
0017 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
0018 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
0019 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
0020 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
0021 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
0022 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
0023 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
0024 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
0025
0026 }}}*/
0027
0028 #ifndef VC_COMMON_X86_PREFETCHES_H_
0029 #define VC_COMMON_X86_PREFETCHES_H_
0030
0031 #include <xmmintrin.h>
0032 #include "macros.h"
0033
0034 namespace Vc_VERSIONED_NAMESPACE
0035 {
0036 namespace Common
0037 {
0038
0039 static constexpr int exclusive_hint = 0;
0040
0041 // TODO: support AMD's prefetchw with correct flags and checks via cpuid
0042
0043 template <typename ExclusiveOrShared = Vc::Shared>
0044 Vc_INTRINSIC void prefetchForOneRead(const void *addr)
0045 {
0046     if (std::is_same<ExclusiveOrShared, Vc::Shared>::value) {
0047         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_NTA);
0048     } else {
0049         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)),
0050                      static_cast<decltype(_MM_HINT_NTA)>(_MM_HINT_NTA | exclusive_hint));
0051     }
0052 }
0053 template <typename ExclusiveOrShared = Vc::Shared>
0054 Vc_INTRINSIC void prefetchClose(const void *addr)
0055 {
0056     if (std::is_same<ExclusiveOrShared, Vc::Shared>::value) {
0057         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T0);
0058     } else {
0059         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)),
0060                      static_cast<decltype(_MM_HINT_T0)>(_MM_HINT_T0 | exclusive_hint));
0061     }
0062 }
0063 template <typename ExclusiveOrShared = Vc::Shared>
0064 Vc_INTRINSIC void prefetchMid(const void *addr)
0065 {
0066     if (std::is_same<ExclusiveOrShared, Vc::Shared>::value) {
0067         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T1);
0068     } else {
0069         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)),
0070                      static_cast<decltype(_MM_HINT_T1)>(_MM_HINT_T1 | exclusive_hint));
0071     }
0072 }
0073 template <typename ExclusiveOrShared = Vc::Shared>
0074 Vc_INTRINSIC void prefetchFar(const void *addr)
0075 {
0076     if (std::is_same<ExclusiveOrShared, Vc::Shared>::value) {
0077         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T2);
0078     } else {
0079         _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)),
0080                      static_cast<decltype(_MM_HINT_T2)>(_MM_HINT_T2 | exclusive_hint));
0081     }
0082 }
0083
0084 /*handlePrefetch/handleLoadPrefetches/handleStorePrefetches{{{*/
0085 namespace
0086 {
0087 template<size_t L1, size_t L2, bool UseExclusivePrefetch> Vc_INTRINSIC void handlePrefetch(const void *addr_, typename std::enable_if<L1 != 0 && L2 != 0, void *>::type = nullptr)
0088 {
0089     const char *addr = static_cast<const char *>(addr_);
0090     prefetchClose<typename std::conditional<UseExclusivePrefetch, Vc::Exclusive, Vc::Shared>::type>(addr + L1);
0091     prefetchMid  <typename std::conditional<UseExclusivePrefetch, Vc::Exclusive, Vc::Shared>::type>(addr + L2);
0092 }
0093 template<size_t L1, size_t L2, bool UseExclusivePrefetch> Vc_INTRINSIC void handlePrefetch(const void *addr_, typename std::enable_if<L1 == 0 && L2 != 0, void *>::type = nullptr)
0094 {
0095     const char *addr = static_cast<const char *>(addr_);
0096     prefetchMid  <typename std::conditional<UseExclusivePrefetch, Vc::Exclusive, Vc::Shared>::type>(addr + L2);
0097 }
0098 template<size_t L1, size_t L2, bool UseExclusivePrefetch> Vc_INTRINSIC void handlePrefetch(const void *addr_, typename std::enable_if<L1 != 0 && L2 == 0, void *>::type = nullptr)
0099 {
0100     const char *addr = static_cast<const char *>(addr_);
0101     prefetchClose<typename std::conditional<UseExclusivePrefetch, Vc::Exclusive, Vc::Shared>::type>(addr + L1);
0102 }
0103 template<size_t L1, size_t L2, bool UseExclusivePrefetch> Vc_INTRINSIC void handlePrefetch(const void *, typename std::enable_if<L1 == 0 && L2 == 0, void *>::type = nullptr)
0104 {
0105 }
0106
0107 template<typename Flags> Vc_INTRINSIC void handleLoadPrefetches(const void *    , Flags, typename Flags::EnableIfNotPrefetch = nullptr) {}
0108 template<typename Flags> Vc_INTRINSIC void handleLoadPrefetches(const void *addr, Flags, typename Flags::EnableIfPrefetch    = nullptr)
0109 {
0110     // load prefetches default to Shared unless Exclusive was explicitely selected
0111     handlePrefetch<Flags::L1Stride, Flags::L2Stride, Flags::IsExclusivePrefetch>(addr);
0112 }
0113
0114 template<typename Flags> Vc_INTRINSIC void handleStorePrefetches(const void *    , Flags, typename Flags::EnableIfNotPrefetch = nullptr) {}
0115 template<typename Flags> Vc_INTRINSIC void handleStorePrefetches(const void *addr, Flags, typename Flags::EnableIfPrefetch    = nullptr)
0116 {
0117     // store prefetches default to Exclusive unless Shared was explicitely selected
0118     handlePrefetch<Flags::L1Stride, Flags::L2Stride, !Flags::IsSharedPrefetch>(addr);
0119 }
0120
0121 } // anonymous namespace
0122 /*}}}*/
0123
0124 }  // namespace Common
0125
0126 using Common::prefetchForOneRead;
0127 using Common::prefetchClose;
0128 using Common::prefetchMid;
0129 using Common::prefetchFar;
0130 }  // namespace Vc
0131
0132 #endif // VC_COMMON_X86_PREFETCHES_H_
0133
0134 // vim: foldmethod=marker