include/tbb/cache_aligned_allocator.h

0001 /*
0002     Copyright (c) 2005-2020 Intel Corporation
0003
0004     Licensed under the Apache License, Version 2.0 (the "License");
0005     you may not use this file except in compliance with the License.
0006     You may obtain a copy of the License at
0007
0008         http://www.apache.org/licenses/LICENSE-2.0
0009
0010     Unless required by applicable law or agreed to in writing, software
0011     distributed under the License is distributed on an "AS IS" BASIS,
0012     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0013     See the License for the specific language governing permissions and
0014     limitations under the License.
0015 */
0016
0017 #ifndef __TBB_cache_aligned_allocator_H
0018 #define __TBB_cache_aligned_allocator_H
0019
0020 #include <new>
0021 #include "tbb_stddef.h"
0022 #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
0023 #include <utility> // std::forward
0024 #endif
0025
0026 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0027 #include <memory_resource>
0028 #endif
0029
0030 namespace tbb {
0031
0032 //! @cond INTERNAL
0033 namespace internal {
0034     //! Cache/sector line size.
0035     /** @ingroup memory_allocation */
0036     size_t __TBB_EXPORTED_FUNC NFS_GetLineSize();
0037
0038     //! Allocate memory on cache/sector line boundary.
0039     /** @ingroup memory_allocation */
0040     void* __TBB_EXPORTED_FUNC NFS_Allocate( size_t n_element, size_t element_size, void* hint );
0041
0042     //! Free memory allocated by NFS_Allocate.
0043     /** Freeing a NULL pointer is allowed, but has no effect.
0044         @ingroup memory_allocation */
0045     void __TBB_EXPORTED_FUNC NFS_Free( void* );
0046 }
0047 //! @endcond
0048
0049 #if _MSC_VER && !defined(__INTEL_COMPILER)
0050     // Workaround for erroneous "unreferenced parameter" warning in method destroy.
0051     #pragma warning (push)
0052     #pragma warning (disable: 4100)
0053 #endif
0054
0055 //! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5
0056 /** The members are ordered the same way they are in section 20.4.1
0057     of the ISO C++ standard.
0058     @ingroup memory_allocation */
0059 template<typename T>
0060 class cache_aligned_allocator {
0061 public:
0062     typedef typename internal::allocator_type<T>::value_type value_type;
0063     typedef value_type* pointer;
0064     typedef const value_type* const_pointer;
0065     typedef value_type& reference;
0066     typedef const value_type& const_reference;
0067     typedef size_t size_type;
0068     typedef ptrdiff_t difference_type;
0069     template<typename U> struct rebind {
0070         typedef cache_aligned_allocator<U> other;
0071     };
0072     cache_aligned_allocator() throw() {}
0073     cache_aligned_allocator( const cache_aligned_allocator& ) throw() {}
0074     template<typename U> cache_aligned_allocator(const cache_aligned_allocator<U>&) throw() {}
0075
0076     pointer address(reference x) const {return &x;}
0077     const_pointer address(const_reference x) const {return &x;}
0078
0079     //! Allocate space for n objects, starting on a cache/sector line.
0080     pointer allocate( size_type n, const void* hint=0 ) {
0081         // The "hint" argument is always ignored in NFS_Allocate thus const_cast shouldn't hurt
0082         return pointer(internal::NFS_Allocate( n, sizeof(value_type), const_cast<void*>(hint) ));
0083     }
0084
0085     //! Free block of memory that starts on a cache line
0086     void deallocate( pointer p, size_type ) {
0087         internal::NFS_Free(p);
0088     }
0089
0090     //! Largest value for which method allocate might succeed.
0091     size_type max_size() const throw() {
0092         return (~size_t(0)-internal::NFS_MaxLineSize)/sizeof(value_type);
0093     }
0094
0095     //! Copy-construct value at location pointed to by p.
0096 #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
0097     template<typename U, typename... Args>
0098     void construct(U *p, Args&&... args)
0099         { ::new((void *)p) U(std::forward<Args>(args)...); }
0100 #else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
0101 #if __TBB_CPP11_RVALUE_REF_PRESENT
0102     void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));}
0103 #endif
0104     void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);}
0105 #endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
0106
0107     //! Destroy value at location pointed to by p.
0108     void destroy( pointer p ) {p->~value_type();}
0109 };
0110
0111 #if _MSC_VER && !defined(__INTEL_COMPILER)
0112     #pragma warning (pop)
0113 #endif // warning 4100 is back
0114
0115 //! Analogous to std::allocator<void>, as defined in ISO C++ Standard, Section 20.4.1
0116 /** @ingroup memory_allocation */
0117 template<>
0118 class cache_aligned_allocator<void> {
0119 public:
0120     typedef void* pointer;
0121     typedef const void* const_pointer;
0122     typedef void value_type;
0123     template<typename U> struct rebind {
0124         typedef cache_aligned_allocator<U> other;
0125     };
0126 };
0127
0128 template<typename T, typename U>
0129 inline bool operator==( const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>& ) {return true;}
0130
0131 template<typename T, typename U>
0132 inline bool operator!=( const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>& ) {return false;}
0133
0134 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0135
0136 //! C++17 memory resource wrapper to ensure cache line size alignment
0137 class cache_aligned_resource : public std::pmr::memory_resource {
0138 public:
0139     cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {}
0140     explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {}
0141
0142     std::pmr::memory_resource* upstream_resource() const {
0143         return m_upstream;
0144     }
0145
0146 private:
0147     //! We don't know what memory resource set. Use padding to guarantee alignment
0148     void* do_allocate(size_t bytes, size_t alignment) override {
0149         size_t cache_line_alignment = correct_alignment(alignment);
0150         uintptr_t base = (uintptr_t)m_upstream->allocate(correct_size(bytes) + cache_line_alignment);
0151         __TBB_ASSERT(base != 0, "Upstream resource returned NULL.");
0152 #if _MSC_VER && !defined(__INTEL_COMPILER)
0153     // unary minus operator applied to unsigned type, result still unsigned
0154     #pragma warning(push)
0155     #pragma warning(disable: 4146 4706)
0156 #endif
0157         // Round up to the next cache line (align the base address)
0158         uintptr_t result = (base + cache_line_alignment) & -cache_line_alignment;
0159 #if _MSC_VER && !defined(__INTEL_COMPILER)
0160     #pragma warning(pop)
0161 #endif
0162         // Record where block actually starts.
0163         ((uintptr_t*)result)[-1] = base;
0164         return (void*)result;
0165     }
0166
0167     void do_deallocate(void* ptr, size_t bytes, size_t alignment) override {
0168         if (ptr) {
0169             // Recover where block actually starts
0170             uintptr_t base = ((uintptr_t*)ptr)[-1];
0171             m_upstream->deallocate((void*)base, correct_size(bytes) + correct_alignment(alignment));
0172         }
0173     }
0174
0175     bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
0176         if (this == &other) { return true; }
0177 #if __TBB_USE_OPTIONAL_RTTI
0178         const cache_aligned_resource* other_res = dynamic_cast<const cache_aligned_resource*>(&other);
0179         return other_res && (this->upstream_resource() == other_res->upstream_resource());
0180 #else
0181         return false;
0182 #endif
0183     }
0184
0185     size_t correct_alignment(size_t alignment) {
0186         __TBB_ASSERT(tbb::internal::is_power_of_two(alignment), "Alignment is not a power of 2");
0187 #if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT
0188         size_t cache_line_size = std::hardware_destructive_interference_size;
0189 #else
0190         size_t cache_line_size = internal::NFS_GetLineSize();
0191 #endif
0192         return alignment < cache_line_size ? cache_line_size : alignment;
0193     }
0194
0195     size_t correct_size(size_t bytes) {
0196         // To handle the case, when small size requested. There could be not
0197         // enough space to store the original pointer.
0198         return bytes < sizeof(uintptr_t) ? sizeof(uintptr_t) : bytes;
0199     }
0200
0201     std::pmr::memory_resource* m_upstream;
0202 };
0203
0204 #endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */
0205
0206 } // namespace tbb
0207
0208 #endif /* __TBB_cache_aligned_allocator_H */
0209