oneapi/tbb/scalable_allocator.h

0001 /*
0002     Copyright (c) 2005-2023 Intel Corporation
0003
0004     Licensed under the Apache License, Version 2.0 (the "License");
0005     you may not use this file except in compliance with the License.
0006     You may obtain a copy of the License at
0007
0008         http://www.apache.org/licenses/LICENSE-2.0
0009
0010     Unless required by applicable law or agreed to in writing, software
0011     distributed under the License is distributed on an "AS IS" BASIS,
0012     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0013     See the License for the specific language governing permissions and
0014     limitations under the License.
0015 */
0016
0017 #ifndef __TBB_scalable_allocator_H
0018 #define __TBB_scalable_allocator_H
0019
0020 #ifdef __cplusplus
0021 #include "oneapi/tbb/detail/_config.h"
0022 #include "oneapi/tbb/detail/_utils.h"
0023 #include "oneapi/tbb/detail/_namespace_injection.h"
0024 #include <cstdlib>
0025 #include <utility>
0026 #include <new> /* std::bad_alloc() */
0027 #else
0028 #include "oneapi/tbb/detail/_export.h"
0029 #include <stddef.h> /* Need ptrdiff_t and size_t from here. */
0030 #if !defined(_MSC_VER) || defined(__clang__)
0031 #include <stdint.h> /* Need intptr_t from here. */
0032 #endif
0033 #endif
0034
0035 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0036 #include <memory_resource>
0037 #endif
0038
0039 #ifdef __cplusplus
0040 extern "C" {
0041 #endif /* __cplusplus */
0042
0043 #if _MSC_VER
0044     #define __TBB_EXPORTED_FUNC __cdecl
0045 #else
0046     #define __TBB_EXPORTED_FUNC
0047 #endif
0048
0049 /** The "malloc" analogue to allocate block of memory of size bytes.
0050   * @ingroup memory_allocation */
0051 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size);
0052
0053 /** The "free" analogue to discard a previously allocated piece of memory.
0054     @ingroup memory_allocation */
0055 TBBMALLOC_EXPORT void   __TBB_EXPORTED_FUNC scalable_free(void* ptr);
0056
0057 /** The "realloc" analogue complementing scalable_malloc.
0058     @ingroup memory_allocation */
0059 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size);
0060
0061 /** The "calloc" analogue complementing scalable_malloc.
0062     @ingroup memory_allocation */
0063 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size);
0064
0065 /** The "posix_memalign" analogue.
0066     @ingroup memory_allocation */
0067 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size);
0068
0069 /** The "_aligned_malloc" analogue.
0070     @ingroup memory_allocation */
0071 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment);
0072
0073 /** The "_aligned_realloc" analogue.
0074     @ingroup memory_allocation */
0075 TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment);
0076
0077 /** The "_aligned_free" analogue.
0078     @ingroup memory_allocation */
0079 TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr);
0080
0081 /** The analogue of _msize/malloc_size/malloc_usable_size.
0082     Returns the usable size of a memory block previously allocated by scalable_*,
0083     or 0 (zero) if ptr does not point to such a block.
0084     @ingroup memory_allocation */
0085 TBBMALLOC_EXPORT size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr);
0086
0087 /* Results for scalable_allocation_* functions */
0088 typedef enum {
0089     TBBMALLOC_OK,
0090     TBBMALLOC_INVALID_PARAM,
0091     TBBMALLOC_UNSUPPORTED,
0092     TBBMALLOC_NO_MEMORY,
0093     TBBMALLOC_NO_EFFECT
0094 } ScalableAllocationResult;
0095
0096 /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages.
0097    scalable_allocation_mode call has priority over environment variable. */
0098 typedef enum {
0099     TBBMALLOC_USE_HUGE_PAGES,  /* value turns using huge pages on and off */
0100     /* deprecated, kept for backward compatibility only */
0101     USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES,
0102     /* try to limit memory consumption value (Bytes), clean internal buffers
0103        if limit is exceeded, but not prevents from requesting memory from OS */
0104     TBBMALLOC_SET_SOFT_HEAP_LIMIT,
0105     /* Lower bound for the size (Bytes), that is interpreted as huge
0106      * and not released during regular cleanup operations. */
0107     TBBMALLOC_SET_HUGE_SIZE_THRESHOLD
0108 } AllocationModeParam;
0109
0110 /** Set TBB allocator-specific allocation modes.
0111     @ingroup memory_allocation */
0112 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value);
0113
0114 typedef enum {
0115     /* Clean internal allocator buffers for all threads.
0116        Returns TBBMALLOC_NO_EFFECT if no buffers cleaned,
0117        TBBMALLOC_OK if some memory released from buffers. */
0118     TBBMALLOC_CLEAN_ALL_BUFFERS,
0119     /* Clean internal allocator buffer for current thread only.
0120        Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */
0121     TBBMALLOC_CLEAN_THREAD_BUFFERS
0122 } ScalableAllocationCmd;
0123
0124 /** Call TBB allocator-specific commands.
0125     @ingroup memory_allocation */
0126 TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param);
0127
0128 #ifdef __cplusplus
0129 } /* extern "C" */
0130 #endif /* __cplusplus */
0131
0132 #ifdef __cplusplus
0133
0134 //! The namespace rml contains components of low-level memory pool interface.
0135 namespace rml {
0136 class MemoryPool;
0137
0138 typedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes);
0139 // returns non-zero in case of error
0140 typedef int   (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes);
0141
0142 struct MemPoolPolicy {
0143     enum {
0144         TBBMALLOC_POOL_VERSION = 1
0145     };
0146
0147     rawAllocType pAlloc;
0148     rawFreeType  pFree;
0149                  // granularity of pAlloc allocations. 0 means default used.
0150     std::size_t  granularity;
0151     int          version;
0152                  // all memory consumed at 1st pAlloc call and never returned,
0153                  // no more pAlloc calls after 1st
0154     unsigned     fixedPool : 1,
0155                  // memory consumed but returned only at pool termination
0156                  keepAllMemory : 1,
0157                  reserved : 30;
0158
0159     MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_,
0160                   std::size_t granularity_ = 0, bool fixedPool_ = false,
0161                   bool keepAllMemory_ = false) :
0162         pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION),
0163         fixedPool(fixedPool_), keepAllMemory(keepAllMemory_),
0164         reserved(0) {}
0165 };
0166
0167 // enums have same values as appropriate enums from ScalableAllocationResult
0168 // TODO: use ScalableAllocationResult in pool_create directly
0169 enum MemPoolError {
0170     // pool created successfully
0171     POOL_OK = TBBMALLOC_OK,
0172     // invalid policy parameters found
0173     INVALID_POLICY = TBBMALLOC_INVALID_PARAM,
0174      // requested pool policy is not supported by allocator library
0175     UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED,
0176     // lack of memory during pool creation
0177     NO_MEMORY = TBBMALLOC_NO_MEMORY,
0178     // action takes no effect
0179     NO_EFFECT = TBBMALLOC_NO_EFFECT
0180 };
0181
0182 TBBMALLOC_EXPORT MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy,
0183                             rml::MemoryPool **pool);
0184
0185 TBBMALLOC_EXPORT bool  pool_destroy(MemoryPool* memPool);
0186 TBBMALLOC_EXPORT void *pool_malloc(MemoryPool* memPool, std::size_t size);
0187 TBBMALLOC_EXPORT void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size);
0188 TBBMALLOC_EXPORT void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment);
0189 TBBMALLOC_EXPORT void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment);
0190 TBBMALLOC_EXPORT bool  pool_reset(MemoryPool* memPool);
0191 TBBMALLOC_EXPORT bool  pool_free(MemoryPool *memPool, void *object);
0192 TBBMALLOC_EXPORT MemoryPool *pool_identify(void *object);
0193 TBBMALLOC_EXPORT std::size_t pool_msize(MemoryPool *memPool, void *object);
0194
0195 } // namespace rml
0196
0197 namespace tbb {
0198 namespace detail {
0199 namespace d1 {
0200
0201 // keep throw in a separate function to prevent code bloat
0202 template<typename E>
0203 void throw_exception(const E &e) {
0204 #if TBB_USE_EXCEPTIONS
0205     throw e;
0206 #else
0207     suppress_unused_warning(e);
0208 #endif
0209 }
0210
0211 template<typename T>
0212 class scalable_allocator {
0213 public:
0214     using value_type = T;
0215     using propagate_on_container_move_assignment = std::true_type;
0216
0217     //! Always defined for TBB containers
0218     using is_always_equal = std::true_type;
0219
0220     scalable_allocator() = default;
0221     template<typename U> scalable_allocator(const scalable_allocator<U>&) noexcept {}
0222
0223     //! Allocate space for n objects.
0224     __TBB_nodiscard T* allocate(std::size_t n) {
0225         T* p = static_cast<T*>(scalable_malloc(n * sizeof(value_type)));
0226         if (!p) {
0227             throw_exception(std::bad_alloc());
0228         }
0229         return p;
0230     }
0231
0232     //! Free previously allocated block of memory
0233     void deallocate(T* p, std::size_t) {
0234         scalable_free(p);
0235     }
0236
0237 #if TBB_ALLOCATOR_TRAITS_BROKEN
0238     using pointer = value_type*;
0239     using const_pointer = const value_type*;
0240     using reference = value_type&;
0241     using const_reference = const value_type&;
0242     using difference_type = std::ptrdiff_t;
0243     using size_type = std::size_t;
0244     template<typename U> struct rebind {
0245         using other = scalable_allocator<U>;
0246     };
0247     //! Largest value for which method allocate might succeed.
0248     size_type max_size() const noexcept {
0249         size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type);
0250         return (absolutemax > 0 ? absolutemax : 1);
0251     }
0252     template<typename U, typename... Args>
0253     void construct(U *p, Args&&... args)
0254         { ::new((void *)p) U(std::forward<Args>(args)...); }
0255     void destroy(pointer p) { p->~value_type(); }
0256     pointer address(reference x) const { return &x; }
0257     const_pointer address(const_reference x) const { return &x; }
0258 #endif // TBB_ALLOCATOR_TRAITS_BROKEN
0259
0260 };
0261
0262 #if TBB_ALLOCATOR_TRAITS_BROKEN
0263     template<>
0264     class scalable_allocator<void> {
0265     public:
0266         using pointer = void*;
0267         using const_pointer = const void*;
0268         using value_type = void;
0269         template<typename U> struct rebind {
0270             using other = scalable_allocator<U>;
0271         };
0272     };
0273 #endif
0274
0275 template<typename T, typename U>
0276 inline bool operator==(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return true; }
0277
0278 #if !__TBB_CPP20_COMPARISONS_PRESENT
0279 template<typename T, typename U>
0280 inline bool operator!=(const scalable_allocator<T>&, const scalable_allocator<U>&) noexcept { return false; }
0281 #endif
0282
0283 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0284
0285 //! C++17 memory resource implementation for scalable allocator
0286 //! ISO C++ Section 23.12.2
0287 class scalable_resource_impl : public std::pmr::memory_resource {
0288 private:
0289     void* do_allocate(std::size_t bytes, std::size_t alignment) override {
0290         void* p = scalable_aligned_malloc(bytes, alignment);
0291         if (!p) {
0292             throw_exception(std::bad_alloc());
0293         }
0294         return p;
0295     }
0296
0297     void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override {
0298         scalable_free(ptr);
0299     }
0300
0301     //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any
0302     //! other instance of this class
0303     bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
0304         return this == &other ||
0305 #if __TBB_USE_OPTIONAL_RTTI
0306             dynamic_cast<const scalable_resource_impl*>(&other) != nullptr;
0307 #else
0308             false;
0309 #endif
0310     }
0311 };
0312
0313 //! Global scalable allocator memory resource provider
0314 inline std::pmr::memory_resource* scalable_memory_resource() noexcept {
0315     static tbb::detail::d1::scalable_resource_impl scalable_res;
0316     return &scalable_res;
0317 }
0318
0319 #endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0320
0321 } // namespace d1
0322 } // namespace detail
0323
0324 inline namespace v1 {
0325 using detail::d1::scalable_allocator;
0326 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0327 using detail::d1::scalable_memory_resource;
0328 #endif
0329 } // namespace v1
0330
0331 } // namespace tbb
0332
0333 #endif /* __cplusplus */
0334
0335 #endif /* __TBB_scalable_allocator_H */