include/tbb/scalable_allocator.h

0001 /*
0002     Copyright (c) 2005-2020 Intel Corporation
0003
0004     Licensed under the Apache License, Version 2.0 (the "License");
0005     you may not use this file except in compliance with the License.
0006     You may obtain a copy of the License at
0007
0008         http://www.apache.org/licenses/LICENSE-2.0
0009
0010     Unless required by applicable law or agreed to in writing, software
0011     distributed under the License is distributed on an "AS IS" BASIS,
0012     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0013     See the License for the specific language governing permissions and
0014     limitations under the License.
0015 */
0016
0017 #ifndef __TBB_scalable_allocator_H
0018 #define __TBB_scalable_allocator_H
0019 /** @file */
0020
0021 #include <stddef.h> /* Need ptrdiff_t and size_t from here. */
0022 #if !_MSC_VER
0023 #include <stdint.h> /* Need intptr_t from here. */
0024 #endif
0025
0026 #if !defined(__cplusplus) && __ICC==1100
0027     #pragma warning (push)
0028     #pragma warning (disable: 991)
0029 #endif
0030
0031 #ifdef __cplusplus
0032 extern "C" {
0033 #endif /* __cplusplus */
0034
0035 #if _MSC_VER >= 1400
0036 #define __TBB_EXPORTED_FUNC   __cdecl
0037 #else
0038 #define __TBB_EXPORTED_FUNC
0039 #endif
0040
0041 /** The "malloc" analogue to allocate block of memory of size bytes.
0042   * @ingroup memory_allocation */
0043 void * __TBB_EXPORTED_FUNC scalable_malloc (size_t size);
0044
0045 /** The "free" analogue to discard a previously allocated piece of memory.
0046     @ingroup memory_allocation */
0047 void   __TBB_EXPORTED_FUNC scalable_free (void* ptr);
0048
0049 /** The "realloc" analogue complementing scalable_malloc.
0050     @ingroup memory_allocation */
0051 void * __TBB_EXPORTED_FUNC scalable_realloc (void* ptr, size_t size);
0052
0053 /** The "calloc" analogue complementing scalable_malloc.
0054     @ingroup memory_allocation */
0055 void * __TBB_EXPORTED_FUNC scalable_calloc (size_t nobj, size_t size);
0056
0057 /** The "posix_memalign" analogue.
0058     @ingroup memory_allocation */
0059 int __TBB_EXPORTED_FUNC scalable_posix_memalign (void** memptr, size_t alignment, size_t size);
0060
0061 /** The "_aligned_malloc" analogue.
0062     @ingroup memory_allocation */
0063 void * __TBB_EXPORTED_FUNC scalable_aligned_malloc (size_t size, size_t alignment);
0064
0065 /** The "_aligned_realloc" analogue.
0066     @ingroup memory_allocation */
0067 void * __TBB_EXPORTED_FUNC scalable_aligned_realloc (void* ptr, size_t size, size_t alignment);
0068
0069 /** The "_aligned_free" analogue.
0070     @ingroup memory_allocation */
0071 void __TBB_EXPORTED_FUNC scalable_aligned_free (void* ptr);
0072
0073 /** The analogue of _msize/malloc_size/malloc_usable_size.
0074     Returns the usable size of a memory block previously allocated by scalable_*,
0075     or 0 (zero) if ptr does not point to such a block.
0076     @ingroup memory_allocation */
0077 size_t __TBB_EXPORTED_FUNC scalable_msize (void* ptr);
0078
0079 /* Results for scalable_allocation_* functions */
0080 typedef enum {
0081     TBBMALLOC_OK,
0082     TBBMALLOC_INVALID_PARAM,
0083     TBBMALLOC_UNSUPPORTED,
0084     TBBMALLOC_NO_MEMORY,
0085     TBBMALLOC_NO_EFFECT
0086 } ScalableAllocationResult;
0087
0088 /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages.
0089    scalable_allocation_mode call has priority over environment variable. */
0090 typedef enum {
0091     TBBMALLOC_USE_HUGE_PAGES,  /* value turns using huge pages on and off */
0092     /* deprecated, kept for backward compatibility only */
0093     USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES,
0094     /* try to limit memory consumption value (Bytes), clean internal buffers
0095        if limit is exceeded, but not prevents from requesting memory from OS */
0096     TBBMALLOC_SET_SOFT_HEAP_LIMIT,
0097     /* Lower bound for the size (Bytes), that is interpreted as huge
0098      * and not released during regular cleanup operations. */
0099     TBBMALLOC_SET_HUGE_SIZE_THRESHOLD
0100 } AllocationModeParam;
0101
0102 /** Set TBB allocator-specific allocation modes.
0103     @ingroup memory_allocation */
0104 int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value);
0105
0106 typedef enum {
0107     /* Clean internal allocator buffers for all threads.
0108        Returns TBBMALLOC_NO_EFFECT if no buffers cleaned,
0109        TBBMALLOC_OK if some memory released from buffers. */
0110     TBBMALLOC_CLEAN_ALL_BUFFERS,
0111     /* Clean internal allocator buffer for current thread only.
0112        Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */
0113     TBBMALLOC_CLEAN_THREAD_BUFFERS
0114 } ScalableAllocationCmd;
0115
0116 /** Call TBB allocator-specific commands.
0117     @ingroup memory_allocation */
0118 int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param);
0119
0120 #ifdef __cplusplus
0121 } /* extern "C" */
0122 #endif /* __cplusplus */
0123
0124 #ifdef __cplusplus
0125
0126 //! The namespace rml contains components of low-level memory pool interface.
0127 namespace rml {
0128 class MemoryPool;
0129
0130 typedef void *(*rawAllocType)(intptr_t pool_id, size_t &bytes);
0131 // returns non-zero in case of error
0132 typedef int   (*rawFreeType)(intptr_t pool_id, void* raw_ptr, size_t raw_bytes);
0133
0134 /*
0135 MemPoolPolicy extension must be compatible with such structure fields layout
0136
0137 struct MemPoolPolicy {
0138     rawAllocType pAlloc;
0139     rawFreeType  pFree;
0140     size_t       granularity;   // granularity of pAlloc allocations
0141 };
0142 */
0143
0144 struct MemPoolPolicy {
0145     enum {
0146         TBBMALLOC_POOL_VERSION = 1
0147     };
0148
0149     rawAllocType pAlloc;
0150     rawFreeType  pFree;
0151                  // granularity of pAlloc allocations. 0 means default used.
0152     size_t       granularity;
0153     int          version;
0154                  // all memory consumed at 1st pAlloc call and never returned,
0155                  // no more pAlloc calls after 1st
0156     unsigned     fixedPool : 1,
0157                  // memory consumed but returned only at pool termination
0158                  keepAllMemory : 1,
0159                  reserved : 30;
0160
0161     MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_,
0162                   size_t granularity_ = 0, bool fixedPool_ = false,
0163                   bool keepAllMemory_ = false) :
0164         pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION),
0165         fixedPool(fixedPool_), keepAllMemory(keepAllMemory_),
0166         reserved(0) {}
0167 };
0168
0169 // enums have same values as appropriate enums from ScalableAllocationResult
0170 // TODO: use ScalableAllocationResult in pool_create directly
0171 enum MemPoolError {
0172     // pool created successfully
0173     POOL_OK = TBBMALLOC_OK,
0174     // invalid policy parameters found
0175     INVALID_POLICY = TBBMALLOC_INVALID_PARAM,
0176      // requested pool policy is not supported by allocator library
0177     UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED,
0178     // lack of memory during pool creation
0179     NO_MEMORY = TBBMALLOC_NO_MEMORY,
0180     // action takes no effect
0181     NO_EFFECT = TBBMALLOC_NO_EFFECT
0182 };
0183
0184 MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy,
0185                             rml::MemoryPool **pool);
0186
0187 bool  pool_destroy(MemoryPool* memPool);
0188 void *pool_malloc(MemoryPool* memPool, size_t size);
0189 void *pool_realloc(MemoryPool* memPool, void *object, size_t size);
0190 void *pool_aligned_malloc(MemoryPool* mPool, size_t size, size_t alignment);
0191 void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t alignment);
0192 bool  pool_reset(MemoryPool* memPool);
0193 bool  pool_free(MemoryPool *memPool, void *object);
0194 MemoryPool *pool_identify(void *object);
0195 size_t pool_msize(MemoryPool *memPool, void *object);
0196
0197 } // namespace rml
0198
0199 #include <new>      /* To use new with the placement argument */
0200
0201 /* Ensure that including this header does not cause implicit linkage with TBB */
0202 #ifndef __TBB_NO_IMPLICIT_LINKAGE
0203     #define __TBB_NO_IMPLICIT_LINKAGE 1
0204     #include "tbb_stddef.h"
0205     #undef  __TBB_NO_IMPLICIT_LINKAGE
0206 #else
0207     #include "tbb_stddef.h"
0208 #endif
0209
0210 #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
0211 #include <utility> // std::forward
0212 #endif
0213
0214 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0215 #include <memory_resource>
0216 #endif
0217
0218 namespace tbb {
0219
0220 #if _MSC_VER && !defined(__INTEL_COMPILER)
0221     // Workaround for erroneous "unreferenced parameter" warning in method destroy.
0222     #pragma warning (push)
0223     #pragma warning (disable: 4100)
0224 #endif
0225
0226 //! @cond INTERNAL
0227 namespace internal {
0228
0229 #if TBB_USE_EXCEPTIONS
0230 // forward declaration is for inlining prevention
0231 template<typename E> __TBB_NOINLINE( void throw_exception(const E &e) );
0232 #endif
0233
0234 // keep throw in a separate function to prevent code bloat
0235 template<typename E>
0236 void throw_exception(const E &e) {
0237     __TBB_THROW(e);
0238 }
0239
0240 } // namespace internal
0241 //! @endcond
0242
0243 //! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5
0244 /** The members are ordered the same way they are in section 20.4.1
0245     of the ISO C++ standard.
0246     @ingroup memory_allocation */
0247 template<typename T>
0248 class scalable_allocator {
0249 public:
0250     typedef typename internal::allocator_type<T>::value_type value_type;
0251     typedef value_type* pointer;
0252     typedef const value_type* const_pointer;
0253     typedef value_type& reference;
0254     typedef const value_type& const_reference;
0255     typedef size_t size_type;
0256     typedef ptrdiff_t difference_type;
0257     template<class U> struct rebind {
0258         typedef scalable_allocator<U> other;
0259     };
0260
0261     scalable_allocator() throw() {}
0262     scalable_allocator( const scalable_allocator& ) throw() {}
0263     template<typename U> scalable_allocator(const scalable_allocator<U>&) throw() {}
0264
0265     pointer address(reference x) const {return &x;}
0266     const_pointer address(const_reference x) const {return &x;}
0267
0268     //! Allocate space for n objects.
0269     pointer allocate( size_type n, const void* /*hint*/ =0 ) {
0270         pointer p = static_cast<pointer>( scalable_malloc( n * sizeof(value_type) ) );
0271         if (!p)
0272             internal::throw_exception(std::bad_alloc());
0273         return p;
0274     }
0275
0276     //! Free previously allocated block of memory
0277     void deallocate( pointer p, size_type ) {
0278         scalable_free( p );
0279     }
0280
0281     //! Largest value for which method allocate might succeed.
0282     size_type max_size() const throw() {
0283         size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type);
0284         return (absolutemax > 0 ? absolutemax : 1);
0285     }
0286 #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
0287     template<typename U, typename... Args>
0288     void construct(U *p, Args&&... args)
0289         { ::new((void *)p) U(std::forward<Args>(args)...); }
0290 #else /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */
0291 #if __TBB_CPP11_RVALUE_REF_PRESENT
0292     void construct( pointer p, value_type&& value ) { ::new((void*)(p)) value_type( std::move( value ) ); }
0293 #endif
0294     void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);}
0295 #endif /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */
0296     void destroy( pointer p ) {p->~value_type();}
0297 };
0298
0299 #if _MSC_VER && !defined(__INTEL_COMPILER)
0300     #pragma warning (pop)
0301 #endif /* warning 4100 is back */
0302
0303 //! Analogous to std::allocator<void>, as defined in ISO C++ Standard, Section 20.4.1
0304 /** @ingroup memory_allocation */
0305 template<>
0306 class scalable_allocator<void> {
0307 public:
0308     typedef void* pointer;
0309     typedef const void* const_pointer;
0310     typedef void value_type;
0311     template<class U> struct rebind {
0312         typedef scalable_allocator<U> other;
0313     };
0314 };
0315
0316 template<typename T, typename U>
0317 inline bool operator==( const scalable_allocator<T>&, const scalable_allocator<U>& ) {return true;}
0318
0319 template<typename T, typename U>
0320 inline bool operator!=( const scalable_allocator<T>&, const scalable_allocator<U>& ) {return false;}
0321
0322 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
0323
0324 namespace internal {
0325
0326 //! C++17 memory resource implementation for scalable allocator
0327 //! ISO C++ Section 23.12.2
0328 class scalable_resource_impl : public std::pmr::memory_resource {
0329 private:
0330     void* do_allocate(size_t bytes, size_t alignment) override {
0331         void* ptr = scalable_aligned_malloc( bytes, alignment );
0332         if (!ptr) {
0333             throw_exception(std::bad_alloc());
0334         }
0335         return ptr;
0336     }
0337
0338     void do_deallocate(void* ptr, size_t /*bytes*/, size_t /*alignment*/) override {
0339         scalable_free(ptr);
0340     }
0341
0342     //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any
0343     //! other instance of this class
0344     bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
0345         return this == &other ||
0346 #if __TBB_USE_OPTIONAL_RTTI
0347             dynamic_cast<const scalable_resource_impl*>(&other) != NULL;
0348 #else
0349             false;
0350 #endif
0351     }
0352 };
0353
0354 } // namespace internal
0355
0356 //! Global scalable allocator memory resource provider
0357 inline std::pmr::memory_resource* scalable_memory_resource() noexcept {
0358     static tbb::internal::scalable_resource_impl scalable_res;
0359     return &scalable_res;
0360 }
0361
0362 #endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */
0363
0364 } // namespace tbb
0365
0366 #if _MSC_VER
0367     #if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBBMALLOC_NO_IMPLICIT_LINKAGE)
0368         #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1
0369     #endif
0370
0371     #if !__TBBMALLOC_NO_IMPLICIT_LINKAGE
0372         #ifdef _DEBUG
0373             #pragma comment(lib, "tbbmalloc_debug.lib")
0374         #else
0375             #pragma comment(lib, "tbbmalloc.lib")
0376         #endif
0377     #endif
0378
0379
0380 #endif
0381
0382 #endif /* __cplusplus */
0383
0384 #if !defined(__cplusplus) && __ICC==1100
0385     #pragma warning (pop)
0386 #endif /* ICC 11.0 warning 991 is back */
0387
0388 #endif /* __TBB_scalable_allocator_H */