internal/mimalloc/mimalloc.h

0001 /* ----------------------------------------------------------------------------
0002 Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
0003 This is free software; you can redistribute it and/or modify it under the
0004 terms of the MIT license. A copy of the license can be found in the file
0005 "LICENSE" at the root of this distribution.
0006 -----------------------------------------------------------------------------*/
0007 #pragma once
0008 #ifndef MIMALLOC_H
0009 #define MIMALLOC_H
0010
0011 #define MI_MALLOC_VERSION 212   // major + 2 digits minor
0012
0013 // ------------------------------------------------------
0014 // Compiler specific attributes
0015 // ------------------------------------------------------
0016
0017 #ifdef __cplusplus
0018   #if (__cplusplus >= 201103L) || (_MSC_VER > 1900)  // C++11
0019     #define mi_attr_noexcept   noexcept
0020   #else
0021     #define mi_attr_noexcept   throw()
0022   #endif
0023 #else
0024   #define mi_attr_noexcept
0025 #endif
0026
0027 #if defined(__cplusplus) && (__cplusplus >= 201703)
0028   #define mi_decl_nodiscard    [[nodiscard]]
0029 #elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)  // includes clang, icc, and clang-cl
0030   #define mi_decl_nodiscard    __attribute__((warn_unused_result))
0031 #elif defined(_HAS_NODISCARD)
0032   #define mi_decl_nodiscard    _NODISCARD
0033 #elif (_MSC_VER >= 1700)
0034   #define mi_decl_nodiscard    _Check_return_
0035 #else
0036   #define mi_decl_nodiscard
0037 #endif
0038
0039 #if defined(_MSC_VER) || defined(__MINGW32__)
0040   #if !defined(MI_SHARED_LIB)
0041     #define mi_decl_export
0042   #elif defined(MI_SHARED_LIB_EXPORT)
0043     #define mi_decl_export              __declspec(dllexport)
0044   #else
0045     #define mi_decl_export              __declspec(dllimport)
0046   #endif
0047   #if defined(__MINGW32__)
0048     #define mi_decl_restrict
0049     #define mi_attr_malloc              __attribute__((malloc))
0050   #else
0051     #if (_MSC_VER >= 1900) && !defined(__EDG__)
0052       #define mi_decl_restrict          __declspec(allocator) __declspec(restrict)
0053     #else
0054       #define mi_decl_restrict          __declspec(restrict)
0055     #endif
0056     #define mi_attr_malloc
0057   #endif
0058   #define mi_cdecl                      __cdecl
0059   #define mi_attr_alloc_size(s)
0060   #define mi_attr_alloc_size2(s1,s2)
0061   #define mi_attr_alloc_align(p)
0062 #elif defined(__GNUC__)                 // includes clang and icc
0063   #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)
0064     #define mi_decl_export              __attribute__((visibility("default")))
0065   #else
0066     #define mi_decl_export
0067   #endif
0068   #define mi_cdecl                      // leads to warnings... __attribute__((cdecl))
0069   #define mi_decl_restrict
0070   #define mi_attr_malloc                __attribute__((malloc))
0071   #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
0072     #define mi_attr_alloc_size(s)
0073     #define mi_attr_alloc_size2(s1,s2)
0074     #define mi_attr_alloc_align(p)
0075   #elif defined(__INTEL_COMPILER)
0076     #define mi_attr_alloc_size(s)       __attribute__((alloc_size(s)))
0077     #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))
0078     #define mi_attr_alloc_align(p)
0079   #else
0080     #define mi_attr_alloc_size(s)       __attribute__((alloc_size(s)))
0081     #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))
0082     #define mi_attr_alloc_align(p)      __attribute__((alloc_align(p)))
0083   #endif
0084 #else
0085   #define mi_cdecl
0086   #define mi_decl_export
0087   #define mi_decl_restrict
0088   #define mi_attr_malloc
0089   #define mi_attr_alloc_size(s)
0090   #define mi_attr_alloc_size2(s1,s2)
0091   #define mi_attr_alloc_align(p)
0092 #endif
0093
0094 // ------------------------------------------------------
0095 // Includes
0096 // ------------------------------------------------------
0097
0098 #include <stddef.h>     // size_t
0099 #include <stdbool.h>    // bool
0100 #include <stdint.h>     // INTPTR_MAX
0101
0102 #ifdef __cplusplus
0103 extern "C" {
0104 #endif
0105
0106 // ------------------------------------------------------
0107 // Standard malloc interface
0108 // ------------------------------------------------------
0109
0110 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0111 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
0112 mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize)      mi_attr_noexcept mi_attr_alloc_size(2);
0113 mi_decl_export void* mi_expand(void* p, size_t newsize)                         mi_attr_noexcept mi_attr_alloc_size(2);
0114
0115 mi_decl_export void mi_free(void* p) mi_attr_noexcept;
0116 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;
0117 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
0118 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc;
0119
0120 // ------------------------------------------------------
0121 // Extended functionality
0122 // ------------------------------------------------------
0123 #define MI_SMALL_WSIZE_MAX  (128)
0124 #define MI_SMALL_SIZE_MAX   (MI_SMALL_WSIZE_MAX*sizeof(void*))
0125
0126 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0127 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0128 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size)       mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0129
0130 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
0131 mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size)        mi_attr_noexcept mi_attr_alloc_size2(2,3);
0132 mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize)                   mi_attr_noexcept mi_attr_alloc_size(2);
0133
0134 mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
0135 mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size)     mi_attr_noexcept;
0136
0137
0138 // ------------------------------------------------------
0139 // Internals
0140 // ------------------------------------------------------
0141
0142 typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
0143 mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;
0144
0145 typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);
0146 mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;
0147
0148 typedef void (mi_cdecl mi_error_fun)(int err, void* arg);
0149 mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);
0150
0151 mi_decl_export void mi_collect(bool force)    mi_attr_noexcept;
0152 mi_decl_export int  mi_version(void)          mi_attr_noexcept;
0153 mi_decl_export void mi_stats_reset(void)      mi_attr_noexcept;
0154 mi_decl_export void mi_stats_merge(void)      mi_attr_noexcept;
0155 mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept;  // backward compatibility: `out` is ignored and should be NULL
0156 mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
0157
0158 mi_decl_export void mi_process_init(void)     mi_attr_noexcept;
0159 mi_decl_export void mi_thread_init(void)      mi_attr_noexcept;
0160 mi_decl_export void mi_thread_done(void)      mi_attr_noexcept;
0161 mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
0162
0163 mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,
0164                                     size_t* current_rss, size_t* peak_rss,
0165                                     size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;
0166
0167 // -------------------------------------------------------------------------------------
0168 // Aligned allocation
0169 // Note that `alignment` always follows `size` for consistency with unaligned
0170 // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.
0171 // -------------------------------------------------------------------------------------
0172
0173 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
0174 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0175 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
0176 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0177 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3);
0178 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
0179 mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
0180 mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
0181
0182
0183 // -------------------------------------------------------------------------------------
0184 // Heaps: first-class, but can only allocate from the same thread that created it.
0185 // -------------------------------------------------------------------------------------
0186
0187 struct mi_heap_s;
0188 typedef struct mi_heap_s mi_heap_t;
0189
0190 mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);
0191 mi_decl_export void       mi_heap_delete(mi_heap_t* heap);
0192 mi_decl_export void       mi_heap_destroy(mi_heap_t* heap);
0193 mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
0194 mi_decl_export mi_heap_t* mi_heap_get_default(void);
0195 mi_decl_export mi_heap_t* mi_heap_get_backing(void);
0196 mi_decl_export void       mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept;
0197
0198 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
0199 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
0200 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
0201 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
0202 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
0203
0204 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize)              mi_attr_noexcept mi_attr_alloc_size(3);
0205 mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(3,4);
0206 mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize)             mi_attr_noexcept mi_attr_alloc_size(3);
0207
0208 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s)            mi_attr_noexcept mi_attr_malloc;
0209 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
0210 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc;
0211
0212 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
0213 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
0214 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
0215 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
0216 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
0217 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
0218 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
0219 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
0220
0221
0222 // --------------------------------------------------------------------------------
0223 // Zero initialized re-allocation.
0224 // Only valid on memory that was originally allocated with zero initialization too.
0225 // e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
0226 // see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
0227 // --------------------------------------------------------------------------------
0228
0229 mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize)                mi_attr_noexcept mi_attr_alloc_size(2);
0230 mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(2,3);
0231
0232 mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
0233 mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
0234 mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);
0235 mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);
0236
0237 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize)                mi_attr_noexcept mi_attr_alloc_size(3);
0238 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size)  mi_attr_noexcept mi_attr_alloc_size2(3,4);
0239
0240 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
0241 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
0242 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5);
0243 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4);
0244
0245
0246 // ------------------------------------------------------
0247 // Analysis
0248 // ------------------------------------------------------
0249
0250 mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
0251 mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
0252 mi_decl_export bool mi_check_owned(const void* p);
0253
0254 // An area of heap space contains blocks of a single size.
0255 typedef struct mi_heap_area_s {
0256   void*  blocks;      // start of the area containing heap blocks
0257   size_t reserved;    // bytes reserved for this area (virtual)
0258   size_t committed;   // current available bytes for this area
0259   size_t used;        // number of allocated blocks
0260   size_t block_size;  // size in bytes of each block
0261   size_t full_block_size; // size in bytes of a full block including padding and metadata.
0262 } mi_heap_area_t;
0263
0264 typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
0265
0266 mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg);
0267
0268 // Experimental
0269 mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
0270 mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
0271
0272 mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
0273 mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
0274
0275 mi_decl_export int  mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
0276 mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
0277
0278 mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;
0279
0280 // Experimental: heaps associated with specific memory arena's
0281 typedef int mi_arena_id_t;
0282 mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
0283 mi_decl_export int   mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
0284 mi_decl_export int   mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
0285 mi_decl_export bool  mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
0286
0287 #if MI_MALLOC_VERSION >= 182
0288 // Create a heap that only allocates in the specified arena
0289 mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
0290 #endif
0291
0292 // deprecated
0293 mi_decl_export int  mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
0294
0295
0296 // ------------------------------------------------------
0297 // Convenience
0298 // ------------------------------------------------------
0299
0300 #define mi_malloc_tp(tp)                ((tp*)mi_malloc(sizeof(tp)))
0301 #define mi_zalloc_tp(tp)                ((tp*)mi_zalloc(sizeof(tp)))
0302 #define mi_calloc_tp(tp,n)              ((tp*)mi_calloc(n,sizeof(tp)))
0303 #define mi_mallocn_tp(tp,n)             ((tp*)mi_mallocn(n,sizeof(tp)))
0304 #define mi_reallocn_tp(p,tp,n)          ((tp*)mi_reallocn(p,n,sizeof(tp)))
0305 #define mi_recalloc_tp(p,tp,n)          ((tp*)mi_recalloc(p,n,sizeof(tp)))
0306
0307 #define mi_heap_malloc_tp(hp,tp)        ((tp*)mi_heap_malloc(hp,sizeof(tp)))
0308 #define mi_heap_zalloc_tp(hp,tp)        ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
0309 #define mi_heap_calloc_tp(hp,tp,n)      ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
0310 #define mi_heap_mallocn_tp(hp,tp,n)     ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
0311 #define mi_heap_reallocn_tp(hp,p,tp,n)  ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))
0312 #define mi_heap_recalloc_tp(hp,p,tp,n)  ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))
0313
0314
0315 // ------------------------------------------------------
0316 // Options
0317 // ------------------------------------------------------
0318
0319 typedef enum mi_option_e {
0320   // stable options
0321   mi_option_show_errors,              // print error messages
0322   mi_option_show_stats,               // print statistics on termination
0323   mi_option_verbose,                  // print verbose messages
0324   // the following options are experimental (see src/options.h)
0325   mi_option_eager_commit,             // eager commit segments? (after `eager_commit_delay` segments) (=1)
0326   mi_option_arena_eager_commit,       // eager commit arenas? Use 2 to enable just on overcommit systems (=2)
0327   mi_option_purge_decommits,          // should a memory purge decommit (or only reset) (=1)
0328   mi_option_allow_large_os_pages,     // allow large (2MiB) OS pages, implies eager commit
0329   mi_option_reserve_huge_os_pages,    // reserve N huge OS pages (1GiB/page) at startup
0330   mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
0331   mi_option_reserve_os_memory,        // reserve specified amount of OS memory in an arena at startup
0332   mi_option_deprecated_segment_cache,
0333   mi_option_deprecated_page_reset,
0334   mi_option_abandoned_page_purge,     // immediately purge delayed purges on thread termination
0335   mi_option_deprecated_segment_reset,
0336   mi_option_eager_commit_delay,
0337   mi_option_purge_delay,              // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all.
0338   mi_option_use_numa_nodes,           // 0 = use all available numa nodes, otherwise use at most N nodes.
0339   mi_option_limit_os_alloc,           // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)
0340   mi_option_os_tag,                   // tag used for OS logging (macOS only for now)
0341   mi_option_max_errors,               // issue at most N error messages
0342   mi_option_max_warnings,             // issue at most N warning messages
0343   mi_option_max_segment_reclaim,
0344   mi_option_destroy_on_exit,          // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe.
0345   mi_option_arena_reserve,            // initial memory size in KiB for arena reservation (1GiB on 64-bit)
0346   mi_option_arena_purge_mult,
0347   mi_option_purge_extend_delay,
0348   _mi_option_last,
0349   // legacy option names
0350   mi_option_large_os_pages = mi_option_allow_large_os_pages,
0351   mi_option_eager_region_commit = mi_option_arena_eager_commit,
0352   mi_option_reset_decommits = mi_option_purge_decommits,
0353   mi_option_reset_delay = mi_option_purge_delay,
0354   mi_option_abandoned_page_reset = mi_option_abandoned_page_purge
0355 } mi_option_t;
0356
0357
0358 mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
0359 mi_decl_export void mi_option_enable(mi_option_t option);
0360 mi_decl_export void mi_option_disable(mi_option_t option);
0361 mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
0362 mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
0363
0364 mi_decl_nodiscard mi_decl_export long   mi_option_get(mi_option_t option);
0365 mi_decl_nodiscard mi_decl_export long   mi_option_get_clamp(mi_option_t option, long min, long max);
0366 mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);
0367 mi_decl_export void mi_option_set(mi_option_t option, long value);
0368 mi_decl_export void mi_option_set_default(mi_option_t option, long value);
0369
0370
0371 // -------------------------------------------------------------------------------------------------------
0372 // "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
0373 // (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
0374 // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing.
0375 // -------------------------------------------------------------------------------------------------------
0376
0377 mi_decl_export void  mi_cfree(void* p) mi_attr_noexcept;
0378 mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
0379 mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p)        mi_attr_noexcept;
0380 mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size)     mi_attr_noexcept;
0381 mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
0382
0383 mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size)   mi_attr_noexcept;
0384 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
0385 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size)  mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0386 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0387 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
0388
0389 mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
0390 mi_decl_nodiscard mi_decl_export int   mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept;
0391 mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
0392 mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
0393
0394 mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc;
0395 mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char*  mi_mbsdup(const unsigned char* s)  mi_attr_noexcept mi_attr_malloc;
0396 mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name)                      mi_attr_noexcept;
0397 mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept;
0398
0399 mi_decl_export void mi_free_size(void* p, size_t size)                           mi_attr_noexcept;
0400 mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
0401 mi_decl_export void mi_free_aligned(void* p, size_t alignment)                   mi_attr_noexcept;
0402
0403 // The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
0404 // (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
0405 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size)                   mi_attr_malloc mi_attr_alloc_size(1);
0406 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
0407 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size)           mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
0408 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
0409 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size)   mi_attr_malloc mi_attr_alloc_size2(1, 2);
0410 mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize)                mi_attr_alloc_size(2);
0411 mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
0412
0413 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size)                mi_attr_malloc mi_attr_alloc_size(2);
0414 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
0415
0416 #ifdef __cplusplus
0417 }
0418 #endif
0419
0420 // ---------------------------------------------------------------------------------------------
0421 // Implement the C++ std::allocator interface for use in STL containers.
0422 // (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)
0423 // ---------------------------------------------------------------------------------------------
0424 #ifdef __cplusplus
0425
0426 #include <cstddef>     // std::size_t
0427 #include <cstdint>     // PTRDIFF_MAX
0428 #if (__cplusplus >= 201103L) || (_MSC_VER > 1900)  // C++11
0429 #include <type_traits> // std::true_type
0430 #include <utility>     // std::forward
0431 #endif
0432
0433 template<class T> struct _mi_stl_allocator_common {
0434   typedef T                 value_type;
0435   typedef std::size_t       size_type;
0436   typedef std::ptrdiff_t    difference_type;
0437   typedef value_type&       reference;
0438   typedef value_type const& const_reference;
0439   typedef value_type*       pointer;
0440   typedef value_type const* const_pointer;
0441
0442   #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
0443   using propagate_on_container_copy_assignment = std::true_type;
0444   using propagate_on_container_move_assignment = std::true_type;
0445   using propagate_on_container_swap            = std::true_type;
0446   template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); }
0447   template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); }
0448   #else
0449   void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }
0450   void destroy(pointer p) { p->~value_type(); }
0451   #endif
0452
0453   size_type     max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }
0454   pointer       address(reference x) const        { return &x; }
0455   const_pointer address(const_reference x) const  { return &x; }
0456 };
0457
0458 template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> {
0459   using typename _mi_stl_allocator_common<T>::size_type;
0460   using typename _mi_stl_allocator_common<T>::value_type;
0461   using typename _mi_stl_allocator_common<T>::pointer;
0462   template <class U> struct rebind { typedef mi_stl_allocator<U> other; };
0463
0464   mi_stl_allocator()                                             mi_attr_noexcept = default;
0465   mi_stl_allocator(const mi_stl_allocator&)                      mi_attr_noexcept = default;
0466   template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { }
0467   mi_stl_allocator  select_on_container_copy_construction() const { return *this; }
0468   void              deallocate(T* p, size_type) { mi_free(p); }
0469
0470   #if (__cplusplus >= 201703L)  // C++17
0471   mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); }
0472   mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
0473   #else
0474   mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); }
0475   #endif
0476
0477   #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
0478   using is_always_equal = std::true_type;
0479   #endif
0480 };
0481
0482 template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; }
0483 template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
0484
0485
0486 #if (__cplusplus >= 201103L) || (_MSC_VER >= 1900)  // C++11
0487 #define MI_HAS_HEAP_STL_ALLOCATOR 1
0488
0489 #include <memory>      // std::shared_ptr
0490
0491 // Common base class for STL allocators in a specific heap
0492 template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
0493   using typename _mi_stl_allocator_common<T>::size_type;
0494   using typename _mi_stl_allocator_common<T>::value_type;
0495   using typename _mi_stl_allocator_common<T>::pointer;
0496
0497   _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp) { }    /* will not delete nor destroy the passed in heap */
0498
0499   #if (__cplusplus >= 201703L)  // C++17
0500   mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }
0501   mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
0502   #else
0503   mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }
0504   #endif
0505
0506   #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900))  // C++11
0507   using is_always_equal = std::false_type;
0508   #endif
0509
0510   void collect(bool force) { mi_heap_collect(this->heap.get(), force); }
0511   template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); }
0512
0513 protected:
0514   std::shared_ptr<mi_heap_t> heap;
0515   template<class U, bool D> friend struct _mi_heap_stl_allocator_common;
0516
0517   _mi_heap_stl_allocator_common() {
0518     mi_heap_t* hp = mi_heap_new();
0519     this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete));  /* calls heap_delete/destroy when the refcount drops to zero */
0520   }
0521   _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }
0522   template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { }
0523
0524 private:
0525   static void heap_delete(mi_heap_t* hp)  { if (hp != NULL) { mi_heap_delete(hp); } }
0526   static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }
0527 };
0528
0529 // STL allocator allocation in a specific heap
0530 template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> {
0531   using typename _mi_heap_stl_allocator_common<T, false>::size_type;
0532   mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called
0533   mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { }  // no delete nor destroy on the passed in heap
0534   template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { }
0535
0536   mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }
0537   void deallocate(T* p, size_type) { mi_free(p); }
0538   template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; };
0539 };
0540
0541 template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
0542 template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
0543
0544
0545 // STL allocator allocation in a specific heap, where `free` does nothing and
0546 // the heap is destroyed in one go on destruction -- use with care!
0547 template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> {
0548   using typename _mi_heap_stl_allocator_common<T, true>::size_type;
0549   mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called
0550   mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { }  // no delete nor destroy on the passed in heap
0551   template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { }
0552
0553   mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }
0554   void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }
0555   template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; };
0556 };
0557
0558 template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
0559 template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
0560
0561 #endif // C++11
0562
0563 #endif // __cplusplus
0564
0565 #endif