23#pragma clang diagnostic ignored "-Wunused-macros"
24#pragma clang diagnostic ignored "-Wunused-function"
25#if __has_warning("-Wreserved-identifier")
26#pragma clang diagnostic ignored "-Wreserved-identifier"
28#if __has_warning("-Wstatic-in-inline")
29#pragma clang diagnostic ignored "-Wstatic-in-inline"
31#elif defined(__GNUC__)
32#pragma GCC diagnostic ignored "-Wunused-macros"
33#pragma GCC diagnostic ignored "-Wunused-function"
36#if !defined(__has_builtin)
37#define __has_builtin(b) 0
40#if defined(__GNUC__) || defined(__clang__)
42#if __has_builtin(__builtin_memcpy_inline)
43#define _rpmalloc_memcpy_const(x, y, s) __builtin_memcpy_inline(x, y, s)
45#define _rpmalloc_memcpy_const(x, y, s) \
47 _Static_assert(__builtin_choose_expr(__builtin_constant_p(s), 1, 0), \
48 "len must be a constant integer"); \
53#if __has_builtin(__builtin_memset_inline)
54#define _rpmalloc_memset_const(x, y, s) __builtin_memset_inline(x, y, s)
56#define _rpmalloc_memset_const(x, y, s) \
58 _Static_assert(__builtin_choose_expr(__builtin_constant_p(s), 1, 0), \
59 "len must be a constant integer"); \
64#define _rpmalloc_memcpy_const(x, y, s) memcpy(x, y, s)
65#define _rpmalloc_memset_const(x, y, s) memset(x, y, s)
68#if __has_builtin(__builtin_assume)
69#define rpmalloc_assume(cond) __builtin_assume(cond)
70#elif defined(__GNUC__)
71#define rpmalloc_assume(cond) \
73 if (!__builtin_expect(cond, 0)) \
74 __builtin_unreachable(); \
76#elif defined(_MSC_VER)
77#define rpmalloc_assume(cond) __assume(cond)
79#define rpmalloc_assume(cond) 0
82#ifndef HEAP_ARRAY_SIZE
84#define HEAP_ARRAY_SIZE 47
86#ifndef ENABLE_THREAD_CACHE
88#define ENABLE_THREAD_CACHE 1
90#ifndef ENABLE_GLOBAL_CACHE
92#define ENABLE_GLOBAL_CACHE 1
94#ifndef ENABLE_VALIDATE_ARGS
96#define ENABLE_VALIDATE_ARGS 0
98#ifndef ENABLE_STATISTICS
100#define ENABLE_STATISTICS 0
102#ifndef ENABLE_ASSERTS
104#define ENABLE_ASSERTS 0
106#ifndef ENABLE_OVERRIDE
108#define ENABLE_OVERRIDE 0
110#ifndef ENABLE_PRELOAD
112#define ENABLE_PRELOAD 0
116#define DISABLE_UNMAP 0
118#ifndef ENABLE_UNLIMITED_CACHE
120#define ENABLE_UNLIMITED_CACHE 0
122#ifndef ENABLE_ADAPTIVE_THREAD_CACHE
124#define ENABLE_ADAPTIVE_THREAD_CACHE 0
126#ifndef DEFAULT_SPAN_MAP_COUNT
129#define DEFAULT_SPAN_MAP_COUNT 64
131#ifndef GLOBAL_CACHE_MULTIPLIER
133#define GLOBAL_CACHE_MULTIPLIER 8
136#if DISABLE_UNMAP && !ENABLE_GLOBAL_CACHE
137#error Must use global cache if unmap is disabled
141#undef ENABLE_UNLIMITED_CACHE
142#define ENABLE_UNLIMITED_CACHE 1
145#if !ENABLE_GLOBAL_CACHE
146#undef ENABLE_UNLIMITED_CACHE
147#define ENABLE_UNLIMITED_CACHE 0
150#if !ENABLE_THREAD_CACHE
151#undef ENABLE_ADAPTIVE_THREAD_CACHE
152#define ENABLE_ADAPTIVE_THREAD_CACHE 0
155#if defined(_WIN32) || defined(__WIN32__) || defined(_WIN64)
156#define PLATFORM_WINDOWS 1
157#define PLATFORM_POSIX 0
159#define PLATFORM_WINDOWS 0
160#define PLATFORM_POSIX 1
164#if defined(_MSC_VER) && !defined(__clang__)
165#pragma warning(disable : 5105)
167#define FORCEINLINE inline __forceinline
169#define _Static_assert static_assert
172#define FORCEINLINE inline __attribute__((__always_inline__))
176#ifndef WIN32_LEAN_AND_MEAN
177#define WIN32_LEAN_AND_MEAN
180#if ENABLE_VALIDATE_ARGS
188#if defined(__linux__) || defined(__ANDROID__)
189#include <sys/prctl.h>
190#if !defined(PR_SET_VMA)
191#define PR_SET_VMA 0x53564d41
192#define PR_SET_VMA_ANON_NAME 0
195#if defined(__APPLE__)
196#include <TargetConditionals.h>
197#if !TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
198#include <mach/mach_vm.h>
199#include <mach/vm_statistics.h>
203#if defined(__HAIKU__) || defined(__TINYC__)
212#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
213#include <fibersapi.h>
221#include <sys/sysctl.h>
222#define MAP_HUGETLB MAP_ALIGNED_SUPER
230extern int madvise(caddr_t,
size_t,
int);
232#ifndef MAP_UNINITIALIZED
233#define MAP_UNINITIALIZED 0
240#if defined(_MSC_VER) && !defined(_DEBUG)
244#define RPMALLOC_TOSTRING_M(x) #x
245#define RPMALLOC_TOSTRING(x) RPMALLOC_TOSTRING_M(x)
246#define rpmalloc_assert(truth, message) \
249 if (_memory_config.error_callback) { \
250 _memory_config.error_callback(message " (" RPMALLOC_TOSTRING( \
251 truth) ") at " __FILE__ ":" RPMALLOC_TOSTRING(__LINE__)); \
253 assert((truth) && message); \
258#define rpmalloc_assert(truth, message) \
272#if defined(_MSC_VER) && !defined(__clang__)
274typedef volatile long atomic32_t;
275typedef volatile long long atomic64_t;
276typedef volatile void *atomicptr_t;
278static FORCEINLINE int32_t atomic_load32(atomic32_t *src) {
279 return (int32_t)InterlockedOr(src, 0);
282 InterlockedExchange(dst, val);
285 return (int32_t)InterlockedIncrement(val);
288 return (int32_t)InterlockedDecrement(val);
291 return (int32_t)InterlockedExchangeAdd(val, add) +
add;
295 return (InterlockedCompareExchange(dst, val, ref) == ref) ? 1 : 0;
298 InterlockedExchange(dst, val);
301 return (int64_t)InterlockedOr64(src, 0);
304 return (int64_t)InterlockedExchangeAdd64(val, add) +
add;
307 return InterlockedCompareExchangePointer(src, 0, 0);
310 InterlockedExchangePointer(dst, val);
313 InterlockedExchangePointer(dst, val);
317 return (
void *)InterlockedExchangePointer((
void *
volatile *)dst, val);
320 return (InterlockedCompareExchangePointer((
void *
volatile *)dst, val, ref) ==
326#define EXPECTED(x) (x)
327#define UNEXPECTED(x) (x)
331#include <stdatomic.h>
334typedef volatile _Atomic(int64_t) atomic64_t;
335typedef volatile _Atomic(
void *) atomicptr_t;
337static FORCEINLINE int32_t atomic_load32(atomic32_t *src) {
338 return atomic_load_explicit(src, memory_order_relaxed);
341 atomic_store_explicit(dst, val, memory_order_relaxed);
344 return atomic_fetch_add_explicit(val, 1, memory_order_relaxed) + 1;
347 return atomic_fetch_add_explicit(val, -1, memory_order_relaxed) - 1;
350 return atomic_fetch_add_explicit(val, add, memory_order_relaxed) + add;
354 return atomic_compare_exchange_weak_explicit(
355 dst, &ref, val, memory_order_acquire, memory_order_relaxed);
358 atomic_store_explicit(dst, val, memory_order_release);
361 return atomic_load_explicit(val, memory_order_relaxed);
364 return atomic_fetch_add_explicit(val, add, memory_order_relaxed) + add;
367 return atomic_load_explicit(src, memory_order_relaxed);
370 atomic_store_explicit(dst, val, memory_order_relaxed);
373 atomic_store_explicit(dst, val, memory_order_release);
377 return atomic_exchange_explicit(dst, val, memory_order_acquire);
380 return atomic_compare_exchange_weak_explicit(
381 dst, &ref, val, memory_order_relaxed, memory_order_relaxed);
384#define EXPECTED(x) __builtin_expect((x), 1)
385#define UNEXPECTED(x) __builtin_expect((x), 0)
397#define _rpmalloc_stat_inc(counter) atomic_incr32(counter)
398#define _rpmalloc_stat_dec(counter) atomic_decr32(counter)
399#define _rpmalloc_stat_add(counter, value) \
400 atomic_add32(counter, (int32_t)(value))
401#define _rpmalloc_stat_add64(counter, value) \
402 atomic_add64(counter, (int64_t)(value))
403#define _rpmalloc_stat_add_peak(counter, value, peak) \
405 int32_t _cur_count = atomic_add32(counter, (int32_t)(value)); \
406 if (_cur_count > (peak)) \
409#define _rpmalloc_stat_sub(counter, value) \
410 atomic_add32(counter, -(int32_t)(value))
411#define _rpmalloc_stat_inc_alloc(heap, class_idx) \
413 int32_t alloc_current = \
414 atomic_incr32(&heap->size_class_use[class_idx].alloc_current); \
415 if (alloc_current > heap->size_class_use[class_idx].alloc_peak) \
416 heap->size_class_use[class_idx].alloc_peak = alloc_current; \
417 atomic_incr32(&heap->size_class_use[class_idx].alloc_total); \
419#define _rpmalloc_stat_inc_free(heap, class_idx) \
421 atomic_decr32(&heap->size_class_use[class_idx].alloc_current); \
422 atomic_incr32(&heap->size_class_use[class_idx].free_total); \
425#define _rpmalloc_stat_inc(counter) \
428#define _rpmalloc_stat_dec(counter) \
431#define _rpmalloc_stat_add(counter, value) \
434#define _rpmalloc_stat_add64(counter, value) \
437#define _rpmalloc_stat_add_peak(counter, value, peak) \
440#define _rpmalloc_stat_sub(counter, value) \
443#define _rpmalloc_stat_inc_alloc(heap, class_idx) \
446#define _rpmalloc_stat_inc_free(heap, class_idx) \
456#define SMALL_GRANULARITY 16
458#define SMALL_GRANULARITY_SHIFT 4
460#define SMALL_CLASS_COUNT 65
462#define SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1))
464#define MEDIUM_GRANULARITY 512
466#define MEDIUM_GRANULARITY_SHIFT 9
468#define MEDIUM_CLASS_COUNT 61
470#define SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT)
472#define LARGE_CLASS_COUNT 63
474#define MEDIUM_SIZE_LIMIT \
475 (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT))
477#define LARGE_SIZE_LIMIT \
478 ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE)
481#define SPAN_HEADER_SIZE 128
483#define MAX_THREAD_SPAN_CACHE 400
485#define THREAD_SPAN_CACHE_TRANSFER 64
488#define MAX_THREAD_SPAN_LARGE_CACHE 100
490#define THREAD_SPAN_LARGE_CACHE_TRANSFER 6
493 "Small granularity must be power of two");
495 "Span header size must be power of two");
497#if ENABLE_VALIDATE_ARGS
500#define MAX_ALLOC_SIZE (((size_t) - 1) - _memory_span_size)
503#define pointer_offset(ptr, ofs) (void *)((char *)(ptr) + (ptrdiff_t)(ofs))
504#define pointer_diff(first, second) \
505 (ptrdiff_t)((const char *)(first) - (const char *)(second))
507#define INVALID_POINTER ((void *)((uintptr_t) - 1))
509#define SIZE_CLASS_LARGE SIZE_CLASS_COUNT
510#define SIZE_CLASS_HUGE ((uint32_t) - 1)
532#define SPAN_FLAG_MASTER 1U
534#define SPAN_FLAG_SUBSPAN 2U
536#define SPAN_FLAG_ALIGNED_BLOCKS 4U
538#define SPAN_FLAG_UNMAPPED_MASTER 8U
540#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
548 atomic32_t spans_deferred;
550 atomic32_t spans_to_global;
552 atomic32_t spans_from_global;
554 atomic32_t spans_to_cache;
556 atomic32_t spans_from_cache;
558 atomic32_t spans_to_reserved;
560 atomic32_t spans_from_reserved;
562 atomic32_t spans_map_calls;
565typedef struct span_use_t span_use_t;
569struct size_class_use_t {
571 atomic32_t alloc_current;
575 atomic32_t alloc_total;
577 atomic32_t free_total;
579 atomic32_t spans_current;
583 atomic32_t spans_to_cache;
585 atomic32_t spans_from_cache;
587 atomic32_t spans_from_reserved;
589 atomic32_t spans_map_calls;
592typedef struct size_class_use_t size_class_use_t;
675#if ENABLE_THREAD_CACHE
701#if ENABLE_THREAD_CACHE
705#if RPMALLOC_FIRST_CLASS_HEAPS
713#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
721 atomic64_t thread_to_global;
723 atomic64_t global_to_thread;
747 size_t extract_count;
762#define _memory_default_span_size (64 * 1024)
763#define _memory_default_span_size_shift 16
764#define _memory_default_span_mask (~((uintptr_t)(_memory_span_size - 1)))
778#if RPMALLOC_CONFIGURABLE
787#define _memory_span_size _memory_default_span_size
788#define _memory_span_size_shift _memory_default_span_size_shift
789#define _memory_span_mask _memory_default_span_mask
803#if ENABLE_GLOBAL_CACHE
819#if RPMALLOC_FIRST_CLASS_HEAPS
821static heap_t *_memory_first_class_orphan_heaps;
825static atomic64_t _allocation_counter;
827static atomic64_t _deallocation_counter;
829static atomic32_t _memory_active_heaps;
831static atomic32_t _mapped_pages;
833static int32_t _mapped_pages_peak;
835static atomic32_t _master_spans;
837static atomic32_t _unmapped_master_spans;
839static atomic32_t _mapped_total;
841static atomic32_t _unmapped_total;
843static atomic32_t _mapped_pages_os;
845static atomic32_t _huge_pages_current;
847static int32_t _huge_pages_peak;
857#if ((defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD) || \
859static pthread_key_t _memory_thread_heap;
862#define _Thread_local __declspec(thread)
866#define TLS_MODEL __attribute__((tls_model("initial-exec")))
870#if !defined(__clang__) && defined(__GNUC__)
871#define _Thread_local __thread
878#if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
879 return pthread_getspecific(_memory_thread_heap);
881 return _memory_thread_heap;
901 return (uintptr_t)((
void *)NtCurrentTeb());
902#elif (defined(__GNUC__) || defined(__clang__)) && !defined(__CYGWIN__)
905 __asm__(
"movl %%gs:0, %0" :
"=r"(tid) : :);
906#elif defined(__x86_64__)
908 __asm__(
"movq %%gs:0, %0" :
"=r"(tid) : :);
910 __asm__(
"movq %%fs:0, %0" :
"=r"(tid) : :);
912#elif defined(__arm__)
913 __asm__
volatile(
"mrc p15, 0, %0, c13, c0, 3" :
"=r"(tid));
914#elif defined(__aarch64__)
917 __asm__
volatile(
"mrs %0, tpidrro_el0" :
"=r"(tid));
919 __asm__
volatile(
"mrs %0, tpidr_el0" :
"=r"(tid));
922#error This platform needs implementation of get_thread_id()
926#error This platform needs implementation of get_thread_id()
932#if ((defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD) || \
934 pthread_setspecific(_memory_thread_heap, heap);
936 _memory_thread_heap = heap;
956#elif defined(__x86_64__) || defined(__i386__)
957 __asm__
volatile(
"pause" :::
"memory");
958#elif defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH >= 7)
959 __asm__
volatile(
"yield" :::
"memory");
960#elif defined(__powerpc__) || defined(__powerpc64__)
962 __asm__
volatile(
"or 27,27,27");
963#elif defined(__sparc__)
964 __asm__
volatile(
"rd %ccr, %g0 \n\trd %ccr, %g0 \n\trd %ccr, %g0");
966 struct timespec ts = {0};
971#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
972static void NTAPI _rpmalloc_thread_destructor(
void *
value) {
991#if defined(__linux__) || defined(__ANDROID__)
994 if (address == MAP_FAILED || !
name)
998 (void)prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)address, size,
1002 (void)
sizeof(address);
1016 _mapped_pages_peak);
1032 "Invalid unmap size");
1053 void *ptr = VirtualAlloc(0, size + padding,
1055 MEM_RESERVE | MEM_COMMIT,
1067 int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_UNINITIALIZED;
1068#if defined(__APPLE__) && !TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
1069 int fd = (int)VM_MAKE_TAG(240U);
1071 fd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
1072 void *ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, fd, 0);
1073#elif defined(MAP_HUGETLB)
1074 void *ptr = mmap(0, size + padding,
1075 PROT_READ | PROT_WRITE | PROT_MAX(PROT_READ | PROT_WRITE),
1077#if defined(MADV_HUGEPAGE)
1082 ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, -1, 0);
1083 if (ptr && ptr != MAP_FAILED) {
1084 int prm = madvise(ptr, size + padding, MADV_HUGEPAGE);
1091#elif defined(MAP_ALIGNED)
1092 const size_t align =
1093 (
sizeof(size_t) * 8) - (size_t)(__builtin_clzl(size - 1));
1095 mmap(0, size + padding, PROT_READ | PROT_WRITE,
1097#elif defined(MAP_ALIGN)
1099 void *ptr = mmap(base, size + padding, PROT_READ | PROT_WRITE,
1102 void *ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, -1, 0);
1104 if ((ptr == MAP_FAILED) || !ptr) {
1108 }
else if (errno != ENOMEM) {
1110 "Failed to map virtual memory block");
1120 "Internal failure in padding");
1121 rpmalloc_assert(final_padding <= padding,
"Internal failure in padding");
1124 *offset = final_padding >> 3;
1128 "Internal failure in padding");
1137 "Invalid unmap size");
1139 if (release && offset) {
1150 if (!VirtualFree(address, release ? 0 : size,
1151 release ? MEM_RELEASE : MEM_DECOMMIT)) {
1156 if (munmap(address, release)) {
1160#if defined(MADV_FREE_REUSABLE)
1162 while ((ret = madvise(address, size, MADV_FREE_REUSABLE)) == -1 &&
1165 if ((ret == -1) && (errno != 0)) {
1166#elif defined(MADV_DONTNEED)
1167 if (madvise(address, size, MADV_DONTNEED)) {
1168#elif defined(MADV_PAGEOUT)
1169 if (madvise(address, size, MADV_PAGEOUT)) {
1170#elif defined(MADV_FREE)
1171 if (madvise(address, size, MADV_FREE)) {
1173 if (posix_madvise(address, size, POSIX_MADV_DONTNEED)) {
1206 size_t reserve_span_count) {
1221 (*head)->prev = span;
1238 if (*head == span) {
1243 prev_span->
next = next_span;
1245 next_span->
prev = prev_span;
1261 size_t reserve_span_count);
1267 size_t span_count) {
1269 "Span master pointer and/or flag mismatch");
1270 if (subspan != master) {
1283 size_t span_count) {
1308 return request_count;
1313 size_t span_count,
size_t align_offset) {
1326 size_t span_count) {
1331 size_t align_offset = 0;
1340 if (aligned_span_count > span_count) {
1343 size_t reserved_count = aligned_span_count - span_count;
1353 "Global spin lock not held as expected");
1375 if (span_count <= heap->spans_reserved)
1378 int use_global_reserve =
1381 if (use_global_reserve) {
1386 size_t reserve_count =
1392 if (reserve_count > span_count) {
1397 reserve_count - span_count);
1406 if (use_global_reserve)
1416 "Span flag corrupted");
1419 "Span flag corrupted");
1428 "Span flag corrupted");
1451 "Span flag corrupted");
1468 "Invalid span size class");
1470#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
1489 void *page_start,
void *block_start,
1493 *first_block = block_start;
1494 if (block_count > 1) {
1502 if (page_end < block_end)
1503 block_end = page_end;
1508 while (next_block < block_end) {
1509 *((
void **)free_block) = next_block;
1510 free_block = next_block;
1514 *((
void **)free_block) = 0;
1530 span->
flags &= ~SPAN_FLAG_ALIGNED_BLOCKS;
1550#if RPMALLOC_FIRST_CLASS_HEAPS
1574 "Span free list corrupted");
1582 if (span == class_span) {
1585 void *last_block = 0;
1597 *((
void **)last_block) = free_list;
1624#if ENABLE_GLOBAL_CACHE
1627static void _rpmalloc_global_cache_finalize(
global_cache_t *cache) {
1631 for (
size_t ispan = 0; ispan < cache->
count; ++ispan)
1644static void _rpmalloc_global_cache_insert_spans(
span_t **span,
1647 const size_t cache_limit =
1654 size_t insert_count =
count;
1658#if ENABLE_STATISTICS
1659 cache->insert_count +=
count;
1661 if ((cache->
count + insert_count) > cache_limit)
1662 insert_count = cache_limit - cache->
count;
1664 memcpy(cache->
span + cache->
count, span,
sizeof(
span_t *) * insert_count);
1667#if ENABLE_UNLIMITED_CACHE
1668 while (insert_count < count) {
1675 span_t *current_span = span[insert_count++];
1682 for (
size_t ispan = insert_count; ispan <
count; ++ispan) {
1683 span_t *current_span = span[ispan];
1688 current_span->
next = keep;
1689 keep = current_span;
1701 for (; islot < cache->
count; ++islot) {
1708 cache->
span[islot] = keep;
1712 if (islot == cache->
count)
1729static size_t _rpmalloc_global_cache_extract_spans(
span_t **span,
1734 size_t extract_count = 0;
1738#if ENABLE_STATISTICS
1739 cache->extract_count +=
count;
1741 size_t want =
count - extract_count;
1742 if (want > cache->
count)
1743 want = cache->
count;
1745 memcpy(span + extract_count, cache->
span + (cache->
count - want),
1746 sizeof(
span_t *) * want);
1748 extract_count += want;
1750 while ((extract_count < count) && cache->
overflow) {
1752 span[extract_count++] = current_span;
1757 for (
size_t ispan = 0; ispan < extract_count; ++ispan) {
1759 "Global cache span count mismatch");
1765 return extract_count;
1781 size_t reserve_span_count) {
1800#if RPMALLOC_FIRST_CLASS_HEAPS
1806 if (single_span && !*single_span)
1807 *single_span = span;
1815 "Span size class invalid");
1818#if RPMALLOC_FIRST_CLASS_HEAPS
1824 if (!idx && single_span && !*single_span)
1825 *single_span = span;
1855#if ENABLE_THREAD_CACHE
1859 span_cache = &heap->span_cache;
1861 span_cache = (
span_cache_t *)(heap->span_large_cache + (iclass - 1));
1862 for (
size_t ispan = 0; ispan < span_cache->
count; ++ispan)
1864 span_cache->
count = 0;
1883 if (list_heap == heap) {
1886 while (list_heap->next_heap != heap)
1902#if ENABLE_THREAD_CACHE
1905 if (span_count == 1) {
1907 span_cache->
span[span_cache->
count++] = span;
1909 const size_t remain_count =
1911#if ENABLE_GLOBAL_CACHE
1916 _rpmalloc_global_cache_insert_spans(span_cache->
span + remain_count,
1923 span_cache->
count = remain_count;
1926 size_t cache_idx = span_count - 2;
1928 span_cache->
span[span_cache->
count++] = span;
1929 const size_t cache_limit =
1931 if (span_cache->
count == cache_limit) {
1932 const size_t transfer_limit = 2 + (cache_limit >> 2);
1933 const size_t transfer_count =
1937 const size_t remain_count = cache_limit - transfer_count;
1938#if ENABLE_GLOBAL_CACHE
1943 _rpmalloc_global_cache_insert_spans(span_cache->
span + remain_count,
1944 span_count, transfer_count);
1946 for (
size_t ispan = 0; ispan < transfer_count; ++ispan)
1949 span_cache->
count = remain_count;
1960 size_t span_count) {
1962#if ENABLE_THREAD_CACHE
1964 if (span_count == 1)
1965 span_cache = &heap->span_cache;
1967 span_cache = (
span_cache_t *)(heap->span_large_cache + (span_count - 2));
1968 if (span_cache->
count) {
1970 return span_cache->
span[--span_cache->
count];
1977 size_t span_count) {
1979 if (span_count == 1) {
1989 size_t span_count) {
1997 size_t span_count) {
1998#if ENABLE_GLOBAL_CACHE
1999#if ENABLE_THREAD_CACHE
2001 size_t wanted_count;
2002 if (span_count == 1) {
2003 span_cache = &heap->span_cache;
2006 span_cache = (
span_cache_t *)(heap->span_large_cache + (span_count - 2));
2009 span_cache->
count = _rpmalloc_global_cache_extract_spans(
2010 span_cache->
span, span_count, wanted_count);
2011 if (span_cache->
count) {
2016 return span_cache->
span[--span_cache->
count];
2020 size_t count = _rpmalloc_global_cache_extract_spans(&span, span_count, 1);
2031 (void)
sizeof(span_count);
2038 (void)
sizeof(span_count);
2039 (void)
sizeof(class_idx);
2040#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
2044 if (current_count > (
uint32_t)atomic_load32(&heap->span_use[idx].high))
2045 atomic_store32(&heap->span_use[idx].high, (int32_t)current_count);
2047 heap->size_class_use[class_idx].spans_peak);
2056 size_t span_count,
uint32_t class_idx) {
2058#if ENABLE_THREAD_CACHE
2059 if (heap_size_class && heap_size_class->
cache) {
2060 span = heap_size_class->
cache;
2061 heap_size_class->
cache =
2062 (heap->span_cache.count
2063 ? heap->span_cache.span[--heap->span_cache.count]
2069 (void)
sizeof(class_idx);
2071 size_t base_span_count = span_count;
2072 size_t limit_span_count =
2073 (span_count > 2) ? (span_count + (span_count >> 1)) : span_count;
2102 }
while (span_count <= limit_span_count);
2123#if RPMALLOC_FIRST_CLASS_HEAPS
2127 (void)
sizeof(first_class);
2140 size_t heap_size =
sizeof(
heap_t);
2141 size_t aligned_heap_size = 16 * ((heap_size + 15) / 16);
2142 size_t request_heap_count = 16;
2143 size_t heap_span_count = ((aligned_heap_size * request_heap_count) +
2147 size_t span_count = heap_span_count;
2159 size_t possible_heap_count =
2160 (block_size -
sizeof(
span_t)) / aligned_heap_size;
2161 if (possible_heap_count >= (request_heap_count * 16))
2162 request_heap_count *= 16;
2163 else if (possible_heap_count < request_heap_count)
2164 request_heap_count = possible_heap_count;
2165 heap_span_count = ((aligned_heap_size * request_heap_count) +
2170 size_t align_offset = 0;
2185 size_t num_heaps = remain_size / aligned_heap_size;
2186 if (num_heaps < request_heap_count)
2187 num_heaps = request_heap_count;
2190 while (num_heaps > 1) {
2198 if (span_count > heap_span_count) {
2200 size_t remain_count = span_count - heap_span_count;
2201 size_t reserve_count =
2208 if (remain_count > reserve_count) {
2212 reserve_count = remain_count - reserve_count;
2221 heap_t *heap = *heap_list;
2231 if (first_class == 0)
2233#if RPMALLOC_FIRST_CLASS_HEAPS
2246 int release_cache) {
2252 if (release_cache || heap->
finalize) {
2253#if ENABLE_THREAD_CACHE
2257 span_cache = &heap->span_cache;
2259 span_cache = (
span_cache_t *)(heap->span_large_cache + (iclass - 1));
2260 if (!span_cache->
count)
2262#if ENABLE_GLOBAL_CACHE
2264 for (
size_t ispan = 0; ispan < span_cache->
count; ++ispan)
2272 _rpmalloc_global_cache_insert_spans(span_cache->
span, iclass + 1,
2276 for (
size_t ispan = 0; ispan < span_cache->
count; ++ispan)
2279 span_cache->
count = 0;
2287#if ENABLE_STATISTICS
2290 "Still active heaps during finalization");
2337#if RPMALLOC_FIRST_CLASS_HEAPS
2338 list = &heap->full_span[iclass];
2350#if ENABLE_THREAD_CACHE
2354 span_cache = &heap->span_cache;
2356 span_cache = (
span_cache_t *)(heap->span_large_cache + (iclass - 1));
2357 for (
size_t ispan = 0; ispan < span_cache->
count; ++ispan)
2359 span_cache->
count = 0;
2363 "Heaps still active during finalization");
2374 void *
block = *list;
2375 *list = *((
void **)
block);
2387 "Span block count corrupted");
2389 "Internal failure");
2409 "Span block count corrupted");
2425#if RPMALLOC_FIRST_CLASS_HEAPS
2496#if RPMALLOC_FIRST_CLASS_HEAPS
2512 size_t align_offset = 0;
2525#if RPMALLOC_FIRST_CLASS_HEAPS
2550#if ENABLE_VALIDATE_ARGS
2551 if ((size + alignment) < size) {
2555 if (alignment & (alignment - 1)) {
2570 "Failed alignment calculation");
2571 if (multiple_size <= (size + alignment))
2576 size_t align_mask = alignment - 1;
2579 if ((uintptr_t)ptr & align_mask) {
2580 ptr = (
void *)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment);
2597 if (alignment & align_mask) {
2613 if (extra_pages > num_pages)
2614 num_pages = 1 + extra_pages;
2616 size_t original_pages = num_pages;
2618 if (limit_pages < (original_pages * 2))
2619 limit_pages = original_pages * 2;
2621 size_t mapped_size, align_offset;
2635 if ((uintptr_t)ptr & align_mask)
2636 ptr = (
void *)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment);
2643 if (num_pages > limit_pages) {
2657#if RPMALLOC_FIRST_CLASS_HEAPS
2680 "Internal failure");
2684#if RPMALLOC_FIRST_CLASS_HEAPS
2737 *((
void **)
block) = free_list;
2739 int all_deferred_free = (free_count == span->
block_count);
2741 if (all_deferred_free) {
2758#if RPMALLOC_FIRST_CLASS_HEAPS
2777 "Span flag corrupted");
2780 "Span flag corrupted");
2783#if RPMALLOC_FIRST_CLASS_HEAPS
2797#if RPMALLOC_FIRST_CLASS_HEAPS
2800#if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
2807#if ENABLE_THREAD_CACHE
2808 const int set_as_reserved =
2809 ((span->
span_count > 1) && (heap->span_cache.count == 0) &&
2812 const int set_as_reserved =
2815 if (set_as_reserved) {
2828 "Master span count corrupted");
2840#if RPMALLOC_FIRST_CLASS_HEAPS
2854#if RPMALLOC_FIRST_CLASS_HEAPS
2890 size_t oldsize,
unsigned int flags) {
2909 memmove(
block, p, oldsize);
2923 if ((current_spans >= num_spans) && (total_size >= (oldsize / 2))) {
2927 memmove(
block, p, oldsize);
2942 if ((current_pages >= num_pages) && (num_pages >= (current_pages / 2))) {
2946 memmove(
block, p, oldsize);
2960 size_t lower_bound = oldsize + (oldsize >> 2) + (oldsize >> 3);
2962 (size > lower_bound) ? size : ((size > oldsize) ? lower_bound : size);
2966 memcpy(
block, p, oldsize < new_size ? oldsize : new_size);
2974 size_t alignment,
size_t size,
2975 size_t oldsize,
unsigned int flags) {
2981 if ((usablesize >= size) && !((uintptr_t)ptr & (alignment - 1))) {
2982 if (no_alloc || (size >= (usablesize / 2)))
2991 oldsize = usablesize;
2992 memcpy(
block, ptr, oldsize < size ? oldsize : size);
3035 size_t prevclass = iclass;
3036 while (prevclass > 0) {
3077 SYSTEM_INFO system_info;
3078 memset(&system_info, 0,
sizeof(system_info));
3079 GetSystemInfo(&system_info);
3085#if RPMALLOC_CONFIGURABLE
3097#if defined(__linux__)
3098 size_t huge_page_size = 0;
3099 FILE *meminfo = fopen(
"/proc/meminfo",
"r");
3102 while (!huge_page_size && fgets(line,
sizeof(line) - 1, meminfo)) {
3103 line[
sizeof(line) - 1] = 0;
3104 if (strstr(line,
"Hugepagesize:"))
3105 huge_page_size = (
size_t)strtol(line + 13, 0, 10) * 1024;
3109 if (huge_page_size) {
3114#elif defined(__FreeBSD__)
3116 size_t sz =
sizeof(
rc);
3118 if (sysctlbyname(
"vm.pmap.pg_ps_enabled", &
rc, &sz, NULL, 0) == 0 &&
3120 static size_t defsize = 2 * 1024 * 1024;
3122 size_t sizes[4] = {0};
3125 if ((nsize = getpagesizes(sizes, 4)) >= 2) {
3127 for (
size_t csize = sizes[nsize]; nsize >= 0 && csize;
3128 --nsize, csize = sizes[nsize]) {
3131 "Invalid page size");
3132 if (defsize < csize) {
3140#elif defined(__APPLE__) || defined(__NetBSD__)
3155 size_t large_page_minimum = GetLargePageMinimum();
3156 if (large_page_minimum)
3157 OpenProcessToken(GetCurrentProcess(),
3158 TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
3161 if (LookupPrivilegeValue(0, SE_LOCK_MEMORY_NAME, &luid)) {
3162 TOKEN_PRIVILEGES token_privileges;
3163 memset(&token_privileges, 0,
sizeof(token_privileges));
3164 token_privileges.PrivilegeCount = 1;
3165 token_privileges.Privileges[0].Luid = luid;
3166 token_privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
3167 if (AdjustTokenPrivileges(token, FALSE, &token_privileges, 0, 0, 0)) {
3168 if (GetLastError() == ERROR_SUCCESS)
3183 size_t min_span_size = 256;
3184 size_t max_page_size;
3185#if UINTPTR_MAX > 0xFFFFFFFF
3186 max_page_size = 4096ULL * 1024ULL * 1024ULL;
3188 max_page_size = 4 * 1024 * 1024;
3196 while (page_size_bit != 1) {
3198 page_size_bit >>= 1;
3202#if RPMALLOC_CONFIGURABLE
3209 if (span_size > (256 * 1024))
3210 span_size = (256 * 1024);
3238#if ((defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD) || \
3243#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
3244 fls_key = FlsAlloc(&_rpmalloc_thread_destructor);
3272#if RPMALLOC_FIRST_CLASS_HEAPS
3273 _memory_first_class_orphan_heaps = 0;
3275#if ENABLE_STATISTICS
3278 _mapped_pages_peak = 0;
3284 _huge_pages_peak = 0;
3321#if ENABLE_GLOBAL_CACHE
3324 _rpmalloc_global_cache_finalize(&_memory_span_cache[iclass]);
3327#if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
3328 pthread_key_delete(_memory_thread_heap);
3330#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
3334#if ENABLE_STATISTICS
3337 rpmalloc_assert(atomic_load32(&_mapped_pages) == 0,
"Memory leak detected");
3339 "Memory leak detected");
3352#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
3353 FlsSetValue(fls_key, heap);
3365#if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
3366 FlsSetValue(fls_key, 0);
3379#if ENABLE_VALIDATE_ARGS
3380 if (size >= MAX_ALLOC_SIZE) {
3393#if ENABLE_VALIDATE_ARGS
3395 int err = SizeTMult(num, size, &total);
3396 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
3401 int err = __builtin_umull_overflow(num, size, &total);
3402 if (err || (total >= MAX_ALLOC_SIZE)) {
3413 memset(
block, 0, total);
3418#if ENABLE_VALIDATE_ARGS
3419 if (size >= MAX_ALLOC_SIZE) {
3429 size_t size,
size_t oldsize,
3430 unsigned int flags) {
3431#if ENABLE_VALIDATE_ARGS
3450#if ENABLE_VALIDATE_ARGS
3452 int err = SizeTMult(num, size, &total);
3453 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
3458 int err = __builtin_umull_overflow(num, size, &total);
3459 if (err || (total >= MAX_ALLOC_SIZE)) {
3469 memset(
block, 0, total);
3484 return *memptr ? 0 : ENOMEM;
3507 free_count += (block_count - span->
used_count);
3513#if ENABLE_THREAD_CACHE
3517 span_cache = &heap->span_cache;
3519 span_cache = (
span_cache_t *)(heap->span_large_cache + (iclass - 1));
3531#if ENABLE_STATISTICS
3536 stats->span_use[iclass].current =
3537 (size_t)atomic_load32(&heap->span_use[iclass].current);
3538 stats->span_use[iclass].peak =
3539 (size_t)atomic_load32(&heap->span_use[iclass].high);
3540 stats->span_use[iclass].to_global =
3541 (size_t)atomic_load32(&heap->span_use[iclass].spans_to_global);
3542 stats->span_use[iclass].from_global =
3543 (size_t)atomic_load32(&heap->span_use[iclass].spans_from_global);
3544 stats->span_use[iclass].to_cache =
3545 (size_t)atomic_load32(&heap->span_use[iclass].spans_to_cache);
3546 stats->span_use[iclass].from_cache =
3547 (size_t)atomic_load32(&heap->span_use[iclass].spans_from_cache);
3548 stats->span_use[iclass].to_reserved =
3549 (size_t)atomic_load32(&heap->span_use[iclass].spans_to_reserved);
3550 stats->span_use[iclass].from_reserved =
3551 (size_t)atomic_load32(&heap->span_use[iclass].spans_from_reserved);
3552 stats->span_use[iclass].map_calls =
3553 (size_t)atomic_load32(&heap->span_use[iclass].spans_map_calls);
3556 stats->size_use[iclass].alloc_current =
3557 (size_t)atomic_load32(&heap->size_class_use[iclass].alloc_current);
3558 stats->size_use[iclass].alloc_peak =
3559 (size_t)heap->size_class_use[iclass].alloc_peak;
3560 stats->size_use[iclass].alloc_total =
3561 (
size_t)atomic_load32(&heap->size_class_use[iclass].alloc_total);
3562 stats->size_use[iclass].free_total =
3563 (size_t)atomic_load32(&heap->size_class_use[iclass].free_total);
3564 stats->size_use[iclass].spans_to_cache =
3565 (size_t)atomic_load32(&heap->size_class_use[iclass].spans_to_cache);
3566 stats->size_use[iclass].spans_from_cache =
3567 (size_t)atomic_load32(&heap->size_class_use[iclass].spans_from_cache);
3568 stats->size_use[iclass].spans_from_reserved = (size_t)atomic_load32(
3569 &heap->size_class_use[iclass].spans_from_reserved);
3570 stats->size_use[iclass].map_calls =
3571 (size_t)atomic_load32(&heap->size_class_use[iclass].spans_map_calls);
3578#if ENABLE_STATISTICS
3581 stats->mapped_total =
3583 stats->unmapped_total =
3589#if ENABLE_GLOBAL_CACHE
3595#if ENABLE_UNLIMITED_CACHE
3597 while (current_span) {
3599 current_span = current_span->
next;
3608#if ENABLE_STATISTICS
3610static void _memory_heap_dump_statistics(
heap_t *heap,
void *
file) {
3611 fprintf(
file,
"Heap %d stats:\n", heap->
id);
3612 fprintf(
file,
"Class CurAlloc PeakAlloc TotAlloc TotFree BlkSize "
3613 "BlkCount SpansCur SpansPeak PeakAllocMiB ToCacheMiB "
3614 "FromCacheMiB FromReserveMiB MmapCalls\n");
3616 if (!atomic_load32(&heap->size_class_use[iclass].alloc_total))
3620 "%3u: %10u %10u %10u %10u %8u %8u %8d %9d %13zu %11zu %12zu %14zu "
3623 atomic_load32(&heap->size_class_use[iclass].alloc_current),
3624 heap->size_class_use[iclass].alloc_peak,
3625 atomic_load32(&heap->size_class_use[iclass].alloc_total),
3626 atomic_load32(&heap->size_class_use[iclass].free_total),
3629 atomic_load32(&heap->size_class_use[iclass].spans_current),
3630 heap->size_class_use[iclass].spans_peak,
3631 ((
size_t)heap->size_class_use[iclass].alloc_peak *
3633 (
size_t)(1024 * 1024),
3634 ((
size_t)atomic_load32(&heap->size_class_use[iclass].spans_to_cache) *
3636 (
size_t)(1024 * 1024),
3637 ((
size_t)atomic_load32(&heap->size_class_use[iclass].spans_from_cache) *
3639 (
size_t)(1024 * 1024),
3640 ((
size_t)atomic_load32(
3641 &heap->size_class_use[iclass].spans_from_reserved) *
3643 (
size_t)(1024 * 1024),
3644 atomic_load32(&heap->size_class_use[iclass].spans_map_calls));
3646 fprintf(
file,
"Spans Current Peak Deferred PeakMiB Cached ToCacheMiB "
3647 "FromCacheMiB ToReserveMiB FromReserveMiB ToGlobalMiB "
3648 "FromGlobalMiB MmapCalls\n");
3650 if (!atomic_load32(&heap->span_use[iclass].high) &&
3651 !atomic_load32(&heap->span_use[iclass].spans_map_calls))
3655 "%4u: %8d %8u %8u %8zu %7u %11zu %12zu %12zu %14zu %11zu %13zu %10u\n",
3656 (
uint32_t)(iclass + 1), atomic_load32(&heap->span_use[iclass].current),
3657 atomic_load32(&heap->span_use[iclass].high),
3658 atomic_load32(&heap->span_use[iclass].spans_deferred),
3659 ((
size_t)atomic_load32(&heap->span_use[iclass].high) *
3661 (
size_t)(1024 * 1024),
3663 (
unsigned int)(!iclass ? heap->span_cache.count
3664 : heap->span_large_cache[iclass - 1].count),
3665 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_to_cache) *
3667 (
size_t)(1024 * 1024),
3668 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_from_cache) *
3670 (
size_t)(1024 * 1024),
3672 0, (
size_t)0, (
size_t)0,
3674 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_to_reserved) *
3676 (
size_t)(1024 * 1024),
3677 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_from_reserved) *
3679 (
size_t)(1024 * 1024),
3680 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_to_global) *
3682 (
size_t)(1024 * 1024),
3683 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_from_global) *
3685 (
size_t)(1024 * 1024),
3686 atomic_load32(&heap->span_use[iclass].spans_map_calls));
3689 fprintf(
file,
"ThreadToGlobalMiB GlobalToThreadMiB\n");
3691 file,
"%17zu %17zu\n",
3692 (
size_t)
atomic_load64(&heap->thread_to_global) / (
size_t)(1024 * 1024),
3693 (
size_t)
atomic_load64(&heap->global_to_thread) / (
size_t)(1024 * 1024));
3699#if ENABLE_STATISTICS
3706 if (!atomic_load32(&heap->size_class_use[iclass].alloc_total)) {
3708 !atomic_load32(&heap->size_class_use[iclass].free_total),
3709 "Heap statistics counter mismatch");
3711 !atomic_load32(&heap->size_class_use[iclass].spans_map_calls),
3712 "Heap statistics counter mismatch");
3719 if (!atomic_load32(&heap->span_use[iclass].high) &&
3720 !atomic_load32(&heap->span_use[iclass].spans_map_calls))
3725 _memory_heap_dump_statistics(heap,
file);
3729 fprintf(
file,
"Global stats:\n");
3730 size_t huge_current =
3733 fprintf(
file,
"HugeCurrentMiB HugePeakMiB\n");
3734 fprintf(
file,
"%14zu %11zu\n", huge_current / (
size_t)(1024 * 1024),
3735 huge_peak / (
size_t)(1024 * 1024));
3737#if ENABLE_GLOBAL_CACHE
3738 fprintf(
file,
"GlobalCacheMiB\n");
3743 size_t global_overflow_cache = 0;
3749 if (global_cache || global_overflow_cache || cache->insert_count ||
3750 cache->extract_count)
3752 "%4zu: %8zuMiB (%8zuMiB overflow) %14zu insert %14zu extract\n",
3753 iclass + 1, global_cache / (
size_t)(1024 * 1024),
3754 global_overflow_cache / (
size_t)(1024 * 1024),
3755 cache->insert_count, cache->extract_count);
3763 size_t mapped_total =
3765 size_t unmapped_total =
3769 "MappedMiB MappedOSMiB MappedPeakMiB MappedTotalMiB UnmappedTotalMiB\n");
3770 fprintf(
file,
"%9zu %11zu %13zu %14zu %16zu\n",
3771 mapped / (
size_t)(1024 * 1024), mapped_os / (
size_t)(1024 * 1024),
3772 mapped_peak / (
size_t)(1024 * 1024),
3773 mapped_total / (
size_t)(1024 * 1024),
3774 unmapped_total / (
size_t)(1024 * 1024));
3776 fprintf(
file,
"\n");
3779 int64_t deallocated =
atomic_load64(&_deallocation_counter);
3780 fprintf(
file,
"Allocation count: %lli\n", allocated);
3781 fprintf(
file,
"Deallocation count: %lli\n", deallocated);
3782 fprintf(
file,
"Current allocations: %lli\n", (allocated - deallocated));
3783 fprintf(
file,
"Master spans: %d\n", atomic_load32(&_master_spans));
3784 fprintf(
file,
"Dangling master spans: %d\n", atomic_load32(&_unmapped_master_spans));
3790#if RPMALLOC_FIRST_CLASS_HEAPS
3792extern inline rpmalloc_heap_t *rpmalloc_heap_acquire(
void) {
3804extern inline void rpmalloc_heap_release(rpmalloc_heap_t *heap) {
3810rpmalloc_heap_alloc(rpmalloc_heap_t *heap,
size_t size) {
3811#if ENABLE_VALIDATE_ARGS
3812 if (size >= MAX_ALLOC_SIZE) {
3821rpmalloc_heap_aligned_alloc(rpmalloc_heap_t *heap,
size_t alignment,
3823#if ENABLE_VALIDATE_ARGS
3824 if (size >= MAX_ALLOC_SIZE) {
3833rpmalloc_heap_calloc(rpmalloc_heap_t *heap,
size_t num,
size_t size) {
3834 return rpmalloc_heap_aligned_calloc(heap, 0, num, size);
3838rpmalloc_heap_aligned_calloc(rpmalloc_heap_t *heap,
size_t alignment,
3839 size_t num,
size_t size) {
3841#if ENABLE_VALIDATE_ARGS
3843 int err = SizeTMult(num, size, &total);
3844 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
3849 int err = __builtin_umull_overflow(num, size, &total);
3850 if (err || (total >= MAX_ALLOC_SIZE)) {
3860 memset(
block, 0, total);
3865rpmalloc_heap_realloc(rpmalloc_heap_t *heap,
void *ptr,
size_t size,
3866 unsigned int flags) {
3867#if ENABLE_VALIDATE_ARGS
3868 if (size >= MAX_ALLOC_SIZE) {
3877rpmalloc_heap_aligned_realloc(rpmalloc_heap_t *heap,
void *ptr,
3878 size_t alignment,
size_t size,
3879 unsigned int flags) {
3880#if ENABLE_VALIDATE_ARGS
3889extern inline void rpmalloc_heap_free(rpmalloc_heap_t *heap,
void *ptr) {
3894extern inline void rpmalloc_heap_free_all(rpmalloc_heap_t *heap) {
3901 span = heap->
size_class[iclass].partial_span;
3903 next_span = span->
next;
3908 span = heap->full_span[iclass];
3910 next_span = span->
next;
3918 heap->size_class[iclass].cache = 0;
3920 memset(heap->size_class, 0,
sizeof(heap->size_class));
3921 memset(heap->full_span, 0,
sizeof(heap->full_span));
3923 span = heap->large_huge_span;
3925 next_span = span->
next;
3932 heap->large_huge_span = 0;
3933 heap->full_span_count = 0;
3935#if ENABLE_THREAD_CACHE
3939 span_cache = &heap->span_cache;
3941 span_cache = (
span_cache_t *)(heap->span_large_cache + (iclass - 1));
3942 if (!span_cache->
count)
3944#if ENABLE_GLOBAL_CACHE
3949 _rpmalloc_global_cache_insert_spans(span_cache->
span, iclass + 1,
3952 for (
size_t ispan = 0; ispan < span_cache->
count; ++ispan)
3955 span_cache->
count = 0;
3959#if ENABLE_STATISTICS
3970extern inline void rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap) {
3972 if (prev_heap != heap) {
3975 rpmalloc_heap_release(prev_heap);
3979extern inline rpmalloc_heap_t *rpmalloc_get_heap_for_ptr(
void *ptr) {
3990#if ENABLE_PRELOAD || ENABLE_OVERRIDE
Given that RA is a live value
dot regions Print regions of function to dot file(with no function bodies)"
unify loop Fixup each natural loop to have a single exit block
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
#define SMALL_GRANULARITY
Preconfigured limits and sizes.
static FORCEINLINE void * atomic_exchange_ptr_acquire(atomicptr_t *dst, void *val)
#define LARGE_CLASS_COUNT
Number of large block size classes.
#define _memory_default_span_size
Global data.
static FORCEINLINE int64_t atomic_load64(atomic64_t *val)
#define _memory_span_size_shift
static void * _rpmalloc_allocate_from_heap_fallback(heap_t *heap, heap_size_class_t *heap_size_class, uint32_t class_idx)
Allocate a small/medium sized memory block from the given heap.
static heap_t * _rpmalloc_heap_allocate(int first_class)
Allocate a new heap, potentially reusing a previously orphaned heap.
#define MEDIUM_GRANULARITY
Granularity of a medium allocation block.
static void _rpmalloc_heap_unmap(heap_t *heap)
_Static_assert((SMALL_GRANULARITY &(SMALL_GRANULARITY - 1))==0, "Small granularity must be power of two")
#define SPAN_FLAG_UNMAPPED_MASTER
Flag indicating an unmapped master span.
static heap_t * get_thread_heap_raw(void)
void rpfree(void *ptr)
Free the given memory block.
#define rpmalloc_assume(cond)
static size_t _memory_global_reserve_count
Global reserved count.
static void _rpmalloc_span_unmap(span_t *span)
Unmap memory pages for the given number of spans (or mark as unused if no partial unmappings)
static void _rpmalloc_heap_release(void *heapptr, int first_class, int release_cache)
static atomic32_t _memory_global_lock
Used to restrict access to mapping memory for huge pages.
static span_t * _memory_global_reserve_master
Global reserved master.
static size_t _memory_map_granularity
Granularity at which memory pages are mapped by OS.
static FORCEINLINE int32_t atomic_incr32(atomic32_t *val)
static FORCEINLINE int atomic_cas32_acquire(atomic32_t *dst, int32_t val, int32_t ref)
static void _rpmalloc_adjust_size_class(size_t iclass)
Adjust and optimize the size class properties for the given class.
#define THREAD_SPAN_CACHE_TRANSFER
Number of spans to transfer between thread and global cache.
RPMALLOC_ALLOCATOR void * rpmalloc(size_t size)
Allocate a memory block of at least the given size.
static void * _rpmalloc_aligned_reallocate(heap_t *heap, void *ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags)
static void _rpmalloc_heap_finalize(heap_t *heap)
static span_t * _rpmalloc_span_map_aligned_count(heap_t *heap, size_t span_count)
Map an aligned set of spans, taking configured mapping granularity and the page size into account.
static FORCEINLINE void atomic_store32_release(atomic32_t *dst, int32_t val)
int rpmalloc_initialize_config(const rpmalloc_config_t *config)
Initialize allocator with given configuration.
#define THREAD_SPAN_LARGE_CACHE_TRANSFER
Number of spans to transfer between thread and global cache for large spans.
static void _rpmalloc_heap_global_finalize(heap_t *heap)
#define SPAN_FLAG_SUBSPAN
Flag indicating span is a secondary (sub) span of a split superspan.
static FORCEINLINE int atomic_cas_ptr(atomicptr_t *dst, void *val, void *ref)
static void _rpmalloc_spin(void)
static size_class_t _memory_size_class[SIZE_CLASS_COUNT]
Global size classes.
static size_t _memory_page_size
Memory page size.
static void _rpmalloc_heap_release_raw_fc(void *heapptr)
static void * _rpmalloc_allocate(heap_t *heap, size_t size)
Allocate a block of the given size.
#define TLS_MODEL
Thread local heap and ID.
static size_t _memory_span_map_count
Number of spans to map in each map call.
static void * _rpmalloc_allocate_small(heap_t *heap, size_t size)
Allocate a small sized memory block from the given heap.
#define _memory_default_span_size_shift
static heap_t * _memory_heaps[HEAP_ARRAY_SIZE]
All heaps.
#define _memory_span_mask
static span_t * _memory_global_reserve
Global reserved spans.
const rpmalloc_config_t * rpmalloc_config(void)
Get allocator configuration.
volatile _Atomic(int32_t)
Atomic access abstraction (since MSVC does not do C11 yet)
static FORCEINLINE int64_t atomic_add64(atomic64_t *val, int64_t add)
static void _rpmalloc_span_double_link_list_pop_head(span_t **head, span_t *span)
Pop head span from double linked list.
#define _rpmalloc_stat_add64(counter, value)
static void set_thread_heap(heap_t *heap)
Set the current thread heap.
#define _rpmalloc_stat_add(counter, value)
static void * _rpmalloc_aligned_allocate(heap_t *heap, size_t alignment, size_t size)
static int _memory_huge_pages
Huge page support.
static FORCEINLINE int32_t atomic_decr32(atomic32_t *val)
RPMALLOC_ALLOCATOR void * rpaligned_realloc(void *ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags)
Reallocate the given block to at least the given size and alignment,.
#define MAX_THREAD_SPAN_LARGE_CACHE
Number of spans in thread cache for large spans (must be greater than LARGE_CLASS_COUNT / 2)
void rpmalloc_set_main_thread(void)
Set main thread ID.
#define MEDIUM_CLASS_COUNT
Number of medium block size classes.
static void _rpmalloc_inc_span_statistics(heap_t *heap, size_t span_count, uint32_t class_idx)
RPMALLOC_ALLOCATOR void * rpcalloc(size_t num, size_t size)
static void _rpmalloc_deallocate_large(span_t *span)
Deallocate the given large memory block to the current heap.
#define _memory_span_size
Hardwired span size.
static void _rpmalloc_heap_initialize(heap_t *heap)
#define _rpmalloc_stat_add_peak(counter, value, peak)
static void _rpmalloc_heap_orphan(heap_t *heap, int first_class)
RPMALLOC_ALLOCATOR void * rpmemalign(size_t alignment, size_t size)
Allocate a memory block of at least the given size and alignment.
static void _rpmalloc_deallocate_huge(span_t *)
Global cache.
static void _rpmalloc_span_release_to_cache(heap_t *heap, span_t *span)
Move the span (used for small or medium allocations) to the heap thread cache.
static int _rpmalloc_initialized
Initialized flag.
#define SPAN_HEADER_SIZE
Size of a span header (must be a multiple of SMALL_GRANULARITY and a power of two)
static heap_t * _rpmalloc_heap_allocate_new(void)
Allocate a new heap from newly mapped memory pages.
static void * _rpmalloc_span_initialize_new(heap_t *heap, heap_size_class_t *heap_size_class, span_t *span, uint32_t class_idx)
Initialize an unused span (from cache or mapped) to be new active span, putting the initial free list...
#define SMALL_GRANULARITY_SHIFT
Small granularity shift count.
static void _rpmalloc_set_name(void *address, size_t size)
Low level memory map/unmap.
static size_t _rpmalloc_span_align_count(size_t span_count)
Get the aligned number of spans to map in based on wanted count, configured mapping granularity and t...
#define LARGE_SIZE_LIMIT
Maximum size of a large block.
static void _rpmalloc_unmap_os(void *address, size_t size, size_t offset, size_t release)
Default implementation to unmap pages from virtual memory.
#define ENABLE_THREAD_CACHE
Enable per-thread cache.
static void _rpmalloc_deallocate_direct_small_or_medium(span_t *span, void *block)
Deallocation entry points.
static void _rpmalloc_deallocate_defer_small_or_medium(span_t *span, void *block)
Put the block in the deferred free list of the owning span.
void rpmalloc_thread_collect(void)
Perform deferred deallocations pending for the calling thread heap.
void rpmalloc_thread_finalize(int release_caches)
Finalize thread, orphan heap.
int rpmalloc_is_thread_initialized(void)
Query if allocator is initialized for calling thread.
void rpmalloc_linker_reference(void)
Dummy empty function for forcing linker symbol inclusion.
#define FORCEINLINE
Platform and arch specifics.
void rpmalloc_dump_statistics(void *file)
Dump all statistics in human readable format to file (should be a FILE*)
RPMALLOC_ALLOCATOR void * rpaligned_alloc(size_t alignment, size_t size)
Allocate a memory block of at least the given size and alignment.
size_t rpmalloc_usable_size(void *ptr)
Query the usable size of the given memory block (from given pointer to the end of block)
static void _rpmalloc_deallocate_small_or_medium(span_t *span, void *p)
static span_t * _rpmalloc_global_get_reserved_spans(size_t span_count)
Use global reserved spans to fulfill a memory map request (reserve size must be checked by caller)
#define _rpmalloc_stat_sub(counter, value)
static void * _rpmalloc_mmap_os(size_t size, size_t *offset)
Default implementation to map new pages to virtual memory.
#define _rpmalloc_memcpy_const(x, y, s)
static uintptr_t get_thread_id(void)
Fast thread ID.
void rpmalloc_thread_initialize(void)
Initialize thread, assign heap.
static span_t * _rpmalloc_heap_thread_cache_deferred_extract(heap_t *heap, size_t span_count)
static void * _rpmalloc_allocate_large(heap_t *heap, size_t size)
Allocate a large sized memory block from the given heap.
RPMALLOC_ALLOCATOR void * rprealloc(void *ptr, size_t size)
Reallocate the given block to at least the given size.
static FORCEINLINE void * atomic_load_ptr(atomicptr_t *src)
static void _rpmalloc_span_mark_as_subspan_unless_master(span_t *master, span_t *subspan, size_t span_count)
Declare the span to be a subspan and store distance from master span and span count.
static FORCEINLINE void atomic_store_ptr_release(atomicptr_t *dst, void *val)
#define MEDIUM_GRANULARITY_SHIFT
Medium granularity shift count.
static size_t _memory_medium_size_limit
Run-time size limit of medium blocks.
#define SPAN_FLAG_MASTER
Flag indicating span is the first (master) span of a split superspan.
static void * _rpmalloc_reallocate(heap_t *heap, void *p, size_t size, size_t oldsize, unsigned int flags)
Reallocate the given block to the given size.
int rpmalloc_initialize(void)
Initialize the allocator and setup global data.
static span_t * _rpmalloc_heap_reserved_extract(heap_t *heap, size_t span_count)
static size_t _memory_page_size_shift
Shift to divide by page size.
static FORCEINLINE void atomic_store32(atomic32_t *dst, int32_t val)
static void _rpmalloc_heap_release_raw(void *heapptr, int release_cache)
static void _rpmalloc_global_set_reserved_spans(span_t *master, span_t *reserve, size_t reserve_span_count)
Store the given spans as global reserve (must only be called from within new heap allocation,...
#define SIZE_CLASS_COUNT
Total number of small + medium size classes.
static int _rpmalloc_span_finalize(heap_t *heap, size_t iclass, span_t *span, span_t **list_head)
#define _rpmalloc_stat_dec(counter)
static uintptr_t _rpmalloc_main_thread_id
Main thread ID.
#define _rpmalloc_stat_inc_free(heap, class_idx)
static span_t * _rpmalloc_heap_thread_cache_extract(heap_t *heap, size_t span_count)
Extract the given number of spans from the different cache levels.
#define pointer_offset(ptr, ofs)
#define _rpmalloc_stat_inc(counter)
Statistics related functions (evaluate to nothing when statistics not enabled)
#define _rpmalloc_memset_const(x, y, s)
static void * _rpmalloc_allocate_huge(heap_t *heap, size_t size)
Allocate a huge block by mapping memory pages directly.
#define _rpmalloc_stat_inc_alloc(heap, class_idx)
static void _rpmalloc_heap_cache_insert(heap_t *heap, span_t *span)
Span control.
static span_t * _rpmalloc_heap_global_cache_extract(heap_t *heap, size_t span_count)
Extract a span from the global cache.
static void _rpmalloc_unmap(void *address, size_t size, size_t offset, size_t release)
Unmap virtual memory.
static void _rpmalloc_span_double_link_list_add(span_t **head, span_t *span)
Span linked list management.
static void * _rpmalloc_allocate_medium(heap_t *heap, size_t size)
Allocate a medium sized memory block from the given heap.
#define HEAP_ARRAY_SIZE
Size of heap hashmap.
static void _rpmalloc_deallocate_defer_free_span(heap_t *heap, span_t *span)
static void * free_list_pop(void **list)
Allocation entry points.
void rpmalloc_thread_statistics(rpmalloc_thread_statistics_t *stats)
Get per-thread statistics.
static heap_t * _rpmalloc_heap_extract_orphan(heap_t **heap_list)
static span_t * _rpmalloc_heap_extract_new_span(heap_t *heap, heap_size_class_t *heap_size_class, size_t span_count, uint32_t class_idx)
Get a span from one of the cache levels (thread cache, reserved, global cache) or fallback to mapping...
#define rpmalloc_assert(truth, message)
#define SPAN_FLAG_ALIGNED_BLOCKS
Flag indicating span has blocks with increased alignment.
static void _rpmalloc_span_initialize(span_t *span, size_t total_span_count, size_t span_count, size_t align_offset)
Setup a newly mapped span.
#define _memory_default_span_mask
static size_t _memory_heap_reserve_count
Number of spans to keep reserved in each heap.
static size_t _rpmalloc_usable_size(void *p)
Reallocation entry points.
static uint32_t free_list_partial_init(void **list, void **first_block, void *page_start, void *block_start, uint32_t block_count, uint32_t block_size)
Initialize a (partial) free list up to next system memory page, while reserving the first block as al...
static int _rpmalloc_span_is_fully_utilized(span_t *span)
static rpmalloc_config_t _memory_config
Configuration.
static FORCEINLINE void atomic_store_ptr(atomicptr_t *dst, void *val)
static void _rpmalloc_deallocate(void *p)
Deallocate the given block.
static heap_t * _memory_orphan_heaps
Orphaned heaps.
static span_t * _rpmalloc_span_map(heap_t *heap, size_t span_count)
Map in memory pages for the given number of spans (or use previously reserved pages)
static void _rpmalloc_heap_set_reserved_spans(heap_t *heap, span_t *master, span_t *reserve, size_t reserve_span_count)
Store the given spans as reserve in the given heap.
RPMALLOC_ALLOCATOR void * rpaligned_calloc(size_t alignment, size_t num, size_t size)
void rpmalloc_global_statistics(rpmalloc_global_statistics_t *stats)
Get global statistics.
int rpposix_memalign(void **memptr, size_t alignment, size_t size)
Allocate a memory block of at least the given size and alignment.
#define MAX_THREAD_SPAN_CACHE
Number of spans in thread cache.
static void _rpmalloc_span_double_link_list_remove(span_t **head, span_t *span)
Remove a span from double linked list.
#define GLOBAL_CACHE_MULTIPLIER
Multiplier for global cache.
#define MEDIUM_SIZE_LIMIT
Maximum size of a medium block.
#define SMALL_SIZE_LIMIT
Maximum size of a small block.
static void _rpmalloc_span_extract_free_list_deferred(span_t *span)
struct span_list_t span_list_t
Span list.
static FORCEINLINE int32_t atomic_add32(atomic32_t *val, int32_t add)
static atomic32_t _memory_heap_id
Heap ID counter.
static void _rpmalloc_heap_cache_adopt_deferred(heap_t *heap, span_t **single_span)
Adopt the deferred span cache list, optionally extracting the first single span for immediate re-use.
#define pointer_diff(first, second)
struct span_active_t span_active_t
Span active data.
#define DEFAULT_SPAN_MAP_COUNT
Default number of spans to map in call to map more virtual memory (default values yield 4MiB here)
static span_t * _rpmalloc_span_map_from_reserve(heap_t *heap, size_t span_count)
Use reserved spans to fulfill a memory map request (reserve size must be checked by caller)
static void * _rpmalloc_mmap(size_t size, size_t *offset)
Map more virtual memory.
static heap_t * get_thread_heap(void)
Get the current thread heap.
#define SMALL_CLASS_COUNT
Number of small block size classes.
void rpmalloc_finalize(void)
Finalize the allocator.
#define RPMALLOC_ALLOCATOR
#define RPMALLOC_GROW_OR_FAIL
Flag to rpaligned_realloc to fail and return null pointer if grow cannot be done in-place,...
#define RPMALLOC_NO_PRESERVE
Flag to rpaligned_realloc to not preserve content in reallocation.
uint32_t count
Cache count.
span_t * overflow
Unlimited cache overflow.
atomic32_t lock
Cache lock.
span_t * span[GLOBAL_CACHE_MULTIPLIER *MAX_THREAD_SPAN_CACHE]
Cached spans.
span_t * cache
Early level cache of fully free spans.
void * free_list
Free list of active span.
span_t * partial_span
Double linked list of partially used spans with free blocks.
int finalize
Finalization state flag.
heap_t * master_heap
Master heap owning the memory pages.
atomicptr_t span_free_deferred
List of deferred free spans (single linked list)
uintptr_t owner_thread
Owning thread ID.
atomic32_t child_count
Child count.
size_t full_span_count
Number of full spans.
heap_size_class_t size_class[SIZE_CLASS_COUNT]
Free lists for each size class.
heap_t * next_heap
Next heap in id list.
heap_t * next_orphan
Next heap in orphan list.
uint32_t spans_reserved
Number of mapped but unused spans.
span_t * span_reserve
Mapped but unused spans.
span_t * span_reserve_master
Master span for mapped but unused spans.
void(* memory_unmap)(void *address, size_t size, size_t offset, size_t release)
Unmap the memory pages starting at address and spanning the given number of bytes.
int(* map_fail_callback)(size_t size)
Called when a call to map memory pages fails (out of memory).
size_t span_map_count
Number of spans to map at each request to map new virtual memory blocks.
const char * page_name
Respectively allocated pages and huge allocated pages names for systems.
int enable_huge_pages
Enable use of large/huge pages.
const char * huge_page_name
size_t span_size
Size of a span of memory blocks.
void *(* memory_map)(size_t size, size_t *offset)
Map memory pages for the given number of bytes.
size_t page_size
Size of memory pages.
uint32_t block_size
Size of blocks in this class.
uint16_t class_idx
Class index this class is merged with.
uint16_t block_count
Number of blocks in each chunk.
span_t * span[MAX_THREAD_SPAN_CACHE]
span_t * span[MAX_THREAD_SPAN_LARGE_CACHE]
uint32_t offset_from_master
Offset from master span for subspans.
uint32_t align_offset
Alignment offset.
atomicptr_t free_list_deferred
Deferred free list.
heap_t * heap
Owning heap.
span_t * prev
Previous span.
atomic32_t remaining_spans
Remaining span counter, for master spans.
uint32_t flags
Flags and counters.
uint32_t size_class
Size class.
uint32_t block_count
Total block count of size class.
void * free_list
Free list.
uint32_t block_size
Size of a block.
uint32_t used_count
Number of used blocks remaining when in partial state.
uint32_t list_size
Size of deferred free list, or list of spans when part of a cache list.
uint32_t span_count
Number of spans.
uint32_t free_list_limit
Index of last block initialized in free list.
uint32_t total_spans
Total span counter for master spans.