#include "rpmalloc.h"
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <stdint.h>
#include <string.h>
#include <stdatomic.h>

Classes
struct	span_t

struct	span_cache_t

struct	span_large_cache_t

struct	heap_size_class_t

struct	heap_t

struct	size_class_t

struct	global_cache_t

Macros
#define	__has_builtin(b) 0
	Build time configurable limits.

#define	_rpmalloc_memcpy_const(x, y, s) memcpy(x, y, s)

#define	_rpmalloc_memset_const(x, y, s) memset(x, y, s)

#define	rpmalloc_assume(cond) 0

#define	HEAP_ARRAY_SIZE 47
	Size of heap hashmap.

#define	ENABLE_THREAD_CACHE 1
	Enable per-thread cache.

#define	ENABLE_GLOBAL_CACHE 1
	Enable global cache shared between all threads, requires thread cache.

#define	ENABLE_VALIDATE_ARGS 0
	Enable validation of args to public entry points.

#define	ENABLE_STATISTICS 0
	Enable statistics collection.

#define	ENABLE_ASSERTS 0
	Enable asserts.

#define	ENABLE_OVERRIDE 0
	Override standard library malloc/free and new/delete entry points.

#define	ENABLE_PRELOAD 0
	Support preloading.

#define	DISABLE_UNMAP 0
	Disable unmapping memory pages (also enables unlimited cache)

#define	ENABLE_UNLIMITED_CACHE 0
	Enable unlimited global cache (no unmapping until finalization)

#define	ENABLE_ADAPTIVE_THREAD_CACHE 0
	Enable adaptive thread cache size based on use heuristics.

#define	DEFAULT_SPAN_MAP_COUNT 64
	Default number of spans to map in call to map more virtual memory (default values yield 4MiB here)

#define	GLOBAL_CACHE_MULTIPLIER 8
	Multiplier for global cache.

#define	ENABLE_UNLIMITED_CACHE 0
	Enable unlimited global cache (no unmapping until finalization)

#define	ENABLE_ADAPTIVE_THREAD_CACHE 0
	Enable adaptive thread cache size based on use heuristics.

#define	PLATFORM_WINDOWS 0

#define	PLATFORM_POSIX 1

#define	FORCEINLINE inline __attribute__((__always_inline__))
	Platform and arch specifics.

#define	rpmalloc_assert(truth, message)

#define	EXPECTED(x) __builtin_expect((x), 1)

#define	UNEXPECTED(x) __builtin_expect((x), 0)

#define	_rpmalloc_stat_inc(counter)
	Statistics related functions (evaluate to nothing when statistics not enabled)

#define	_rpmalloc_stat_dec(counter)

#define	_rpmalloc_stat_add(counter, value)

#define	_rpmalloc_stat_add64(counter, value)

#define	_rpmalloc_stat_add_peak(counter, value, peak)

#define	_rpmalloc_stat_sub(counter, value)

#define	_rpmalloc_stat_inc_alloc(heap, class_idx)

#define	_rpmalloc_stat_inc_free(heap, class_idx)

#define	SMALL_GRANULARITY 16
	Preconfigured limits and sizes.

#define	SMALL_GRANULARITY_SHIFT 4
	Small granularity shift count.

#define	SMALL_CLASS_COUNT 65
	Number of small block size classes.

#define	SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1))
	Maximum size of a small block.

#define	MEDIUM_GRANULARITY 512
	Granularity of a medium allocation block.

#define	MEDIUM_GRANULARITY_SHIFT 9
	Medium granularity shift count.

#define	MEDIUM_CLASS_COUNT 61
	Number of medium block size classes.

#define	SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT)
	Total number of small + medium size classes.

#define	LARGE_CLASS_COUNT 63
	Number of large block size classes.

#define	MEDIUM_SIZE_LIMIT (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT))
	Maximum size of a medium block.

#define	LARGE_SIZE_LIMIT ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE)
	Maximum size of a large block.

#define	SPAN_HEADER_SIZE 128
	Size of a span header (must be a multiple of SMALL_GRANULARITY and a power of two)

#define	MAX_THREAD_SPAN_CACHE 400
	Number of spans in thread cache.

#define	THREAD_SPAN_CACHE_TRANSFER 64
	Number of spans to transfer between thread and global cache.

#define	MAX_THREAD_SPAN_LARGE_CACHE 100
	Number of spans in thread cache for large spans (must be greater than LARGE_CLASS_COUNT / 2)

#define	THREAD_SPAN_LARGE_CACHE_TRANSFER 6
	Number of spans to transfer between thread and global cache for large spans.

#define	pointer_offset(ptr, ofs) (void )((char )(ptr) + (ptrdiff_t)(ofs))

#define	pointer_diff(first, second) (ptrdiff_t)((const char )(first) - (const char )(second))

#define	INVALID_POINTER ((void *)((uintptr_t) - 1))

#define	SIZE_CLASS_LARGE SIZE_CLASS_COUNT

#define	SIZE_CLASS_HUGE ((uint32_t) - 1)

#define	SPAN_FLAG_MASTER 1U
	Flag indicating span is the first (master) span of a split superspan.

#define	SPAN_FLAG_SUBSPAN 2U
	Flag indicating span is a secondary (sub) span of a split superspan.

#define	SPAN_FLAG_ALIGNED_BLOCKS 4U
	Flag indicating span has blocks with increased alignment.

#define	SPAN_FLAG_UNMAPPED_MASTER 8U
	Flag indicating an unmapped master span.

#define	_memory_default_span_size (64 * 1024)
	Global data.

#define	_memory_default_span_size_shift 16

#define	_memory_default_span_mask (~((uintptr_t)(_memory_span_size - 1)))

#define	_memory_span_size _memory_default_span_size
	Hardwired span size.

#define	_memory_span_size_shift _memory_default_span_size_shift

#define	_memory_span_mask _memory_default_span_mask

#define	TLS_MODEL __attribute__((tls_model("initial-exec")))
	Thread local heap and ID.

Typedefs
typedef volatile	_Atomic(int32_t)
	Atomic access abstraction (since MSVC does not do C11 yet)

typedef struct heap_t	heap_t
	Data types.

typedef struct span_t	span_t
	Span of memory pages.

typedef struct span_list_t	span_list_t
	Span list.

typedef struct span_active_t	span_active_t
	Span active data.

typedef struct size_class_t	size_class_t
	Size class definition.

typedef struct global_cache_t	global_cache_t
	Global cache.

typedef struct span_cache_t	span_cache_t

typedef struct span_large_cache_t	span_large_cache_t

typedef struct heap_size_class_t	heap_size_class_t

Functions
static FORCEINLINE void	atomic_store32 (atomic32_t *dst, int32_t val)

static FORCEINLINE int32_t	atomic_incr32 (atomic32_t *val)

static FORCEINLINE int32_t	atomic_decr32 (atomic32_t *val)

static FORCEINLINE int32_t	atomic_add32 (atomic32_t *val, int32_t add)

static FORCEINLINE int	atomic_cas32_acquire (atomic32_t *dst, int32_t val, int32_t ref)

static FORCEINLINE void	atomic_store32_release (atomic32_t *dst, int32_t val)

static FORCEINLINE int64_t	atomic_load64 (atomic64_t *val)

static FORCEINLINE int64_t	atomic_add64 (atomic64_t *val, int64_t add)

static FORCEINLINE void *	atomic_load_ptr (atomicptr_t *src)

static FORCEINLINE void	atomic_store_ptr (atomicptr_t dst, void val)

static FORCEINLINE void	atomic_store_ptr_release (atomicptr_t dst, void val)

static FORCEINLINE void *	atomic_exchange_ptr_acquire (atomicptr_t dst, void val)

static FORCEINLINE int	atomic_cas_ptr (atomicptr_t dst, void val, void *ref)

	_Static_assert ((SMALL_GRANULARITY &(SMALL_GRANULARITY - 1))==0, "Small granularity must be power of two")

	_Static_assert ((SPAN_HEADER_SIZE &(SPAN_HEADER_SIZE - 1))==0, "Span header size must be power of two")

	_Static_assert (sizeof(span_t)<=SPAN_HEADER_SIZE, "span size mismatch")

	_Static_assert (sizeof(size_class_t)==8, "Size class size mismatch")

static heap_t *	get_thread_heap_raw (void)

static heap_t *	get_thread_heap (void)
	Get the current thread heap.

static uintptr_t	get_thread_id (void)
	Fast thread ID.

static void	set_thread_heap (heap_t *heap)
	Set the current thread heap.

void	rpmalloc_set_main_thread (void)
	Set main thread ID.

static void	_rpmalloc_spin (void)

static void	_rpmalloc_set_name (void *address, size_t size)
	Low level memory map/unmap.

static void *	_rpmalloc_mmap (size_t size, size_t *offset)
	Map more virtual memory.

static void	_rpmalloc_unmap (void *address, size_t size, size_t offset, size_t release)
	Unmap virtual memory.

static void *	_rpmalloc_mmap_os (size_t size, size_t *offset)
	Default implementation to map new pages to virtual memory.

static void	_rpmalloc_unmap_os (void *address, size_t size, size_t offset, size_t release)
	Default implementation to unmap pages from virtual memory.

static void	_rpmalloc_span_mark_as_subspan_unless_master (span_t master, span_t subspan, size_t span_count)
	Declare the span to be a subspan and store distance from master span and span count.

static span_t *	_rpmalloc_global_get_reserved_spans (size_t span_count)
	Use global reserved spans to fulfill a memory map request (reserve size must be checked by caller)

static void	_rpmalloc_global_set_reserved_spans (span_t master, span_t reserve, size_t reserve_span_count)
	Store the given spans as global reserve (must only be called from within new heap allocation, not thread safe)

static void	_rpmalloc_span_double_link_list_add (span_t *head, span_t span)
	Span linked list management.

static void	_rpmalloc_span_double_link_list_pop_head (span_t *head, span_t span)
	Pop head span from double linked list.

static void	_rpmalloc_span_double_link_list_remove (span_t *head, span_t span)
	Remove a span from double linked list.

static void	_rpmalloc_heap_cache_insert (heap_t heap, span_t span)
	Span control.

static void	_rpmalloc_heap_finalize (heap_t *heap)

static void	_rpmalloc_heap_set_reserved_spans (heap_t heap, span_t master, span_t *reserve, size_t reserve_span_count)
	Store the given spans as reserve in the given heap.

static span_t *	_rpmalloc_span_map_from_reserve (heap_t *heap, size_t span_count)
	Use reserved spans to fulfill a memory map request (reserve size must be checked by caller)

static size_t	_rpmalloc_span_align_count (size_t span_count)
	Get the aligned number of spans to map in based on wanted count, configured mapping granularity and the page size.

static void	_rpmalloc_span_initialize (span_t *span, size_t total_span_count, size_t span_count, size_t align_offset)
	Setup a newly mapped span.

static void	_rpmalloc_span_unmap (span_t *span)
	Unmap memory pages for the given number of spans (or mark as unused if no partial unmappings)

static span_t *	_rpmalloc_span_map_aligned_count (heap_t *heap, size_t span_count)
	Map an aligned set of spans, taking configured mapping granularity and the page size into account.

static span_t *	_rpmalloc_span_map (heap_t *heap, size_t span_count)
	Map in memory pages for the given number of spans (or use previously reserved pages)

static void	_rpmalloc_span_release_to_cache (heap_t heap, span_t span)
	Move the span (used for small or medium allocations) to the heap thread cache.

static uint32_t	free_list_partial_init (void list, void first_block, void page_start, void block_start, uint32_t block_count, uint32_t block_size)
	Initialize a (partial) free list up to next system memory page, while reserving the first block as allocated, returning number of blocks in list.

static void *	_rpmalloc_span_initialize_new (heap_t heap, heap_size_class_t heap_size_class, span_t *span, uint32_t class_idx)
	Initialize an unused span (from cache or mapped) to be new active span, putting the initial free list in heap class free list.

static void	_rpmalloc_span_extract_free_list_deferred (span_t *span)

static int	_rpmalloc_span_is_fully_utilized (span_t *span)

static int	_rpmalloc_span_finalize (heap_t heap, size_t iclass, span_t span, span_t **list_head)

static void	_rpmalloc_deallocate_huge (span_t *)
	Global cache.

static void	_rpmalloc_heap_cache_adopt_deferred (heap_t heap, span_t *single_span)
	Adopt the deferred span cache list, optionally extracting the first single span for immediate re-use.

static void	_rpmalloc_heap_unmap (heap_t *heap)

static void	_rpmalloc_heap_global_finalize (heap_t *heap)

static span_t *	_rpmalloc_heap_thread_cache_extract (heap_t *heap, size_t span_count)
	Extract the given number of spans from the different cache levels.

static span_t *	_rpmalloc_heap_thread_cache_deferred_extract (heap_t *heap, size_t span_count)

static span_t *	_rpmalloc_heap_reserved_extract (heap_t *heap, size_t span_count)

static span_t *	_rpmalloc_heap_global_cache_extract (heap_t *heap, size_t span_count)
	Extract a span from the global cache.

static void	_rpmalloc_inc_span_statistics (heap_t *heap, size_t span_count, uint32_t class_idx)

static span_t *	_rpmalloc_heap_extract_new_span (heap_t heap, heap_size_class_t heap_size_class, size_t span_count, uint32_t class_idx)
	Get a span from one of the cache levels (thread cache, reserved, global cache) or fallback to mapping more memory.

static void	_rpmalloc_heap_initialize (heap_t *heap)

static void	_rpmalloc_heap_orphan (heap_t *heap, int first_class)

static heap_t *	_rpmalloc_heap_allocate_new (void)
	Allocate a new heap from newly mapped memory pages.

static heap_t *	_rpmalloc_heap_extract_orphan (heap_t **heap_list)

static heap_t *	_rpmalloc_heap_allocate (int first_class)
	Allocate a new heap, potentially reusing a previously orphaned heap.

static void	_rpmalloc_heap_release (void *heapptr, int first_class, int release_cache)

static void	_rpmalloc_heap_release_raw (void *heapptr, int release_cache)

static void	_rpmalloc_heap_release_raw_fc (void *heapptr)

static void *	free_list_pop (void **list)
	Allocation entry points.

static void *	_rpmalloc_allocate_from_heap_fallback (heap_t heap, heap_size_class_t heap_size_class, uint32_t class_idx)
	Allocate a small/medium sized memory block from the given heap.

static void *	_rpmalloc_allocate_small (heap_t *heap, size_t size)
	Allocate a small sized memory block from the given heap.

static void *	_rpmalloc_allocate_medium (heap_t *heap, size_t size)
	Allocate a medium sized memory block from the given heap.

static void *	_rpmalloc_allocate_large (heap_t *heap, size_t size)
	Allocate a large sized memory block from the given heap.

static void *	_rpmalloc_allocate_huge (heap_t *heap, size_t size)
	Allocate a huge block by mapping memory pages directly.

static void *	_rpmalloc_allocate (heap_t *heap, size_t size)
	Allocate a block of the given size.

static void *	_rpmalloc_aligned_allocate (heap_t *heap, size_t alignment, size_t size)

static void	_rpmalloc_deallocate_direct_small_or_medium (span_t span, void block)
	Deallocation entry points.

static void	_rpmalloc_deallocate_defer_free_span (heap_t heap, span_t span)

static void	_rpmalloc_deallocate_defer_small_or_medium (span_t span, void block)
	Put the block in the deferred free list of the owning span.

static void	_rpmalloc_deallocate_small_or_medium (span_t span, void p)

static void	_rpmalloc_deallocate_large (span_t *span)
	Deallocate the given large memory block to the current heap.

static void	_rpmalloc_deallocate (void *p)
	Deallocate the given block.

static size_t	_rpmalloc_usable_size (void *p)
	Reallocation entry points.

static void *	_rpmalloc_reallocate (heap_t heap, void p, size_t size, size_t oldsize, unsigned int flags)
	Reallocate the given block to the given size.

static void *	_rpmalloc_aligned_reallocate (heap_t heap, void ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags)

static void	_rpmalloc_adjust_size_class (size_t iclass)
	Adjust and optimize the size class properties for the given class.

int	rpmalloc_initialize (void)
	Initialize the allocator and setup global data.

int	rpmalloc_initialize_config (const rpmalloc_config_t *config)
	Initialize allocator with given configuration.

void	rpmalloc_finalize (void)
	Finalize the allocator.

void	rpmalloc_thread_initialize (void)
	Initialize thread, assign heap.

void	rpmalloc_thread_finalize (int release_caches)
	Finalize thread, orphan heap.

int	rpmalloc_is_thread_initialized (void)
	Query if allocator is initialized for calling thread.

const rpmalloc_config_t *	rpmalloc_config (void)
	Get allocator configuration.

RPMALLOC_ALLOCATOR void *	rpmalloc (size_t size)
	Allocate a memory block of at least the given size.

void	rpfree (void *ptr)
	Free the given memory block.

RPMALLOC_ALLOCATOR void *	rpcalloc (size_t num, size_t size)

RPMALLOC_ALLOCATOR void *	rprealloc (void *ptr, size_t size)
	Reallocate the given block to at least the given size.

RPMALLOC_ALLOCATOR void *	rpaligned_realloc (void *ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags)
	Reallocate the given block to at least the given size and alignment,.

RPMALLOC_ALLOCATOR void *	rpaligned_alloc (size_t alignment, size_t size)
	Allocate a memory block of at least the given size and alignment.

RPMALLOC_ALLOCATOR void *	rpaligned_calloc (size_t alignment, size_t num, size_t size)

RPMALLOC_ALLOCATOR void *	rpmemalign (size_t alignment, size_t size)
	Allocate a memory block of at least the given size and alignment.

int	rpposix_memalign (void **memptr, size_t alignment, size_t size)
	Allocate a memory block of at least the given size and alignment.

size_t	rpmalloc_usable_size (void *ptr)
	Query the usable size of the given memory block (from given pointer to the end of block)

void	rpmalloc_thread_collect (void)
	Perform deferred deallocations pending for the calling thread heap.

void	rpmalloc_thread_statistics (rpmalloc_thread_statistics_t *stats)
	Get per-thread statistics.

void	rpmalloc_global_statistics (rpmalloc_global_statistics_t *stats)
	Get global statistics.

void	rpmalloc_dump_statistics (void *file)
	Dump all statistics in human readable format to file (should be a FILE*)

void	rpmalloc_linker_reference (void)
	Dummy empty function for forcing linker symbol inclusion.

Variables
static int	_rpmalloc_initialized
	Initialized flag.

static uintptr_t	_rpmalloc_main_thread_id
	Main thread ID.

static rpmalloc_config_t	_memory_config
	Configuration.

static size_t	_memory_page_size
	Memory page size.

static size_t	_memory_page_size_shift
	Shift to divide by page size.

static size_t	_memory_map_granularity
	Granularity at which memory pages are mapped by OS.

static size_t	_memory_span_map_count
	Number of spans to map in each map call.

static size_t	_memory_heap_reserve_count
	Number of spans to keep reserved in each heap.

static size_class_t	_memory_size_class [SIZE_CLASS_COUNT]
	Global size classes.

static size_t	_memory_medium_size_limit
	Run-time size limit of medium blocks.

static atomic32_t	_memory_heap_id
	Heap ID counter.

static int	_memory_huge_pages
	Huge page support.

static span_t *	_memory_global_reserve
	Global reserved spans.

static size_t	_memory_global_reserve_count
	Global reserved count.

static span_t *	_memory_global_reserve_master
	Global reserved master.

static heap_t *	_memory_heaps [HEAP_ARRAY_SIZE]
	All heaps.

static atomic32_t	_memory_global_lock
	Used to restrict access to mapping memory for huge pages.

static heap_t *	_memory_orphan_heaps
	Orphaned heaps.

static _Thread_local heap_t *_memory_thread_heap	TLS_MODEL

Macro Definition Documentation

◆ __has_builtin

#define __has_builtin ( b ) 0

Build time configurable limits.

Definition at line 37 of file rpmalloc.c.

◆ _memory_default_span_mask

#define _memory_default_span_mask (~((uintptr_t)(_memory_span_size - 1)))

Definition at line 760 of file rpmalloc.c.

◆ _memory_default_span_size

#define _memory_default_span_size (64 * 1024)

Global data.

Default span size (64KiB)

Definition at line 758 of file rpmalloc.c.

◆ _memory_default_span_size_shift

#define _memory_default_span_size_shift 16

Definition at line 759 of file rpmalloc.c.

◆ _memory_span_mask

#define _memory_span_mask _memory_default_span_mask

Definition at line 785 of file rpmalloc.c.

◆ _memory_span_size

#define _memory_span_size _memory_default_span_size

Hardwired span size.

Definition at line 783 of file rpmalloc.c.

◆ _memory_span_size_shift

#define _memory_span_size_shift _memory_default_span_size_shift

Definition at line 784 of file rpmalloc.c.

◆ _rpmalloc_memcpy_const

#define _rpmalloc_memcpy_const	(	x,
		y,
		s
	)	memcpy(x, y, s)

Definition at line 64 of file rpmalloc.c.

◆ _rpmalloc_memset_const

#define _rpmalloc_memset_const	(	x,
		y,
		s
	)	memset(x, y, s)

Definition at line 65 of file rpmalloc.c.

◆ _rpmalloc_stat_add

#define _rpmalloc_stat_add	(	counter,
		value
	)

Value:

do { \

} while (0)

Definition at line 427 of file rpmalloc.c.

◆ _rpmalloc_stat_add64

#define _rpmalloc_stat_add64	(	counter,
		value
	)

Value:

do { \

} while (0)

Definition at line 430 of file rpmalloc.c.

◆ _rpmalloc_stat_add_peak

#define _rpmalloc_stat_add_peak	(	counter,
		value,
		peak
	)

Value:

do { \

} while (0)

Definition at line 433 of file rpmalloc.c.

◆ _rpmalloc_stat_dec

#define _rpmalloc_stat_dec ( counter )

Value:

do { \

} while (0)

Definition at line 424 of file rpmalloc.c.

◆ _rpmalloc_stat_inc

#define _rpmalloc_stat_inc ( counter )

Value:

do { \

} while (0)

Statistics related functions (evaluate to nothing when statistics not enabled)

Definition at line 421 of file rpmalloc.c.

◆ _rpmalloc_stat_inc_alloc

#define _rpmalloc_stat_inc_alloc	(	heap,
		class_idx
	)

Value:

do { \

} while (0)

Definition at line 439 of file rpmalloc.c.

◆ _rpmalloc_stat_inc_free

#define _rpmalloc_stat_inc_free	(	heap,
		class_idx
	)

Value:

do { \

} while (0)

Definition at line 442 of file rpmalloc.c.

◆ _rpmalloc_stat_sub

#define _rpmalloc_stat_sub	(	counter,
		value
	)

Value:

do { \

} while (0)

Definition at line 436 of file rpmalloc.c.

◆ DEFAULT_SPAN_MAP_COUNT

#define DEFAULT_SPAN_MAP_COUNT 64

Default number of spans to map in call to map more virtual memory (default values yield 4MiB here)

Definition at line 129 of file rpmalloc.c.

◆ DISABLE_UNMAP

#define DISABLE_UNMAP 0

Disable unmapping memory pages (also enables unlimited cache)

Definition at line 116 of file rpmalloc.c.

◆ ENABLE_ADAPTIVE_THREAD_CACHE [1/2]

#define ENABLE_ADAPTIVE_THREAD_CACHE 0

Enable adaptive thread cache size based on use heuristics.

Definition at line 152 of file rpmalloc.c.

◆ ENABLE_ADAPTIVE_THREAD_CACHE [2/2]

#define ENABLE_ADAPTIVE_THREAD_CACHE 0

Enable adaptive thread cache size based on use heuristics.

Definition at line 152 of file rpmalloc.c.

◆ ENABLE_ASSERTS

#define ENABLE_ASSERTS 0

Enable asserts.

Definition at line 104 of file rpmalloc.c.

◆ ENABLE_GLOBAL_CACHE

#define ENABLE_GLOBAL_CACHE 1

Enable global cache shared between all threads, requires thread cache.

Definition at line 92 of file rpmalloc.c.

◆ ENABLE_OVERRIDE

#define ENABLE_OVERRIDE 0

Override standard library malloc/free and new/delete entry points.

Definition at line 108 of file rpmalloc.c.

◆ ENABLE_PRELOAD

#define ENABLE_PRELOAD 0

Support preloading.

Definition at line 112 of file rpmalloc.c.

◆ ENABLE_STATISTICS

#define ENABLE_STATISTICS 0

Enable statistics collection.

Definition at line 100 of file rpmalloc.c.

◆ ENABLE_THREAD_CACHE

#define ENABLE_THREAD_CACHE 1

Enable per-thread cache.

Definition at line 88 of file rpmalloc.c.

◆ ENABLE_UNLIMITED_CACHE [1/2]

#define ENABLE_UNLIMITED_CACHE 0

Enable unlimited global cache (no unmapping until finalization)

Definition at line 147 of file rpmalloc.c.

◆ ENABLE_UNLIMITED_CACHE [2/2]

#define ENABLE_UNLIMITED_CACHE 0

Enable unlimited global cache (no unmapping until finalization)

Definition at line 147 of file rpmalloc.c.

◆ ENABLE_VALIDATE_ARGS

#define ENABLE_VALIDATE_ARGS 0

Enable validation of args to public entry points.

Definition at line 96 of file rpmalloc.c.

◆ EXPECTED

#define EXPECTED ( x ) __builtin_expect((x), 1)

Definition at line 380 of file rpmalloc.c.

◆ FORCEINLINE

#define FORCEINLINE inline __attribute__((__always_inline__))

Platform and arch specifics.

Definition at line 172 of file rpmalloc.c.

◆ GLOBAL_CACHE_MULTIPLIER

#define GLOBAL_CACHE_MULTIPLIER 8

Multiplier for global cache.

Definition at line 133 of file rpmalloc.c.

◆ HEAP_ARRAY_SIZE

#define HEAP_ARRAY_SIZE 47

Size of heap hashmap.

Definition at line 84 of file rpmalloc.c.

◆ INVALID_POINTER

#define INVALID_POINTER ((void *)((uintptr_t) - 1))

Definition at line 503 of file rpmalloc.c.

◆ LARGE_CLASS_COUNT

#define LARGE_CLASS_COUNT 63

Number of large block size classes.

Definition at line 468 of file rpmalloc.c.

◆ LARGE_SIZE_LIMIT

#define LARGE_SIZE_LIMIT ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE)

Maximum size of a large block.

Definition at line 473 of file rpmalloc.c.

◆ MAX_THREAD_SPAN_CACHE

#define MAX_THREAD_SPAN_CACHE 400

Number of spans in thread cache.

Definition at line 479 of file rpmalloc.c.

◆ MAX_THREAD_SPAN_LARGE_CACHE

#define MAX_THREAD_SPAN_LARGE_CACHE 100

Number of spans in thread cache for large spans (must be greater than LARGE_CLASS_COUNT / 2)

Definition at line 484 of file rpmalloc.c.

◆ MEDIUM_CLASS_COUNT

#define MEDIUM_CLASS_COUNT 61

Number of medium block size classes.

Definition at line 464 of file rpmalloc.c.

◆ MEDIUM_GRANULARITY

#define MEDIUM_GRANULARITY 512

Granularity of a medium allocation block.

Definition at line 460 of file rpmalloc.c.

◆ MEDIUM_GRANULARITY_SHIFT

#define MEDIUM_GRANULARITY_SHIFT 9

Medium granularity shift count.

Definition at line 462 of file rpmalloc.c.

◆ MEDIUM_SIZE_LIMIT

#define MEDIUM_SIZE_LIMIT (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT))

Maximum size of a medium block.

Definition at line 470 of file rpmalloc.c.

◆ PLATFORM_POSIX

#define PLATFORM_POSIX 1

Definition at line 160 of file rpmalloc.c.

◆ PLATFORM_WINDOWS

#define PLATFORM_WINDOWS 0

Definition at line 159 of file rpmalloc.c.

◆ pointer_diff

#define pointer_diff	(	first,
		second
	)	(ptrdiff_t)((const char )(first) - (const char )(second))

Definition at line 500 of file rpmalloc.c.

◆ pointer_offset

#define pointer_offset	(	ptr,
		ofs
	)	(void )((char )(ptr) + (ptrdiff_t)(ofs))

Definition at line 499 of file rpmalloc.c.

◆ rpmalloc_assert

#define rpmalloc_assert	(	truth,
		message
	)

Value:

do { \

} while (0)

Definition at line 258 of file rpmalloc.c.

◆ rpmalloc_assume

#define rpmalloc_assume ( cond ) 0

Definition at line 79 of file rpmalloc.c.

◆ SIZE_CLASS_COUNT

#define SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT)

Total number of small + medium size classes.

Definition at line 466 of file rpmalloc.c.

◆ SIZE_CLASS_HUGE

#define SIZE_CLASS_HUGE ((uint32_t) - 1)

Definition at line 506 of file rpmalloc.c.

◆ SIZE_CLASS_LARGE

#define SIZE_CLASS_LARGE SIZE_CLASS_COUNT

Definition at line 505 of file rpmalloc.c.

◆ SMALL_CLASS_COUNT

#define SMALL_CLASS_COUNT 65

Number of small block size classes.

Definition at line 456 of file rpmalloc.c.

◆ SMALL_GRANULARITY

#define SMALL_GRANULARITY 16

Preconfigured limits and sizes.

Granularity of a small allocation block (must be power of two)

Definition at line 452 of file rpmalloc.c.

◆ SMALL_GRANULARITY_SHIFT

#define SMALL_GRANULARITY_SHIFT 4

Small granularity shift count.

Definition at line 454 of file rpmalloc.c.

◆ SMALL_SIZE_LIMIT

#define SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1))

Maximum size of a small block.

Definition at line 458 of file rpmalloc.c.

◆ SPAN_FLAG_ALIGNED_BLOCKS

#define SPAN_FLAG_ALIGNED_BLOCKS 4U

Flag indicating span has blocks with increased alignment.

Definition at line 532 of file rpmalloc.c.

◆ SPAN_FLAG_MASTER

#define SPAN_FLAG_MASTER 1U

Flag indicating span is the first (master) span of a split superspan.

Definition at line 528 of file rpmalloc.c.

◆ SPAN_FLAG_SUBSPAN

#define SPAN_FLAG_SUBSPAN 2U

Flag indicating span is a secondary (sub) span of a split superspan.

Definition at line 530 of file rpmalloc.c.

◆ SPAN_FLAG_UNMAPPED_MASTER

#define SPAN_FLAG_UNMAPPED_MASTER 8U

Flag indicating an unmapped master span.

Definition at line 534 of file rpmalloc.c.

◆ SPAN_HEADER_SIZE

#define SPAN_HEADER_SIZE 128

Size of a span header (must be a multiple of SMALL_GRANULARITY and a power of two)

Definition at line 477 of file rpmalloc.c.

◆ THREAD_SPAN_CACHE_TRANSFER

#define THREAD_SPAN_CACHE_TRANSFER 64

Number of spans to transfer between thread and global cache.

Definition at line 481 of file rpmalloc.c.

◆ THREAD_SPAN_LARGE_CACHE_TRANSFER

#define THREAD_SPAN_LARGE_CACHE_TRANSFER 6

Number of spans to transfer between thread and global cache for large spans.

Definition at line 486 of file rpmalloc.c.

◆ TLS_MODEL

#define TLS_MODEL __attribute__((tls_model("initial-exec")))

Thread local heap and ID.

Current thread heap

Definition at line 862 of file rpmalloc.c.

◆ UNEXPECTED

#define UNEXPECTED ( x ) __builtin_expect((x), 0)

Definition at line 381 of file rpmalloc.c.

Typedef Documentation

◆ _Atomic

typedef volatile _Atomic(int32_t)

Atomic access abstraction (since MSVC does not do C11 yet)

Definition at line 329 of file rpmalloc.c.

◆ global_cache_t

typedef struct global_cache_t global_cache_t

Global cache.

Definition at line 525 of file rpmalloc.c.

◆ heap_size_class_t

typedef struct heap_size_class_t heap_size_class_t

Definition at line 662 of file rpmalloc.c.

◆ heap_t

typedef struct heap_t heap_t

Data types.

A memory heap, per thread

Definition at line 515 of file rpmalloc.c.

◆ size_class_t

typedef struct size_class_t size_class_t

Size class definition.

Definition at line 523 of file rpmalloc.c.

◆ span_active_t

typedef struct span_active_t span_active_t

Span active data.

Definition at line 521 of file rpmalloc.c.

◆ span_cache_t

typedef struct span_cache_t span_cache_t

Definition at line 645 of file rpmalloc.c.

◆ span_large_cache_t

typedef struct span_large_cache_t span_large_cache_t

Definition at line 651 of file rpmalloc.c.

◆ span_list_t

typedef struct span_list_t span_list_t

Span list.

Definition at line 519 of file rpmalloc.c.

◆ span_t

typedef struct span_t span_t

Span of memory pages.

Definition at line 517 of file rpmalloc.c.

Function Documentation

◆ _rpmalloc_adjust_size_class()

static void _rpmalloc_adjust_size_class ( size_t iclass )

static

Adjust and optimize the size class properties for the given class.

Definition at line 3022 of file rpmalloc.c.

References _memory_size_class, _memory_span_size, _rpmalloc_memcpy_const, size_class_t::block_count, size_class_t::block_size, size_class_t::class_idx, SMALL_CLASS_COUNT, and SPAN_HEADER_SIZE.

Referenced by rpmalloc_initialize_config().

◆ _rpmalloc_aligned_allocate()

static void * _rpmalloc_aligned_allocate	(	heap_t *	heap,
		size_t	alignment,
		size_t	size
	)

static

◆ _rpmalloc_aligned_reallocate()

static void * _rpmalloc_aligned_reallocate	(	heap_t *	heap,
		void *	ptr,
		size_t	alignment,
		size_t	size,
		size_t	oldsize,
		unsigned int	flags
	)

static

Definition at line 2969 of file rpmalloc.c.

References _rpmalloc_aligned_allocate(), _rpmalloc_deallocate(), _rpmalloc_reallocate(), _rpmalloc_usable_size(), block, EXPECTED, RPMALLOC_GROW_OR_FAIL, RPMALLOC_NO_PRESERVE, and SMALL_GRANULARITY.

Referenced by rpaligned_realloc().

◆ _rpmalloc_allocate()

static void * _rpmalloc_allocate	(	heap_t *	heap,
		size_t	size
	)

static

Allocate a block of the given size.

Definition at line 2530 of file rpmalloc.c.

References _memory_medium_size_limit, _rpmalloc_allocate_huge(), _rpmalloc_allocate_large(), _rpmalloc_allocate_medium(), _rpmalloc_allocate_small(), _rpmalloc_stat_add64, EXPECTED, LARGE_SIZE_LIMIT, and SMALL_SIZE_LIMIT.

Referenced by _rpmalloc_aligned_allocate(), _rpmalloc_reallocate(), rpcalloc(), and rpmalloc().

◆ _rpmalloc_allocate_from_heap_fallback()

static void * _rpmalloc_allocate_from_heap_fallback	(	heap_t *	heap,
		heap_size_class_t *	heap_size_class,
		uint32_t	class_idx
	)

static

Allocate a small/medium sized memory block from the given heap.

Definition at line 2376 of file rpmalloc.c.

References _memory_page_size, _memory_size_class, _rpmalloc_heap_extract_new_span(), _rpmalloc_span_double_link_list_add(), _rpmalloc_span_double_link_list_pop_head(), _rpmalloc_span_extract_free_list_deferred(), _rpmalloc_span_initialize_new(), _rpmalloc_span_is_fully_utilized(), atomic_load_ptr(), block, span_t::block_count, size_class_t::block_count, span_t::block_size, EXPECTED, span_t::free_list, heap_size_class_t::free_list, span_t::free_list_deferred, span_t::free_list_limit, free_list_partial_init(), free_list_pop(), heap_t::full_span_count, heap_size_class_t::partial_span, pointer_offset, rpmalloc_assert, rpmalloc_assume, span_t::size_class, SPAN_HEADER_SIZE, and span_t::used_count.

Referenced by _rpmalloc_allocate_medium(), and _rpmalloc_allocate_small().

◆ _rpmalloc_allocate_huge()

static void * _rpmalloc_allocate_huge	(	heap_t *	heap,
		size_t	size
	)

static

Allocate a huge block by mapping memory pages directly.

Definition at line 2501 of file rpmalloc.c.

References _memory_page_size, _memory_page_size_shift, _rpmalloc_heap_cache_adopt_deferred(), _rpmalloc_mmap(), _rpmalloc_span_double_link_list_add(), _rpmalloc_stat_add_peak, span_t::align_offset, heap_t::full_span_count, span_t::heap, pointer_offset, rpmalloc_assert, span_t::size_class, SIZE_CLASS_HUGE, span_t::span_count, and SPAN_HEADER_SIZE.

Referenced by _rpmalloc_allocate().

◆ _rpmalloc_allocate_large()

static void * _rpmalloc_allocate_large	(	heap_t *	heap,
		size_t	size
	)

static

Allocate a large sized memory block from the given heap.

Definition at line 2471 of file rpmalloc.c.

References _memory_span_size, _memory_span_size_shift, _rpmalloc_heap_extract_new_span(), _rpmalloc_span_double_link_list_add(), heap_t::full_span_count, span_t::heap, pointer_offset, rpmalloc_assert, span_t::size_class, SIZE_CLASS_LARGE, span_t::span_count, and SPAN_HEADER_SIZE.

Referenced by _rpmalloc_allocate().

◆ _rpmalloc_allocate_medium()

static void * _rpmalloc_allocate_medium	(	heap_t *	heap,
		size_t	size
	)

static

Allocate a medium sized memory block from the given heap.

Definition at line 2454 of file rpmalloc.c.

References _memory_size_class, _rpmalloc_allocate_from_heap_fallback(), _rpmalloc_stat_inc_alloc, size_class_t::class_idx, EXPECTED, heap_size_class_t::free_list, free_list_pop(), MEDIUM_GRANULARITY_SHIFT, rpmalloc_assert, heap_t::size_class, SMALL_CLASS_COUNT, and SMALL_SIZE_LIMIT.

Referenced by _rpmalloc_allocate().

◆ _rpmalloc_allocate_small()

static void * _rpmalloc_allocate_small	(	heap_t *	heap,
		size_t	size
	)

static

Allocate a small sized memory block from the given heap.

Definition at line 2440 of file rpmalloc.c.

References _rpmalloc_allocate_from_heap_fallback(), _rpmalloc_stat_inc_alloc, EXPECTED, heap_size_class_t::free_list, free_list_pop(), rpmalloc_assert, heap_t::size_class, SMALL_GRANULARITY, and SMALL_GRANULARITY_SHIFT.

Referenced by _rpmalloc_allocate().

◆ _rpmalloc_deallocate()

static void _rpmalloc_deallocate ( void * p )

static

Deallocate the given block.

Definition at line 2862 of file rpmalloc.c.

References _memory_span_mask, _rpmalloc_deallocate_huge(), _rpmalloc_deallocate_large(), _rpmalloc_deallocate_small_or_medium(), _rpmalloc_stat_add64, EXPECTED, span_t::size_class, SIZE_CLASS_COUNT, SIZE_CLASS_LARGE, and UNEXPECTED.

Referenced by _rpmalloc_aligned_reallocate(), _rpmalloc_reallocate(), and rpfree().

◆ _rpmalloc_deallocate_defer_free_span()

static void _rpmalloc_deallocate_defer_free_span	(	heap_t *	heap,
		span_t *	span
	)

static

Definition at line 2711 of file rpmalloc.c.

References _rpmalloc_stat_inc, atomic_cas_ptr(), atomic_load_ptr(), span_t::free_list, span_t::size_class, SIZE_CLASS_HUGE, span_t::span_count, and heap_t::span_free_deferred.

Referenced by _rpmalloc_deallocate_defer_small_or_medium(), _rpmalloc_deallocate_huge(), and _rpmalloc_deallocate_large().

◆ _rpmalloc_deallocate_defer_small_or_medium()

static void _rpmalloc_deallocate_defer_small_or_medium	(	span_t *	span,
		void *	block
	)

static

Put the block in the deferred free list of the owning span.

Definition at line 2721 of file rpmalloc.c.

References _rpmalloc_deallocate_defer_free_span(), atomic_exchange_ptr_acquire(), atomic_store_ptr_release(), block, span_t::block_count, span_t::free_list_deferred, span_t::heap, INVALID_POINTER, and span_t::list_size.

Referenced by _rpmalloc_deallocate_small_or_medium().

◆ _rpmalloc_deallocate_direct_small_or_medium()

static void _rpmalloc_deallocate_direct_small_or_medium	(	span_t *	span,
		void *	block
	)

static

Deallocation entry points.

Deallocate the given small/medium memory block in the current thread local heap

Definition at line 2671 of file rpmalloc.c.

References _rpmalloc_span_double_link_list_add(), _rpmalloc_span_double_link_list_remove(), _rpmalloc_span_is_fully_utilized(), _rpmalloc_span_release_to_cache(), atomic_exchange_ptr_acquire(), atomic_store_ptr_release(), block, span_t::block_count, heap_t::finalize, span_t::free_list, span_t::free_list_deferred, heap_t::full_span_count, get_thread_id(), span_t::heap, INVALID_POINTER, span_t::list_size, heap_t::owner_thread, heap_size_class_t::partial_span, rpmalloc_assert, span_t::size_class, heap_t::size_class, UNEXPECTED, and span_t::used_count.

Referenced by _rpmalloc_deallocate_small_or_medium().

◆ _rpmalloc_deallocate_huge()

static void _rpmalloc_deallocate_huge ( span_t * span )

static

Global cache.

Deallocate the given huge span.

Heap control

Definition at line 2834 of file rpmalloc.c.

References _memory_page_size, _rpmalloc_deallocate_defer_free_span(), _rpmalloc_span_double_link_list_remove(), _rpmalloc_stat_sub, _rpmalloc_unmap(), span_t::align_offset, heap_t::finalize, heap_t::full_span_count, get_thread_id(), span_t::heap, heap_t::owner_thread, rpmalloc_assert, and span_t::span_count.

Referenced by _rpmalloc_deallocate(), and _rpmalloc_heap_cache_adopt_deferred().

◆ _rpmalloc_deallocate_large()

static void _rpmalloc_deallocate_large ( span_t * span )

static

Deallocate the given large memory block to the current heap.

Definition at line 2769 of file rpmalloc.c.

References _memory_span_size, _rpmalloc_deallocate_defer_free_span(), _rpmalloc_heap_cache_insert(), _rpmalloc_span_double_link_list_remove(), _rpmalloc_stat_inc, atomic_decr32(), heap_t::finalize, span_t::flags, heap_t::full_span_count, get_thread_id(), span_t::heap, span_t::offset_from_master, heap_t::owner_thread, pointer_offset, span_t::remaining_spans, rpmalloc_assert, span_t::size_class, SIZE_CLASS_LARGE, span_t::span_count, SPAN_FLAG_MASTER, SPAN_FLAG_SUBSPAN, heap_t::span_reserve, heap_t::span_reserve_master, and heap_t::spans_reserved.

Referenced by _rpmalloc_deallocate().

◆ _rpmalloc_deallocate_small_or_medium()

static void _rpmalloc_deallocate_small_or_medium	(	span_t *	span,
		void *	p
	)

static

Definition at line 2745 of file rpmalloc.c.

References _rpmalloc_deallocate_defer_small_or_medium(), _rpmalloc_deallocate_direct_small_or_medium(), _rpmalloc_stat_inc_free, span_t::block_size, heap_t::finalize, span_t::flags, get_thread_id(), span_t::heap, heap_t::owner_thread, pointer_diff, pointer_offset, span_t::size_class, SPAN_FLAG_ALIGNED_BLOCKS, and SPAN_HEADER_SIZE.

Referenced by _rpmalloc_deallocate().

◆ _rpmalloc_global_get_reserved_spans()

static span_t * _rpmalloc_global_get_reserved_spans ( size_t span_count )

static

Use global reserved spans to fulfill a memory map request (reserve size must be checked by caller)

Definition at line 1186 of file rpmalloc.c.

References _memory_global_reserve, _memory_global_reserve_count, _memory_global_reserve_master, _memory_span_size_shift, _rpmalloc_span_mark_as_subspan_unless_master(), and pointer_offset.

Referenced by _rpmalloc_heap_allocate_new(), and _rpmalloc_span_map().

◆ _rpmalloc_global_set_reserved_spans()

static void _rpmalloc_global_set_reserved_spans	(	span_t *	master,
		span_t *	reserve,
		size_t	reserve_span_count
	)

static

Store the given spans as global reserve (must only be called from within new heap allocation, not thread safe)

Definition at line 1201 of file rpmalloc.c.

References _memory_global_reserve, _memory_global_reserve_count, and _memory_global_reserve_master.

Referenced by _rpmalloc_heap_allocate_new(), and _rpmalloc_span_map_aligned_count().

◆ _rpmalloc_heap_allocate()

static heap_t * _rpmalloc_heap_allocate ( int first_class )

static

Allocate a new heap, potentially reusing a previously orphaned heap.

Definition at line 2223 of file rpmalloc.c.

References _memory_global_lock, _memory_orphan_heaps, _rpmalloc_heap_allocate_new(), _rpmalloc_heap_cache_adopt_deferred(), _rpmalloc_heap_extract_orphan(), _rpmalloc_spin(), atomic_cas32_acquire(), and atomic_store32_release().

Referenced by rpmalloc_thread_initialize().

◆ _rpmalloc_heap_allocate_new()

static heap_t * _rpmalloc_heap_allocate_new ( void )

static

Allocate a new heap from newly mapped memory pages.

Definition at line 2131 of file rpmalloc.c.

References _memory_global_reserve_count, _memory_heap_reserve_count, _memory_page_size, _memory_span_size, _rpmalloc_global_get_reserved_spans(), _rpmalloc_global_set_reserved_spans(), _rpmalloc_heap_initialize(), _rpmalloc_heap_orphan(), _rpmalloc_heap_set_reserved_spans(), _rpmalloc_mmap(), _rpmalloc_span_initialize(), _rpmalloc_stat_inc, atomic_store32(), heap_t::master_heap, and pointer_offset.

Referenced by _rpmalloc_heap_allocate().

◆ _rpmalloc_heap_cache_adopt_deferred()

static void _rpmalloc_heap_cache_adopt_deferred	(	heap_t *	heap,
		span_t **	single_span
	)

static

Adopt the deferred span cache list, optionally extracting the first single span for immediate re-use.

Definition at line 1785 of file rpmalloc.c.

References _rpmalloc_deallocate_huge(), _rpmalloc_heap_cache_insert(), _rpmalloc_span_double_link_list_remove(), _rpmalloc_stat_dec, atomic_exchange_ptr_acquire(), EXPECTED, span_t::free_list, heap_t::full_span_count, span_t::heap, rpmalloc_assert, span_t::size_class, SIZE_CLASS_COUNT, SIZE_CLASS_HUGE, SIZE_CLASS_LARGE, span_t::span_count, and heap_t::span_free_deferred.

Referenced by _rpmalloc_allocate_huge(), _rpmalloc_heap_allocate(), _rpmalloc_heap_finalize(), _rpmalloc_heap_release(), and _rpmalloc_heap_thread_cache_deferred_extract().

◆ _rpmalloc_heap_cache_insert()

static void _rpmalloc_heap_cache_insert	(	heap_t *	heap,
		span_t *	span
	)

static

Span control.

Insert a single span into thread heap cache, releasing to global cache if overflow.

Definition at line 1892 of file rpmalloc.c.

References _memory_span_size, _rpmalloc_heap_global_finalize(), _rpmalloc_span_unmap(), _rpmalloc_stat_add, _rpmalloc_stat_add64, _rpmalloc_stat_inc, span_cache_t::count, span_large_cache_t::count, heap_t::finalize, MAX_THREAD_SPAN_CACHE, MAX_THREAD_SPAN_LARGE_CACHE, span_cache_t::span, span_large_cache_t::span, span_t::span_count, THREAD_SPAN_CACHE_TRANSFER, THREAD_SPAN_LARGE_CACHE_TRANSFER, and UNEXPECTED.

Referenced by _rpmalloc_deallocate_large(), _rpmalloc_heap_cache_adopt_deferred(), _rpmalloc_span_map_aligned_count(), and _rpmalloc_span_release_to_cache().

◆ _rpmalloc_heap_extract_new_span()

static span_t * _rpmalloc_heap_extract_new_span	(	heap_t *	heap,
		heap_size_class_t *	heap_size_class,
		size_t	span_count,
		uint32_t	class_idx
	)

static

Get a span from one of the cache levels (thread cache, reserved, global cache) or fallback to mapping more memory.

Definition at line 2050 of file rpmalloc.c.

References _rpmalloc_heap_global_cache_extract(), _rpmalloc_heap_reserved_extract(), _rpmalloc_heap_thread_cache_deferred_extract(), _rpmalloc_heap_thread_cache_extract(), _rpmalloc_inc_span_statistics(), _rpmalloc_span_map(), _rpmalloc_stat_inc, heap_size_class_t::cache, EXPECTED, and LARGE_CLASS_COUNT.

Referenced by _rpmalloc_allocate_from_heap_fallback(), and _rpmalloc_allocate_large().

◆ _rpmalloc_heap_extract_orphan()

static heap_t * _rpmalloc_heap_extract_orphan ( heap_t ** heap_list )

static

Definition at line 2216 of file rpmalloc.c.

References heap_t::next_orphan.

Referenced by _rpmalloc_heap_allocate().

◆ _rpmalloc_heap_finalize()

static void _rpmalloc_heap_finalize ( heap_t * heap )

static

◆ _rpmalloc_heap_global_cache_extract()

static span_t * _rpmalloc_heap_global_cache_extract	(	heap_t *	heap,
		size_t	span_count
	)

static

Extract a span from the global cache.

Definition at line 1992 of file rpmalloc.c.

References _memory_span_size, _rpmalloc_stat_add, _rpmalloc_stat_add64, span_cache_t::count, span_cache_t::span, THREAD_SPAN_CACHE_TRANSFER, and THREAD_SPAN_LARGE_CACHE_TRANSFER.

Referenced by _rpmalloc_heap_extract_new_span().

◆ _rpmalloc_heap_global_finalize()

static void _rpmalloc_heap_global_finalize ( heap_t * heap )

static

Definition at line 1843 of file rpmalloc.c.

References _memory_heaps, _rpmalloc_heap_finalize(), _rpmalloc_heap_unmap(), _rpmalloc_span_unmap(), span_cache_t::count, heap_t::finalize, heap_size_class_t::free_list, heap_t::full_span_count, HEAP_ARRAY_SIZE, heap_t::id, if(), LARGE_CLASS_COUNT, heap_t::next_heap, heap_size_class_t::partial_span, heap_t::size_class, SIZE_CLASS_COUNT, and span_cache_t::span.

Referenced by _rpmalloc_heap_cache_insert(), and rpmalloc_finalize().

◆ _rpmalloc_heap_initialize()

static void _rpmalloc_heap_initialize ( heap_t * heap )

static

Definition at line 2106 of file rpmalloc.c.

References _memory_heap_id, _memory_heaps, _rpmalloc_memset_const, atomic_incr32(), HEAP_ARRAY_SIZE, heap_t::id, and heap_t::next_heap.

Referenced by _rpmalloc_heap_allocate_new().

◆ _rpmalloc_heap_orphan()

static void _rpmalloc_heap_orphan	(	heap_t *	heap,
		int	first_class
	)

static

Definition at line 2117 of file rpmalloc.c.

References _memory_orphan_heaps, heap_t::next_orphan, and heap_t::owner_thread.

Referenced by _rpmalloc_heap_allocate_new(), and _rpmalloc_heap_release().

◆ _rpmalloc_heap_release()

static void _rpmalloc_heap_release	(	void *	heapptr,
		int	first_class,
		int	release_cache
	)

static

◆ _rpmalloc_heap_release_raw()

static void _rpmalloc_heap_release_raw	(	void *	heapptr,
		int	release_cache
	)

static

Definition at line 2299 of file rpmalloc.c.

References _rpmalloc_heap_release().

Referenced by _rpmalloc_heap_release_raw_fc(), and rpmalloc_thread_finalize().

◆ _rpmalloc_heap_release_raw_fc()

static void _rpmalloc_heap_release_raw_fc ( void * heapptr )

static

Definition at line 2303 of file rpmalloc.c.

References _rpmalloc_heap_release_raw().

Referenced by rpmalloc_initialize_config().

◆ _rpmalloc_heap_reserved_extract()

static span_t * _rpmalloc_heap_reserved_extract	(	heap_t *	heap,
		size_t	span_count
	)

static

Definition at line 1984 of file rpmalloc.c.

References _rpmalloc_span_map(), and heap_t::spans_reserved.

Referenced by _rpmalloc_heap_extract_new_span().

◆ _rpmalloc_heap_set_reserved_spans()

static void _rpmalloc_heap_set_reserved_spans	(	heap_t *	heap,
		span_t *	master,
		span_t *	reserve,
		size_t	reserve_span_count
	)

static

Store the given spans as reserve in the given heap.

Definition at line 1775 of file rpmalloc.c.

References heap_t::span_reserve, heap_t::span_reserve_master, and heap_t::spans_reserved.

Referenced by _rpmalloc_heap_allocate_new(), _rpmalloc_span_map(), and _rpmalloc_span_map_aligned_count().

◆ _rpmalloc_heap_thread_cache_deferred_extract()

static span_t * _rpmalloc_heap_thread_cache_deferred_extract	(	heap_t *	heap,
		size_t	span_count
	)

static

Definition at line 1972 of file rpmalloc.c.

References _rpmalloc_heap_cache_adopt_deferred(), and _rpmalloc_heap_thread_cache_extract().

Referenced by _rpmalloc_heap_extract_new_span().

◆ _rpmalloc_heap_thread_cache_extract()

static span_t * _rpmalloc_heap_thread_cache_extract	(	heap_t *	heap,
		size_t	span_count
	)

static

Extract the given number of spans from the different cache levels.

Definition at line 1955 of file rpmalloc.c.

References _rpmalloc_stat_inc, span_cache_t::count, and span_cache_t::span.

Referenced by _rpmalloc_heap_extract_new_span(), and _rpmalloc_heap_thread_cache_deferred_extract().

◆ _rpmalloc_heap_unmap()

static void _rpmalloc_heap_unmap ( heap_t * heap )

static

Definition at line 1830 of file rpmalloc.c.

References _memory_span_mask, _rpmalloc_heap_unmap(), _rpmalloc_span_unmap(), atomic_decr32(), heap_t::child_count, heap_t::finalize, and heap_t::master_heap.

Referenced by _rpmalloc_heap_global_finalize(), and _rpmalloc_heap_unmap().

◆ _rpmalloc_inc_span_statistics()

static void _rpmalloc_inc_span_statistics	(	heap_t *	heap,
		size_t	span_count,
		uint32_t	class_idx
	)

static

Definition at line 2031 of file rpmalloc.c.

References _rpmalloc_stat_add_peak, atomic_incr32(), and atomic_store32().

Referenced by _rpmalloc_heap_extract_new_span().

◆ _rpmalloc_mmap()

static void * _rpmalloc_mmap	(	size_t	size,
		size_t *	offset
	)

static

Map more virtual memory.

Definition at line 1006 of file rpmalloc.c.

References _memory_config, _memory_page_size, _memory_page_size_shift, _rpmalloc_stat_add, _rpmalloc_stat_add_peak, EXPECTED, rpmalloc_config_t::memory_map, and rpmalloc_assert.

Referenced by _rpmalloc_aligned_allocate(), _rpmalloc_allocate_huge(), _rpmalloc_heap_allocate_new(), and _rpmalloc_span_map_aligned_count().

◆ _rpmalloc_mmap_os()

static void * _rpmalloc_mmap_os	(	size_t	size,
		size_t *	offset
	)

static

Default implementation to map new pages to virtual memory.

Definition at line 1038 of file rpmalloc.c.

References _memory_config, _memory_huge_pages, _memory_map_granularity, _memory_page_size, _memory_page_size_shift, _memory_span_mask, _memory_span_size, _rpmalloc_mmap_os(), _rpmalloc_set_name(), _rpmalloc_stat_add, rpmalloc_config_t::map_fail_callback, pointer_offset, and rpmalloc_assert.

Referenced by _rpmalloc_mmap_os(), and rpmalloc_initialize_config().

◆ _rpmalloc_reallocate()

static void * _rpmalloc_reallocate	(	heap_t *	heap,
		void *	p,
		size_t	size,
		size_t	oldsize,
		unsigned int	flags
	)

static

Reallocate the given block to the given size.

Definition at line 2885 of file rpmalloc.c.

References _memory_page_size, _memory_page_size_shift, _memory_span_mask, _memory_span_size, _memory_span_size_shift, _rpmalloc_allocate(), _rpmalloc_deallocate(), block, span_t::block_size, EXPECTED, pointer_diff, pointer_offset, rpmalloc_assert, RPMALLOC_GROW_OR_FAIL, RPMALLOC_NO_PRESERVE, span_t::size_class, SIZE_CLASS_COUNT, SIZE_CLASS_LARGE, span_t::span_count, and SPAN_HEADER_SIZE.

Referenced by _rpmalloc_aligned_reallocate(), and rprealloc().

◆ _rpmalloc_set_name()

static void _rpmalloc_set_name	(	void *	address,
		size_t	size
	)

static

Low level memory map/unmap.

Definition at line 986 of file rpmalloc.c.

References _memory_config, _memory_huge_pages, rpmalloc_config_t::huge_page_name, name, and rpmalloc_config_t::page_name.

Referenced by _rpmalloc_mmap_os().

◆ _rpmalloc_span_align_count()

static size_t _rpmalloc_span_align_count ( size_t span_count )

static

Get the aligned number of spans to map in based on wanted count, configured mapping granularity and the page size.

Definition at line 1296 of file rpmalloc.c.

References _memory_page_size, _memory_span_map_count, and _memory_span_size.

Referenced by _rpmalloc_span_map_aligned_count().

◆ _rpmalloc_span_double_link_list_add()

static void _rpmalloc_span_double_link_list_add	(	span_t **	head,
		span_t *	span
	)

static

Span linked list management.

Add a span to double linked list at the head

Definition at line 1215 of file rpmalloc.c.

References span_t::next.

Referenced by _rpmalloc_aligned_allocate(), _rpmalloc_allocate_from_heap_fallback(), _rpmalloc_allocate_huge(), _rpmalloc_allocate_large(), _rpmalloc_deallocate_direct_small_or_medium(), _rpmalloc_heap_finalize(), and _rpmalloc_span_initialize_new().

◆ _rpmalloc_span_double_link_list_pop_head()

static void _rpmalloc_span_double_link_list_pop_head	(	span_t **	head,
		span_t *	span
	)

static

Pop head span from double linked list.

Definition at line 1223 of file rpmalloc.c.

References span_t::next, and rpmalloc_assert.

Referenced by _rpmalloc_allocate_from_heap_fallback().

◆ _rpmalloc_span_double_link_list_remove()

static void _rpmalloc_span_double_link_list_remove	(	span_t **	head,
		span_t *	span
	)

static

Remove a span from double linked list.

Definition at line 1231 of file rpmalloc.c.

References EXPECTED, span_t::next, span_t::prev, and rpmalloc_assert.

Referenced by _rpmalloc_deallocate_direct_small_or_medium(), _rpmalloc_deallocate_huge(), _rpmalloc_deallocate_large(), _rpmalloc_heap_cache_adopt_deferred(), _rpmalloc_heap_finalize(), and _rpmalloc_span_finalize().

◆ _rpmalloc_span_extract_free_list_deferred()

static void _rpmalloc_span_extract_free_list_deferred ( span_t * span )

static

Definition at line 1555 of file rpmalloc.c.

References atomic_exchange_ptr_acquire(), atomic_store_ptr_release(), span_t::free_list, span_t::free_list_deferred, INVALID_POINTER, span_t::list_size, and span_t::used_count.

Referenced by _rpmalloc_allocate_from_heap_fallback().

◆ _rpmalloc_span_finalize()

static int _rpmalloc_span_finalize	(	heap_t *	heap,
		size_t	iclass,
		span_t *	span,
		span_t **	list_head
	)

static

Definition at line 1574 of file rpmalloc.c.

References _memory_span_mask, _rpmalloc_span_double_link_list_remove(), _rpmalloc_span_unmap(), _rpmalloc_stat_dec, block, span_t::free_list, heap_size_class_t::free_list, span_t::list_size, rpmalloc_assert, heap_t::size_class, and span_t::used_count.

Referenced by _rpmalloc_heap_finalize().

◆ _rpmalloc_span_initialize()

static void _rpmalloc_span_initialize	(	span_t *	span,
		size_t	total_span_count,
		size_t	span_count,
		size_t	align_offset
	)

static

Setup a newly mapped span.

Definition at line 1308 of file rpmalloc.c.

References span_t::align_offset, atomic_store32(), span_t::flags, span_t::remaining_spans, span_t::span_count, SPAN_FLAG_MASTER, and span_t::total_spans.

Referenced by _rpmalloc_heap_allocate_new(), and _rpmalloc_span_map_aligned_count().

◆ _rpmalloc_span_initialize_new()

static void * _rpmalloc_span_initialize_new	(	heap_t *	heap,
		heap_size_class_t *	heap_size_class,
		span_t *	span,
		uint32_t	class_idx
	)

static

Initialize an unused span (from cache or mapped) to be new active span, putting the initial free list in heap class free list.

Definition at line 1519 of file rpmalloc.c.

References _memory_size_class, _rpmalloc_span_double_link_list_add(), atomic_store_ptr_release(), block, span_t::block_count, size_class_t::block_count, span_t::block_size, size_class_t::block_size, span_t::flags, span_t::free_list, heap_size_class_t::free_list, span_t::free_list_deferred, span_t::free_list_limit, free_list_partial_init(), heap_t::full_span_count, span_t::heap, span_t::list_size, heap_size_class_t::partial_span, pointer_offset, rpmalloc_assert, span_t::size_class, span_t::span_count, SPAN_HEADER_SIZE, and span_t::used_count.

Referenced by _rpmalloc_allocate_from_heap_fallback().

◆ _rpmalloc_span_is_fully_utilized()

static int _rpmalloc_span_is_fully_utilized ( span_t * span )

static

Definition at line 1568 of file rpmalloc.c.

References span_t::block_count, span_t::free_list, span_t::free_list_limit, and rpmalloc_assert.

Referenced by _rpmalloc_allocate_from_heap_fallback(), and _rpmalloc_deallocate_direct_small_or_medium().

◆ _rpmalloc_span_map()

static span_t * _rpmalloc_span_map	(	heap_t *	heap,
		size_t	span_count
	)

static

Map in memory pages for the given number of spans (or use previously reserved pages)

Definition at line 1370 of file rpmalloc.c.

References _memory_global_lock, _memory_global_reserve_count, _memory_global_reserve_master, _memory_heap_reserve_count, _memory_page_size, _memory_span_map_count, _memory_span_size, _memory_span_size_shift, _rpmalloc_global_get_reserved_spans(), _rpmalloc_heap_set_reserved_spans(), _rpmalloc_span_map_aligned_count(), _rpmalloc_span_map_from_reserve(), _rpmalloc_spin(), atomic_cas32_acquire(), atomic_store32_release(), pointer_offset, span_t::span_count, and heap_t::spans_reserved.

Referenced by _rpmalloc_heap_extract_new_span(), _rpmalloc_heap_finalize(), and _rpmalloc_heap_reserved_extract().

◆ _rpmalloc_span_map_aligned_count()

static span_t * _rpmalloc_span_map_aligned_count	(	heap_t *	heap,
		size_t	span_count
	)

static

Map an aligned set of spans, taking configured mapping granularity and the page size into account.

Definition at line 1321 of file rpmalloc.c.

References _memory_global_lock, _memory_global_reserve, _memory_global_reserve_count, _memory_global_reserve_master, _memory_heap_reserve_count, _memory_span_size, _rpmalloc_global_set_reserved_spans(), _rpmalloc_heap_cache_insert(), _rpmalloc_heap_set_reserved_spans(), _rpmalloc_mmap(), _rpmalloc_span_align_count(), _rpmalloc_span_initialize(), _rpmalloc_span_mark_as_subspan_unless_master(), _rpmalloc_span_unmap(), _rpmalloc_stat_inc, LARGE_CLASS_COUNT, pointer_offset, rpmalloc_assert, heap_t::span_reserve, heap_t::span_reserve_master, and heap_t::spans_reserved.

Referenced by _rpmalloc_span_map().

◆ _rpmalloc_span_map_from_reserve()

static span_t * _rpmalloc_span_map_from_reserve	(	heap_t *	heap,
		size_t	span_count
	)

static

Use reserved spans to fulfill a memory map request (reserve size must be checked by caller)

Definition at line 1278 of file rpmalloc.c.

References _memory_span_size, _rpmalloc_span_mark_as_subspan_unless_master(), _rpmalloc_stat_inc, LARGE_CLASS_COUNT, pointer_offset, heap_t::span_reserve, heap_t::span_reserve_master, and heap_t::spans_reserved.

Referenced by _rpmalloc_span_map().

◆ _rpmalloc_span_mark_as_subspan_unless_master()

static void _rpmalloc_span_mark_as_subspan_unless_master	(	span_t *	master,
		span_t *	subspan,
		size_t	span_count
	)

static

Declare the span to be a subspan and store distance from master span and span count.

Definition at line 1261 of file rpmalloc.c.

References _memory_span_size_shift, span_t::align_offset, span_t::flags, span_t::offset_from_master, pointer_diff, rpmalloc_assert, span_t::span_count, SPAN_FLAG_MASTER, and SPAN_FLAG_SUBSPAN.

Referenced by _rpmalloc_global_get_reserved_spans(), _rpmalloc_span_map_aligned_count(), _rpmalloc_span_map_from_reserve(), and _rpmalloc_unmap_os().

◆ _rpmalloc_span_release_to_cache()

static void _rpmalloc_span_release_to_cache	(	heap_t *	heap,
		span_t *	span
	)

static

Move the span (used for small or medium allocations) to the heap thread cache.

Definition at line 1461 of file rpmalloc.c.

References _rpmalloc_heap_cache_insert(), _rpmalloc_span_unmap(), _rpmalloc_stat_dec, _rpmalloc_stat_inc, atomic_decr32(), heap_size_class_t::cache, heap_t::finalize, span_t::heap, rpmalloc_assert, span_t::size_class, heap_t::size_class, SIZE_CLASS_COUNT, and span_t::span_count.

Referenced by _rpmalloc_deallocate_direct_small_or_medium().

◆ _rpmalloc_span_unmap()

static void _rpmalloc_span_unmap ( span_t * span )

static

Unmap memory pages for the given number of spans (or mark as unused if no partial unmappings)

Definition at line 1409 of file rpmalloc.c.

References _memory_page_size, _memory_span_size, _rpmalloc_stat_add, _rpmalloc_stat_sub, _rpmalloc_unmap(), span_t::align_offset, atomic_add32(), span_t::flags, span_t::offset_from_master, pointer_offset, span_t::remaining_spans, rpmalloc_assert, span_t::span_count, SPAN_FLAG_MASTER, SPAN_FLAG_SUBSPAN, SPAN_FLAG_UNMAPPED_MASTER, and span_t::total_spans.

Referenced by _rpmalloc_heap_cache_insert(), _rpmalloc_heap_finalize(), _rpmalloc_heap_global_finalize(), _rpmalloc_heap_release(), _rpmalloc_heap_unmap(), _rpmalloc_span_finalize(), _rpmalloc_span_map_aligned_count(), and _rpmalloc_span_release_to_cache().

◆ _rpmalloc_spin()

static void _rpmalloc_spin ( void )

static

Definition at line 945 of file rpmalloc.c.

Referenced by _rpmalloc_heap_allocate(), _rpmalloc_heap_release(), _rpmalloc_span_map(), and rpmalloc_global_statistics().

◆ _rpmalloc_unmap()

static void _rpmalloc_unmap	(	void *	address,
		size_t	size,
		size_t	offset,
		size_t	release
	)

static

Unmap virtual memory.

Definition at line 1024 of file rpmalloc.c.

References _memory_config, _memory_page_size, _memory_page_size_shift, _rpmalloc_stat_add, _rpmalloc_stat_sub, rpmalloc_config_t::memory_unmap, and rpmalloc_assert.

Referenced by _rpmalloc_aligned_allocate(), _rpmalloc_deallocate_huge(), and _rpmalloc_span_unmap().

◆ _rpmalloc_unmap_os()

static void _rpmalloc_unmap_os	(	void *	address,
		size_t	size,
		size_t	offset,
		size_t	release
	)

static

Default implementation to unmap pages from virtual memory.

Definition at line 1129 of file rpmalloc.c.

References _memory_map_granularity, _memory_page_size, _memory_page_size_shift, _memory_span_size, _rpmalloc_span_mark_as_subspan_unless_master(), _rpmalloc_stat_sub, pointer_offset, and rpmalloc_assert.

Referenced by rpmalloc_initialize_config().

◆ _rpmalloc_usable_size()

static size_t _rpmalloc_usable_size ( void * p )

static

Reallocation entry points.

Initialization, finalization and utility.

Get the usable size of the given block

Definition at line 3002 of file rpmalloc.c.

References _memory_page_size, _memory_span_mask, _memory_span_size, span_t::block_size, pointer_diff, pointer_offset, span_t::size_class, SIZE_CLASS_COUNT, SIZE_CLASS_LARGE, span_t::span_count, and SPAN_HEADER_SIZE.

Referenced by _rpmalloc_aligned_reallocate(), and rpmalloc_usable_size().

◆ _Static_assert() [1/4]

_Static_assert	(	(SMALL_GRANULARITY &(SMALL_GRANULARITY - 1))	= `=0`,
		"Small granularity must be power of two"
	)

◆ _Static_assert() [2/4]

_Static_assert	(	(SPAN_HEADER_SIZE &(SPAN_HEADER_SIZE - 1))	= `=0`,
		"Span header size must be power of two"
	)

◆ _Static_assert() [3/4]

_Static_assert	(	sizeof(size_class_t)	= `=8`,
		"Size class size mismatch"
	)

◆ _Static_assert() [4/4]

_Static_assert	(	sizeof(span_t)<=	SPAN_HEADER_SIZE,
		"span size mismatch"
	)

◆ atomic_add32()

static FORCEINLINE int32_t atomic_add32	(	atomic32_t *	val,
		int32_t	add
	)

static

Definition at line 345 of file rpmalloc.c.

Referenced by _rpmalloc_span_unmap(), and rpmalloc_finalize().

◆ atomic_add64()

static FORCEINLINE int64_t atomic_add64	(	atomic64_t *	val,
		int64_t	add
	)

static

Definition at line 359 of file rpmalloc.c.

◆ atomic_cas32_acquire()

static FORCEINLINE int atomic_cas32_acquire	(	atomic32_t *	dst,
		int32_t	val,
		int32_t	ref
	)

static

Definition at line 348 of file rpmalloc.c.

Referenced by _rpmalloc_heap_allocate(), _rpmalloc_heap_release(), _rpmalloc_span_map(), and rpmalloc_global_statistics().

◆ atomic_cas_ptr()

static FORCEINLINE int atomic_cas_ptr	(	atomicptr_t *	dst,
		void *	val,
		void *	ref
	)

static

Definition at line 375 of file rpmalloc.c.

Referenced by _rpmalloc_deallocate_defer_free_span().

◆ atomic_decr32()

static FORCEINLINE int32_t atomic_decr32 ( atomic32_t * val )

static

Definition at line 342 of file rpmalloc.c.

Referenced by _rpmalloc_deallocate_large(), _rpmalloc_heap_release(), _rpmalloc_heap_unmap(), and _rpmalloc_span_release_to_cache().

◆ atomic_exchange_ptr_acquire()

static FORCEINLINE void * atomic_exchange_ptr_acquire	(	atomicptr_t *	dst,
		void *	val
	)

static

Definition at line 371 of file rpmalloc.c.

Referenced by _rpmalloc_deallocate_defer_small_or_medium(), _rpmalloc_deallocate_direct_small_or_medium(), _rpmalloc_heap_cache_adopt_deferred(), and _rpmalloc_span_extract_free_list_deferred().

◆ atomic_incr32()

static FORCEINLINE int32_t atomic_incr32 ( atomic32_t * val )

static

Definition at line 339 of file rpmalloc.c.

Referenced by _rpmalloc_heap_initialize(), and _rpmalloc_inc_span_statistics().

◆ atomic_load64()

static FORCEINLINE int64_t atomic_load64 ( atomic64_t * val )

static

Definition at line 356 of file rpmalloc.c.

Referenced by rpmalloc_dump_statistics(), and rpmalloc_thread_statistics().

◆ atomic_load_ptr()

static FORCEINLINE void * atomic_load_ptr ( atomicptr_t * src )

static

Definition at line 362 of file rpmalloc.c.

Referenced by _rpmalloc_allocate_from_heap_fallback(), _rpmalloc_deallocate_defer_free_span(), _rpmalloc_heap_finalize(), and rpmalloc_thread_statistics().

◆ atomic_store32()

static FORCEINLINE void atomic_store32	(	atomic32_t *	dst,
		int32_t	val
	)

static

Definition at line 336 of file rpmalloc.c.

Referenced by _rpmalloc_heap_allocate_new(), _rpmalloc_inc_span_statistics(), _rpmalloc_span_initialize(), and rpmalloc_initialize_config().

◆ atomic_store32_release()

static FORCEINLINE void atomic_store32_release	(	atomic32_t *	dst,
		int32_t	val
	)

static

Definition at line 353 of file rpmalloc.c.

Referenced by _rpmalloc_heap_allocate(), _rpmalloc_heap_release(), _rpmalloc_span_map(), rpmalloc_finalize(), rpmalloc_global_statistics(), and rpmalloc_initialize_config().

◆ atomic_store_ptr()

static FORCEINLINE void atomic_store_ptr	(	atomicptr_t *	dst,
		void *	val
	)

static

Definition at line 365 of file rpmalloc.c.

◆ atomic_store_ptr_release()

static FORCEINLINE void atomic_store_ptr_release	(	atomicptr_t *	dst,
		void *	val
	)

static

Definition at line 368 of file rpmalloc.c.

Referenced by _rpmalloc_deallocate_defer_small_or_medium(), _rpmalloc_deallocate_direct_small_or_medium(), _rpmalloc_span_extract_free_list_deferred(), and _rpmalloc_span_initialize_new().

◆ free_list_partial_init()

static uint32_t free_list_partial_init	(	void **	list,
		void **	first_block,
		void *	page_start,
		void *	block_start,
		uint32_t	block_count,
		uint32_t	block_size
	)

static

Initialize a (partial) free list up to next system memory page, while reserving the first block as allocated, returning number of blocks in list.

Definition at line 1484 of file rpmalloc.c.

References _memory_page_size, pointer_offset, and rpmalloc_assert.

Referenced by _rpmalloc_allocate_from_heap_fallback(), and _rpmalloc_span_initialize_new().

◆ free_list_pop()

static void * free_list_pop ( void ** list )

static

Allocation entry points.

Pop first block from a free list

Definition at line 2369 of file rpmalloc.c.

References block.

Referenced by _rpmalloc_allocate_from_heap_fallback(), _rpmalloc_allocate_medium(), and _rpmalloc_allocate_small().

◆ get_thread_heap()

static heap_t * get_thread_heap ( void )

inlinestatic

Get the current thread heap.

Definition at line 882 of file rpmalloc.c.

References EXPECTED, get_thread_heap_raw(), and rpmalloc_initialize().

Referenced by rpaligned_alloc(), rpaligned_realloc(), rpcalloc(), rpmalloc(), and rprealloc().

◆ get_thread_heap_raw()

static heap_t * get_thread_heap_raw ( void )

inlinestatic

Definition at line 873 of file rpmalloc.c.

Referenced by _rpmalloc_heap_release(), get_thread_heap(), rpmalloc_is_thread_initialized(), rpmalloc_thread_finalize(), rpmalloc_thread_initialize(), and rpmalloc_thread_statistics().

◆ get_thread_id()

static uintptr_t get_thread_id ( void )

inlinestatic

Fast thread ID.

Definition at line 895 of file rpmalloc.c.

Referenced by _rpmalloc_deallocate_direct_small_or_medium(), _rpmalloc_deallocate_huge(), _rpmalloc_deallocate_large(), _rpmalloc_deallocate_small_or_medium(), _rpmalloc_heap_release(), rpmalloc_set_main_thread(), and set_thread_heap().

◆ rpaligned_alloc()

RPMALLOC_ALLOCATOR void * rpaligned_alloc	(	size_t	alignment,
		size_t	size
	)

Allocate a memory block of at least the given size and alignment.

Definition at line 3438 of file rpmalloc.c.

References _rpmalloc_aligned_allocate(), and get_thread_heap().

Referenced by rpaligned_calloc(), rpmemalign(), and rpposix_memalign().

◆ rpaligned_calloc()

RPMALLOC_ALLOCATOR void * rpaligned_calloc	(	size_t	alignment,
		size_t	num,
		size_t	size
	)

inline

Definition at line 3444 of file rpmalloc.c.

References block, and rpaligned_alloc().

◆ rpaligned_realloc()

RPMALLOC_ALLOCATOR void * rpaligned_realloc	(	void *	ptr,
		size_t	alignment,
		size_t	size,
		size_t	oldsize,
		unsigned int	flags
	)

Reallocate the given block to at least the given size and alignment,.

Definition at line 3424 of file rpmalloc.c.

References _memory_page_size, _rpmalloc_aligned_reallocate(), and get_thread_heap().

◆ rpcalloc()

RPMALLOC_ALLOCATOR void * rpcalloc	(	size_t	num,
		size_t	size
	)

inline

Definition at line 3387 of file rpmalloc.c.

References _rpmalloc_allocate(), block, and get_thread_heap().

◆ rpfree()

void rpfree ( void * ptr )

inline

Free the given memory block.

Definition at line 3385 of file rpmalloc.c.

References _rpmalloc_deallocate().

◆ rpmalloc()

RPMALLOC_ALLOCATOR void * rpmalloc ( size_t size )

inline

Allocate a memory block of at least the given size.

Definition at line 3374 of file rpmalloc.c.

References _rpmalloc_allocate(), and get_thread_heap().

◆ rpmalloc_config()

const rpmalloc_config_t * rpmalloc_config ( void )

Get allocator configuration.

Definition at line 3370 of file rpmalloc.c.

References _memory_config.

◆ rpmalloc_dump_statistics()

void rpmalloc_dump_statistics ( void * file )

Dump all statistics in human readable format to file (should be a FILE*)

Definition at line 3694 of file rpmalloc.c.

References _memory_heaps, _memory_page_size, _memory_span_size, atomic_load64(), global_cache_t::count, file(), HEAP_ARRAY_SIZE, LARGE_CLASS_COUNT, span_t::next, heap_t::next_heap, global_cache_t::overflow, rpmalloc_assert, SIZE_CLASS_COUNT, and while().

◆ rpmalloc_finalize()

void rpmalloc_finalize ( void )

Finalize the allocator.

Finalize allocator.

Definition at line 3293 of file rpmalloc.c.

References _memory_global_lock, _memory_global_reserve, _memory_global_reserve_count, _memory_global_reserve_master, _memory_heaps, _rpmalloc_heap_global_finalize(), _rpmalloc_initialized, atomic_add32(), atomic_store32_release(), heap_t::finalize, HEAP_ARRAY_SIZE, LARGE_CLASS_COUNT, heap_t::next_heap, span_t::remaining_spans, rpmalloc_assert, and rpmalloc_thread_finalize().

◆ rpmalloc_global_statistics()

void rpmalloc_global_statistics ( rpmalloc_global_statistics_t * stats )

Get global statistics.

Definition at line 3572 of file rpmalloc.c.

References _memory_page_size, _memory_span_size, _rpmalloc_spin(), atomic_cas32_acquire(), atomic_store32_release(), global_cache_t::count, LARGE_CLASS_COUNT, global_cache_t::lock, span_t::next, and global_cache_t::overflow.

◆ rpmalloc_initialize()

int rpmalloc_initialize ( void )

inline

Initialize the allocator and setup global data.

Initialize allocator with default configuration.

Definition at line 3047 of file rpmalloc.c.

References _rpmalloc_initialized, rpmalloc_initialize_config(), and rpmalloc_thread_initialize().

Referenced by get_thread_heap().

◆ rpmalloc_initialize_config()

int rpmalloc_initialize_config ( const rpmalloc_config_t * config )

Initialize allocator with given configuration.

Definition at line 3055 of file rpmalloc.c.

References _memory_config, _memory_default_span_mask, _memory_default_span_size, _memory_default_span_size_shift, _memory_global_lock, _memory_heap_reserve_count, _memory_heaps, _memory_huge_pages, _memory_map_granularity, _memory_medium_size_limit, _memory_orphan_heaps, _memory_page_size, _memory_page_size_shift, _memory_size_class, _memory_span_map_count, _memory_span_mask, _memory_span_size, _memory_span_size_shift, _rpmalloc_adjust_size_class(), _rpmalloc_heap_release_raw_fc(), _rpmalloc_initialized, _rpmalloc_memset_const, _rpmalloc_mmap_os(), _rpmalloc_unmap_os(), atomic_store32(), atomic_store32_release(), size_class_t::block_size, DEFAULT_SPAN_MAP_COUNT, rpmalloc_config_t::enable_huge_pages, MEDIUM_CLASS_COUNT, MEDIUM_GRANULARITY, MEDIUM_SIZE_LIMIT, rpmalloc_config_t::memory_map, rpmalloc_config_t::memory_unmap, rpmalloc_config_t::page_size, rc, rpmalloc_assert, rpmalloc_linker_reference(), rpmalloc_thread_initialize(), SMALL_CLASS_COUNT, SMALL_GRANULARITY, SMALL_SIZE_LIMIT, SPAN_HEADER_SIZE, rpmalloc_config_t::span_map_count, and rpmalloc_config_t::span_size.

Referenced by rpmalloc_initialize().

◆ rpmalloc_is_thread_initialized()

int rpmalloc_is_thread_initialized ( void )

Query if allocator is initialized for calling thread.

Definition at line 3366 of file rpmalloc.c.

References get_thread_heap_raw().

◆ rpmalloc_linker_reference()

void rpmalloc_linker_reference ( void )

Dummy empty function for forcing linker symbol inclusion.

Definition at line 3992 of file rpmalloc.c.

References _rpmalloc_initialized.

Referenced by rpmalloc_initialize_config().

◆ rpmalloc_set_main_thread()

void rpmalloc_set_main_thread ( void )

Set main thread ID.

Definition at line 941 of file rpmalloc.c.

References _rpmalloc_main_thread_id, and get_thread_id().

◆ rpmalloc_thread_collect()

void rpmalloc_thread_collect ( void )

inline

Perform deferred deallocations pending for the calling thread heap.

Definition at line 3487 of file rpmalloc.c.

◆ rpmalloc_thread_finalize()

void rpmalloc_thread_finalize ( int release_caches )

Finalize thread, orphan heap.

Finalize allocator for calling thread.

Definition at line 3356 of file rpmalloc.c.

References _rpmalloc_heap_release_raw(), get_thread_heap_raw(), and set_thread_heap().

Referenced by rpmalloc_finalize().

◆ rpmalloc_thread_initialize()

void rpmalloc_thread_initialize ( void )

inline

Initialize thread, assign heap.

Initialize allocator for calling thread.

Definition at line 3342 of file rpmalloc.c.

References _rpmalloc_heap_allocate(), _rpmalloc_stat_inc, get_thread_heap_raw(), and set_thread_heap().

Referenced by rpmalloc_initialize(), and rpmalloc_initialize_config().

◆ rpmalloc_thread_statistics()

void rpmalloc_thread_statistics ( rpmalloc_thread_statistics_t * stats )

Get per-thread statistics.

Definition at line 3489 of file rpmalloc.c.

References _memory_size_class, _memory_span_size, atomic_load64(), atomic_load_ptr(), size_class_t::block_count, size_class_t::block_size, span_cache_t::count, span_t::free_list, span_t::free_list_limit, get_thread_heap_raw(), LARGE_CLASS_COUNT, span_t::list_size, span_t::next, heap_size_class_t::partial_span, span_t::size_class, heap_t::size_class, SIZE_CLASS_COUNT, SIZE_CLASS_HUGE, span_t::span_count, heap_t::span_free_deferred, and span_t::used_count.

◆ rpmalloc_usable_size()

size_t rpmalloc_usable_size ( void * ptr )

inline

Query the usable size of the given memory block (from given pointer to the end of block)

Definition at line 3483 of file rpmalloc.c.

References _rpmalloc_usable_size().

◆ rpmemalign()

RPMALLOC_ALLOCATOR void * rpmemalign	(	size_t	alignment,
		size_t	size
	)

inline

Allocate a memory block of at least the given size and alignment.

Definition at line 3469 of file rpmalloc.c.

References rpaligned_alloc().

◆ rpposix_memalign()

int rpposix_memalign	(	void **	memptr,
		size_t	alignment,
		size_t	size
	)

inline

Allocate a memory block of at least the given size and alignment.

Definition at line 3474 of file rpmalloc.c.

References rpaligned_alloc().

◆ rprealloc()

RPMALLOC_ALLOCATOR void * rprealloc	(	void *	ptr,
		size_t	size
	)

inline

Reallocate the given block to at least the given size.

Definition at line 3413 of file rpmalloc.c.

References _rpmalloc_reallocate(), and get_thread_heap().

◆ set_thread_heap()

static void set_thread_heap ( heap_t * heap )

static

Set the current thread heap.

Definition at line 927 of file rpmalloc.c.

References get_thread_id(), and heap_t::owner_thread.

Referenced by _rpmalloc_heap_release(), rpmalloc_thread_finalize(), and rpmalloc_thread_initialize().

Variable Documentation

◆ _memory_config

rpmalloc_config_t _memory_config

static

Configuration.

Definition at line 767 of file rpmalloc.c.

Referenced by _rpmalloc_mmap(), _rpmalloc_mmap_os(), _rpmalloc_set_name(), _rpmalloc_unmap(), rpmalloc_config(), and rpmalloc_initialize_config().

◆ _memory_global_lock

atomic32_t _memory_global_lock

static

Used to restrict access to mapping memory for huge pages.

Definition at line 812 of file rpmalloc.c.

Referenced by _rpmalloc_heap_allocate(), _rpmalloc_heap_release(), _rpmalloc_span_map(), _rpmalloc_span_map_aligned_count(), rpmalloc_finalize(), and rpmalloc_initialize_config().

◆ _memory_global_reserve

span_t* _memory_global_reserve

static

Global reserved spans.

Definition at line 804 of file rpmalloc.c.

Referenced by _rpmalloc_global_get_reserved_spans(), _rpmalloc_global_set_reserved_spans(), _rpmalloc_span_map_aligned_count(), and rpmalloc_finalize().

◆ _memory_global_reserve_count

size_t _memory_global_reserve_count

static

Global reserved count.

Definition at line 806 of file rpmalloc.c.

Referenced by _rpmalloc_global_get_reserved_spans(), _rpmalloc_global_set_reserved_spans(), _rpmalloc_heap_allocate_new(), _rpmalloc_span_map(), _rpmalloc_span_map_aligned_count(), and rpmalloc_finalize().

◆ _memory_global_reserve_master

span_t* _memory_global_reserve_master

static

Global reserved master.

Definition at line 808 of file rpmalloc.c.

Referenced by _rpmalloc_global_get_reserved_spans(), _rpmalloc_global_set_reserved_spans(), _rpmalloc_span_map(), _rpmalloc_span_map_aligned_count(), and rpmalloc_finalize().

◆ _memory_heap_id

atomic32_t _memory_heap_id

static

Heap ID counter.

Definition at line 796 of file rpmalloc.c.

Referenced by _rpmalloc_heap_initialize().

◆ _memory_heap_reserve_count

size_t _memory_heap_reserve_count

static

Number of spans to keep reserved in each heap.

Definition at line 790 of file rpmalloc.c.

Referenced by _rpmalloc_heap_allocate_new(), _rpmalloc_span_map(), _rpmalloc_span_map_aligned_count(), and rpmalloc_initialize_config().

◆ _memory_heaps

heap_t* _memory_heaps[HEAP_ARRAY_SIZE]

static

All heaps.

Definition at line 810 of file rpmalloc.c.

Referenced by _rpmalloc_heap_global_finalize(), _rpmalloc_heap_initialize(), rpmalloc_dump_statistics(), rpmalloc_finalize(), and rpmalloc_initialize_config().

◆ _memory_huge_pages

int _memory_huge_pages

static

Huge page support.

Definition at line 798 of file rpmalloc.c.

Referenced by _rpmalloc_mmap_os(), _rpmalloc_set_name(), and rpmalloc_initialize_config().

◆ _memory_map_granularity

size_t _memory_map_granularity

static

Granularity at which memory pages are mapped by OS.

Definition at line 773 of file rpmalloc.c.

Referenced by _rpmalloc_mmap_os(), _rpmalloc_unmap_os(), and rpmalloc_initialize_config().

◆ _memory_medium_size_limit

size_t _memory_medium_size_limit

static

Run-time size limit of medium blocks.

Definition at line 794 of file rpmalloc.c.

Referenced by _rpmalloc_aligned_allocate(), _rpmalloc_allocate(), and rpmalloc_initialize_config().

◆ _memory_orphan_heaps

heap_t* _memory_orphan_heaps

static

Orphaned heaps.

Definition at line 814 of file rpmalloc.c.

Referenced by _rpmalloc_heap_allocate(), _rpmalloc_heap_orphan(), and rpmalloc_initialize_config().

◆ _memory_page_size

size_t _memory_page_size

static

◆ _memory_page_size_shift

size_t _memory_page_size_shift

static

Shift to divide by page size.

Definition at line 771 of file rpmalloc.c.

Referenced by _rpmalloc_allocate_huge(), _rpmalloc_mmap(), _rpmalloc_mmap_os(), _rpmalloc_reallocate(), _rpmalloc_unmap(), _rpmalloc_unmap_os(), and rpmalloc_initialize_config().

◆ _memory_size_class

size_class_t _memory_size_class[SIZE_CLASS_COUNT]

static

Global size classes.

Definition at line 792 of file rpmalloc.c.

Referenced by _rpmalloc_adjust_size_class(), _rpmalloc_allocate_from_heap_fallback(), _rpmalloc_allocate_medium(), _rpmalloc_span_initialize_new(), rpmalloc_initialize_config(), and rpmalloc_thread_statistics().

◆ _memory_span_map_count

size_t _memory_span_map_count

static

Number of spans to map in each map call.

Definition at line 788 of file rpmalloc.c.

Referenced by _rpmalloc_span_align_count(), _rpmalloc_span_map(), and rpmalloc_initialize_config().

◆ _rpmalloc_initialized

int _rpmalloc_initialized

static

Initialized flag.

Definition at line 763 of file rpmalloc.c.

Referenced by rpmalloc_finalize(), rpmalloc_initialize(), rpmalloc_initialize_config(), and rpmalloc_linker_reference().

◆ _rpmalloc_main_thread_id

uintptr_t _rpmalloc_main_thread_id

static

Main thread ID.

Definition at line 765 of file rpmalloc.c.

Referenced by _rpmalloc_heap_release(), and rpmalloc_set_main_thread().

◆ TLS_MODEL

_Thread_local heap_t* _memory_thread_heap TLS_MODEL

static

Definition at line 870 of file rpmalloc.c.

Classes

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ __has_builtin

◆ _memory_default_span_mask

◆ _memory_default_span_size

◆ _memory_default_span_size_shift

◆ _memory_span_mask

◆ _memory_span_size

◆ _memory_span_size_shift

◆ _rpmalloc_memcpy_const

◆ _rpmalloc_memset_const

◆ _rpmalloc_stat_add

◆ _rpmalloc_stat_add64

◆ _rpmalloc_stat_add_peak

◆ _rpmalloc_stat_dec

◆ _rpmalloc_stat_inc

◆ _rpmalloc_stat_inc_alloc

◆ _rpmalloc_stat_inc_free

◆ _rpmalloc_stat_sub

◆ DEFAULT_SPAN_MAP_COUNT

◆ DISABLE_UNMAP

◆ ENABLE_ADAPTIVE_THREAD_CACHE [1/2]

◆ ENABLE_ADAPTIVE_THREAD_CACHE [2/2]

◆ ENABLE_ASSERTS

◆ ENABLE_GLOBAL_CACHE

◆ ENABLE_OVERRIDE

◆ ENABLE_PRELOAD

◆ ENABLE_STATISTICS

◆ ENABLE_THREAD_CACHE

◆ ENABLE_UNLIMITED_CACHE [1/2]

◆ ENABLE_UNLIMITED_CACHE [2/2]

◆ ENABLE_VALIDATE_ARGS

◆ EXPECTED

◆ FORCEINLINE

◆ GLOBAL_CACHE_MULTIPLIER

◆ HEAP_ARRAY_SIZE

◆ INVALID_POINTER

◆ LARGE_CLASS_COUNT

◆ LARGE_SIZE_LIMIT

◆ MAX_THREAD_SPAN_CACHE

◆ MAX_THREAD_SPAN_LARGE_CACHE

◆ MEDIUM_CLASS_COUNT

◆ MEDIUM_GRANULARITY

◆ MEDIUM_GRANULARITY_SHIFT

◆ MEDIUM_SIZE_LIMIT

◆ PLATFORM_POSIX

◆ PLATFORM_WINDOWS

◆ pointer_diff

◆ pointer_offset

◆ rpmalloc_assert

◆ rpmalloc_assume

◆ SIZE_CLASS_COUNT

◆ SIZE_CLASS_HUGE

◆ SIZE_CLASS_LARGE

◆ SMALL_CLASS_COUNT

◆ SMALL_GRANULARITY

◆ SMALL_GRANULARITY_SHIFT

◆ SMALL_SIZE_LIMIT

◆ SPAN_FLAG_ALIGNED_BLOCKS

◆ SPAN_FLAG_MASTER

◆ SPAN_FLAG_SUBSPAN

◆ SPAN_FLAG_UNMAPPED_MASTER

◆ SPAN_HEADER_SIZE

◆ THREAD_SPAN_CACHE_TRANSFER

◆ THREAD_SPAN_LARGE_CACHE_TRANSFER

◆ TLS_MODEL

◆ UNEXPECTED

Typedef Documentation

◆ _Atomic

◆ global_cache_t

◆ heap_size_class_t

◆ heap_t

◆ size_class_t

◆ span_active_t

◆ span_cache_t

◆ span_large_cache_t