Bug Summary

File:build/source/openmp/runtime/src/kmp_alloc.cpp
Warning:line 2134, column 15
Although the value stored to 'idx' is used in the enclosing expression, the value is never actually read from 'idx'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name kmp_alloc.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D omp_EXPORTS -I projects/openmp/runtime/src -I /build/source/openmp/runtime/src -I include -I /build/source/llvm/include -I /build/source/openmp/runtime/src/i18n -I /build/source/openmp/runtime/src/include -I /build/source/openmp/runtime/src/thirdparty/ittnotify -D _FORTIFY_SOURCE=2 -D NDEBUG -D _GNU_SOURCE -D _REENTRANT -D _FORTIFY_SOURCE=2 -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1668078801 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -Wno-enum-constexpr-conversion -Wno-extra -Wno-pedantic -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-covered-switch-default -Wno-frame-address -Wno-strict-aliasing -Wno-stringop-truncation -Wno-switch -Wno-uninitialized -Wno-return-type-c-linkage -Wno-cast-qual -Wno-int-to-void-pointer-cast -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-11-10-135928-647445-1 -x c++ /build/source/openmp/runtime/src/kmp_alloc.cpp
1/*
2 * kmp_alloc.cpp -- private/shared dynamic memory allocation and management
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//
12
13#include "kmp.h"
14#include "kmp_io.h"
15#include "kmp_wrapper_malloc.h"
16
17// Disable bget when it is not used
18#if KMP_USE_BGET1
19
20/* Thread private buffer management code */
21
22typedef int (*bget_compact_t)(size_t, int);
23typedef void *(*bget_acquire_t)(size_t);
24typedef void (*bget_release_t)(void *);
25
26/* NOTE: bufsize must be a signed datatype */
27
28#if KMP_OS_WINDOWS0
29#if KMP_ARCH_X860 || KMP_ARCH_ARM
30typedef kmp_int32 bufsize;
31#else
32typedef kmp_int64 bufsize;
33#endif
34#else
35typedef ssize_t bufsize;
36#endif // KMP_OS_WINDOWS
37
38/* The three modes of operation are, fifo search, lifo search, and best-fit */
39
40typedef enum bget_mode {
41 bget_mode_fifo = 0,
42 bget_mode_lifo = 1,
43 bget_mode_best = 2
44} bget_mode_t;
45
46static void bpool(kmp_info_t *th, void *buffer, bufsize len);
47static void *bget(kmp_info_t *th, bufsize size);
48static void *bgetz(kmp_info_t *th, bufsize size);
49static void *bgetr(kmp_info_t *th, void *buffer, bufsize newsize);
50static void brel(kmp_info_t *th, void *buf);
51static void bectl(kmp_info_t *th, bget_compact_t compact,
52 bget_acquire_t acquire, bget_release_t release,
53 bufsize pool_incr);
54
55/* BGET CONFIGURATION */
56/* Buffer allocation size quantum: all buffers allocated are a
57 multiple of this size. This MUST be a power of two. */
58
59/* On IA-32 architecture with Linux* OS, malloc() does not
60 ensure 16 byte alignment */
61
62#if KMP_ARCH_X860 || !KMP_HAVE_QUAD0
63
64#define SizeQuant8 8
65#define AlignTypedouble double
66
67#else
68
69#define SizeQuant8 16
70#define AlignTypedouble _Quad
71
72#endif
73
74// Define this symbol to enable the bstats() function which calculates the
75// total free space in the buffer pool, the largest available buffer, and the
76// total space currently allocated.
77#define BufStats1 1
78
79#ifdef KMP_DEBUG1
80
81// Define this symbol to enable the bpoold() function which dumps the buffers
82// in a buffer pool.
83#define BufDump1 1
84
85// Define this symbol to enable the bpoolv() function for validating a buffer
86// pool.
87#define BufValid1 1
88
89// Define this symbol to enable the bufdump() function which allows dumping the
90// contents of an allocated or free buffer.
91#define DumpData1 1
92
93#ifdef NOT_USED_NOW
94
95// Wipe free buffers to a guaranteed pattern of garbage to trip up miscreants
96// who attempt to use pointers into released buffers.
97#define FreeWipe 1
98
99// Use a best fit algorithm when searching for space for an allocation request.
100// This uses memory more efficiently, but allocation will be much slower.
101#define BestFit 1
102
103#endif /* NOT_USED_NOW */
104#endif /* KMP_DEBUG */
105
106static bufsize bget_bin_size[] = {
107 0,
108 // 1 << 6, /* .5 Cache line */
109 1 << 7, /* 1 Cache line, new */
110 1 << 8, /* 2 Cache lines */
111 1 << 9, /* 4 Cache lines, new */
112 1 << 10, /* 8 Cache lines */
113 1 << 11, /* 16 Cache lines, new */
114 1 << 12, 1 << 13, /* new */
115 1 << 14, 1 << 15, /* new */
116 1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20, /* 1MB */
117 1 << 21, /* 2MB */
118 1 << 22, /* 4MB */
119 1 << 23, /* 8MB */
120 1 << 24, /* 16MB */
121 1 << 25, /* 32MB */
122};
123
124#define MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize)) (int)(sizeof(bget_bin_size) / sizeof(bufsize))
125
126struct bfhead;
127
128// Declare the interface, including the requested buffer size type, bufsize.
129
130/* Queue links */
131typedef struct qlinks {
132 struct bfhead *flink; /* Forward link */
133 struct bfhead *blink; /* Backward link */
134} qlinks_t;
135
136/* Header in allocated and free buffers */
137typedef struct bhead2 {
138 kmp_info_t *bthr; /* The thread which owns the buffer pool */
139 bufsize prevfree; /* Relative link back to previous free buffer in memory or
140 0 if previous buffer is allocated. */
141 bufsize bsize; /* Buffer size: positive if free, negative if allocated. */
142} bhead2_t;
143
144/* Make sure the bhead structure is a multiple of SizeQuant in size. */
145typedef union bhead {
146 KMP_ALIGN(SizeQuant)__attribute__((aligned(8)))
147 AlignTypedouble b_align;
148 char b_pad[sizeof(bhead2_t) + (SizeQuant8 - (sizeof(bhead2_t) % SizeQuant8))];
149 bhead2_t bb;
150} bhead_t;
151#define BH(p)((bhead_t *)(p)) ((bhead_t *)(p))
152
153/* Header in directly allocated buffers (by acqfcn) */
154typedef struct bdhead {
155 bufsize tsize; /* Total size, including overhead */
156 bhead_t bh; /* Common header */
157} bdhead_t;
158#define BDH(p)((bdhead_t *)(p)) ((bdhead_t *)(p))
159
160/* Header in free buffers */
161typedef struct bfhead {
162 bhead_t bh; /* Common allocated/free header */
163 qlinks_t ql; /* Links on free list */
164} bfhead_t;
165#define BFH(p)((bfhead_t *)(p)) ((bfhead_t *)(p))
166
167typedef struct thr_data {
168 bfhead_t freelist[MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize))];
169#if BufStats1
170 size_t totalloc; /* Total space currently allocated */
171 long numget, numrel; /* Number of bget() and brel() calls */
172 long numpblk; /* Number of pool blocks */
173 long numpget, numprel; /* Number of block gets and rels */
174 long numdget, numdrel; /* Number of direct gets and rels */
175#endif /* BufStats */
176
177 /* Automatic expansion block management functions */
178 bget_compact_t compfcn;
179 bget_acquire_t acqfcn;
180 bget_release_t relfcn;
181
182 bget_mode_t mode; /* what allocation mode to use? */
183
184 bufsize exp_incr; /* Expansion block size */
185 bufsize pool_len; /* 0: no bpool calls have been made
186 -1: not all pool blocks are the same size
187 >0: (common) block size for all bpool calls made so far
188 */
189 bfhead_t *last_pool; /* Last pool owned by this thread (delay deallocation) */
190} thr_data_t;
191
192/* Minimum allocation quantum: */
193#define QLSize(sizeof(qlinks_t)) (sizeof(qlinks_t))
194#define SizeQ((8 > (sizeof(qlinks_t))) ? 8 : (sizeof(qlinks_t))) ((SizeQuant8 > QLSize(sizeof(qlinks_t))) ? SizeQuant8 : QLSize(sizeof(qlinks_t)))
195#define MaxSize(bufsize)( ~(((bufsize)(1) << (sizeof(bufsize) * 8 - 1)
) | (8 - 1)))
\
196 (bufsize)( \
197 ~(((bufsize)(1) << (sizeof(bufsize) * CHAR_BIT8 - 1)) | (SizeQuant8 - 1)))
198// Maximum for the requested size.
199
200/* End sentinel: value placed in bsize field of dummy block delimiting
201 end of pool block. The most negative number which will fit in a
202 bufsize, defined in a way that the compiler will accept. */
203
204#define ESent((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8
- 2)) - 1) * 2) - 2))
\
205 ((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2))
206
207/* Thread Data management routines */
208static int bget_get_bin(bufsize size) {
209 // binary chop bins
210 int lo = 0, hi = MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize)) - 1;
211
212 KMP_DEBUG_ASSERT(size > 0)if (!(size > 0)) { __kmp_debug_assert("size > 0", "openmp/runtime/src/kmp_alloc.cpp"
, 212); }
;
213
214 while ((hi - lo) > 1) {
215 int mid = (lo + hi) >> 1;
216 if (size < bget_bin_size[mid])
217 hi = mid - 1;
218 else
219 lo = mid;
220 }
221
222 KMP_DEBUG_ASSERT((lo >= 0) && (lo < MAX_BGET_BINS))if (!((lo >= 0) && (lo < (int)(sizeof(bget_bin_size
) / sizeof(bufsize))))) { __kmp_debug_assert("(lo >= 0) && (lo < (int)(sizeof(bget_bin_size) / sizeof(bufsize)))"
, "openmp/runtime/src/kmp_alloc.cpp", 222); }
;
223
224 return lo;
225}
226
227static void set_thr_data(kmp_info_t *th) {
228 int i;
229 thr_data_t *data;
230
231 data = (thr_data_t *)((!th->th.th_local.bget_data)
232 ? __kmp_allocate(sizeof(*data))___kmp_allocate((sizeof(*data)), "openmp/runtime/src/kmp_alloc.cpp"
, 232)
233 : th->th.th_local.bget_data);
234
235 memset(data, '\0', sizeof(*data));
236
237 for (i = 0; i < MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize)); ++i) {
238 data->freelist[i].ql.flink = &data->freelist[i];
239 data->freelist[i].ql.blink = &data->freelist[i];
240 }
241
242 th->th.th_local.bget_data = data;
243 th->th.th_local.bget_list = 0;
244#if !USE_CMP_XCHG_FOR_BGET1
245#ifdef USE_QUEUING_LOCK_FOR_BGET
246 __kmp_init_lock(&th->th.th_local.bget_lock);
247#else
248 __kmp_init_bootstrap_lock(&th->th.th_local.bget_lock);
249#endif /* USE_LOCK_FOR_BGET */
250#endif /* ! USE_CMP_XCHG_FOR_BGET */
251}
252
253static thr_data_t *get_thr_data(kmp_info_t *th) {
254 thr_data_t *data;
255
256 data = (thr_data_t *)th->th.th_local.bget_data;
257
258 KMP_DEBUG_ASSERT(data != 0)if (!(data != 0)) { __kmp_debug_assert("data != 0", "openmp/runtime/src/kmp_alloc.cpp"
, 258); }
;
259
260 return data;
261}
262
263/* Walk the free list and release the enqueued buffers */
264static void __kmp_bget_dequeue(kmp_info_t *th) {
265 void *p = TCR_SYNC_PTR(th->th.th_local.bget_list)((void *)(th->th.th_local.bget_list));
266
267 if (p != 0) {
268#if USE_CMP_XCHG_FOR_BGET1
269 {
270 volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list)((void *)(th->th.th_local.bget_list));
271 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list,__sync_bool_compare_and_swap((void *volatile *)(&th->th
.th_local.bget_list), (void *)(const_cast<void *>(old_value
)), (void *)(nullptr))
272 CCAST(void *, old_value), nullptr)__sync_bool_compare_and_swap((void *volatile *)(&th->th
.th_local.bget_list), (void *)(const_cast<void *>(old_value
)), (void *)(nullptr))
) {
273 KMP_CPU_PAUSE()__kmp_x86_pause();
274 old_value = TCR_SYNC_PTR(th->th.th_local.bget_list)((void *)(th->th.th_local.bget_list));
275 }
276 p = CCAST(void *, old_value)const_cast<void *>(old_value);
277 }
278#else /* ! USE_CMP_XCHG_FOR_BGET */
279#ifdef USE_QUEUING_LOCK_FOR_BGET
280 __kmp_acquire_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th));
281#else
282 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock);
283#endif /* USE_QUEUING_LOCK_FOR_BGET */
284
285 p = (void *)th->th.th_local.bget_list;
286 th->th.th_local.bget_list = 0;
287
288#ifdef USE_QUEUING_LOCK_FOR_BGET
289 __kmp_release_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th));
290#else
291 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock);
292#endif
293#endif /* USE_CMP_XCHG_FOR_BGET */
294
295 /* Check again to make sure the list is not empty */
296 while (p != 0) {
297 void *buf = p;
298 bfhead_t *b = BFH(((char *)p) - sizeof(bhead_t))((bfhead_t *)(((char *)p) - sizeof(bhead_t)));
299
300 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0)if (!(b->bh.bb.bsize != 0)) { __kmp_debug_assert("b->bh.bb.bsize != 0"
, "openmp/runtime/src/kmp_alloc.cpp", 300); }
;
301 KMP_DEBUG_ASSERT(((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1) ==if (!(((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) ==
(kmp_uintptr_t)th)) { __kmp_debug_assert("((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) == (kmp_uintptr_t)th"
, "openmp/runtime/src/kmp_alloc.cpp", 302); }
302 (kmp_uintptr_t)th)if (!(((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) ==
(kmp_uintptr_t)th)) { __kmp_debug_assert("((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) == (kmp_uintptr_t)th"
, "openmp/runtime/src/kmp_alloc.cpp", 302); }
; // clear possible mark
303 KMP_DEBUG_ASSERT(b->ql.blink == 0)if (!(b->ql.blink == 0)) { __kmp_debug_assert("b->ql.blink == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 303); }
;
304
305 p = (void *)b->ql.flink;
306
307 brel(th, buf);
308 }
309 }
310}
311
312/* Chain together the free buffers by using the thread owner field */
313static void __kmp_bget_enqueue(kmp_info_t *th, void *buf
314#ifdef USE_QUEUING_LOCK_FOR_BGET
315 ,
316 kmp_int32 rel_gtid
317#endif
318) {
319 bfhead_t *b = BFH(((char *)buf) - sizeof(bhead_t))((bfhead_t *)(((char *)buf) - sizeof(bhead_t)));
320
321 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0)if (!(b->bh.bb.bsize != 0)) { __kmp_debug_assert("b->bh.bb.bsize != 0"
, "openmp/runtime/src/kmp_alloc.cpp", 321); }
;
322 KMP_DEBUG_ASSERT(((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1) ==if (!(((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) ==
(kmp_uintptr_t)th)) { __kmp_debug_assert("((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) == (kmp_uintptr_t)th"
, "openmp/runtime/src/kmp_alloc.cpp", 323); }
323 (kmp_uintptr_t)th)if (!(((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) ==
(kmp_uintptr_t)th)) { __kmp_debug_assert("((kmp_uintptr_t)((void *)(b->bh.bb.bthr)) & ~1) == (kmp_uintptr_t)th"
, "openmp/runtime/src/kmp_alloc.cpp", 323); }
; // clear possible mark
324
325 b->ql.blink = 0;
326
327 KC_TRACE(10, ("__kmp_bget_enqueue: moving buffer to T#%d list\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_bget_enqueue: moving buffer to T#%d list\n"
, __kmp_gtid_from_thread(th)); }
328 __kmp_gtid_from_thread(th)))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_bget_enqueue: moving buffer to T#%d list\n"
, __kmp_gtid_from_thread(th)); }
;
329
330#if USE_CMP_XCHG_FOR_BGET1
331 {
332 volatile void *old_value = TCR_PTR(th->th.th_local.bget_list)((void *)(th->th.th_local.bget_list));
333 /* the next pointer must be set before setting bget_list to buf to avoid
334 exposing a broken list to other threads, even for an instant. */
335 b->ql.flink = BFH(CCAST(void *, old_value))((bfhead_t *)(const_cast<void *>(old_value)));
336
337 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list,__sync_bool_compare_and_swap((void *volatile *)(&th->th
.th_local.bget_list), (void *)(const_cast<void *>(old_value
)), (void *)(buf))
338 CCAST(void *, old_value), buf)__sync_bool_compare_and_swap((void *volatile *)(&th->th
.th_local.bget_list), (void *)(const_cast<void *>(old_value
)), (void *)(buf))
) {
339 KMP_CPU_PAUSE()__kmp_x86_pause();
340 old_value = TCR_PTR(th->th.th_local.bget_list)((void *)(th->th.th_local.bget_list));
341 /* the next pointer must be set before setting bget_list to buf to avoid
342 exposing a broken list to other threads, even for an instant. */
343 b->ql.flink = BFH(CCAST(void *, old_value))((bfhead_t *)(const_cast<void *>(old_value)));
344 }
345 }
346#else /* ! USE_CMP_XCHG_FOR_BGET */
347#ifdef USE_QUEUING_LOCK_FOR_BGET
348 __kmp_acquire_lock(&th->th.th_local.bget_lock, rel_gtid);
349#else
350 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock);
351#endif
352
353 b->ql.flink = BFH(th->th.th_local.bget_list)((bfhead_t *)(th->th.th_local.bget_list));
354 th->th.th_local.bget_list = (void *)buf;
355
356#ifdef USE_QUEUING_LOCK_FOR_BGET
357 __kmp_release_lock(&th->th.th_local.bget_lock, rel_gtid);
358#else
359 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock);
360#endif
361#endif /* USE_CMP_XCHG_FOR_BGET */
362}
363
364/* insert buffer back onto a new freelist */
365static void __kmp_bget_insert_into_freelist(thr_data_t *thr, bfhead_t *b) {
366 int bin;
367
368 KMP_DEBUG_ASSERT(((size_t)b) % SizeQuant == 0)if (!(((size_t)b) % 8 == 0)) { __kmp_debug_assert("((size_t)b) % 8 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 368); }
;
369 KMP_DEBUG_ASSERT(b->bh.bb.bsize % SizeQuant == 0)if (!(b->bh.bb.bsize % 8 == 0)) { __kmp_debug_assert("b->bh.bb.bsize % 8 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 369); }
;
370
371 bin = bget_get_bin(b->bh.bb.bsize);
372
373 KMP_DEBUG_ASSERT(thr->freelist[bin].ql.blink->ql.flink ==if (!(thr->freelist[bin].ql.blink->ql.flink == &thr
->freelist[bin])) { __kmp_debug_assert("thr->freelist[bin].ql.blink->ql.flink == &thr->freelist[bin]"
, "openmp/runtime/src/kmp_alloc.cpp", 374); }
374 &thr->freelist[bin])if (!(thr->freelist[bin].ql.blink->ql.flink == &thr
->freelist[bin])) { __kmp_debug_assert("thr->freelist[bin].ql.blink->ql.flink == &thr->freelist[bin]"
, "openmp/runtime/src/kmp_alloc.cpp", 374); }
;
375 KMP_DEBUG_ASSERT(thr->freelist[bin].ql.flink->ql.blink ==if (!(thr->freelist[bin].ql.flink->ql.blink == &thr
->freelist[bin])) { __kmp_debug_assert("thr->freelist[bin].ql.flink->ql.blink == &thr->freelist[bin]"
, "openmp/runtime/src/kmp_alloc.cpp", 376); }
376 &thr->freelist[bin])if (!(thr->freelist[bin].ql.flink->ql.blink == &thr
->freelist[bin])) { __kmp_debug_assert("thr->freelist[bin].ql.flink->ql.blink == &thr->freelist[bin]"
, "openmp/runtime/src/kmp_alloc.cpp", 376); }
;
377
378 b->ql.flink = &thr->freelist[bin];
379 b->ql.blink = thr->freelist[bin].ql.blink;
380
381 thr->freelist[bin].ql.blink = b;
382 b->ql.blink->ql.flink = b;
383}
384
385/* unlink the buffer from the old freelist */
386static void __kmp_bget_remove_from_freelist(bfhead_t *b) {
387 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b)if (!(b->ql.blink->ql.flink == b)) { __kmp_debug_assert
("b->ql.blink->ql.flink == b", "openmp/runtime/src/kmp_alloc.cpp"
, 387); }
;
388 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b)if (!(b->ql.flink->ql.blink == b)) { __kmp_debug_assert
("b->ql.flink->ql.blink == b", "openmp/runtime/src/kmp_alloc.cpp"
, 388); }
;
389
390 b->ql.blink->ql.flink = b->ql.flink;
391 b->ql.flink->ql.blink = b->ql.blink;
392}
393
394/* GET STATS -- check info on free list */
395static void bcheck(kmp_info_t *th, bufsize *max_free, bufsize *total_free) {
396 thr_data_t *thr = get_thr_data(th);
397 int bin;
398
399 *total_free = *max_free = 0;
400
401 for (bin = 0; bin < MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize)); ++bin) {
402 bfhead_t *b, *best;
403
404 best = &thr->freelist[bin];
405 b = best->ql.flink;
406
407 while (b != &thr->freelist[bin]) {
408 *total_free += (b->bh.bb.bsize - sizeof(bhead_t));
409 if ((best == &thr->freelist[bin]) || (b->bh.bb.bsize < best->bh.bb.bsize))
410 best = b;
411
412 /* Link to next buffer */
413 b = b->ql.flink;
414 }
415
416 if (*max_free < best->bh.bb.bsize)
417 *max_free = best->bh.bb.bsize;
418 }
419
420 if (*max_free > (bufsize)sizeof(bhead_t))
421 *max_free -= sizeof(bhead_t);
422}
423
424/* BGET -- Allocate a buffer. */
425static void *bget(kmp_info_t *th, bufsize requested_size) {
426 thr_data_t *thr = get_thr_data(th);
427 bufsize size = requested_size;
428 bfhead_t *b;
429 void *buf;
430 int compactseq = 0;
431 int use_blink = 0;
432 /* For BestFit */
433 bfhead_t *best;
434
435 if (size < 0 || size + sizeof(bhead_t) > MaxSize(bufsize)( ~(((bufsize)(1) << (sizeof(bufsize) * 8 - 1)
) | (8 - 1)))
) {
436 return NULL__null;
437 }
438
439 __kmp_bget_dequeue(th); /* Release any queued buffers */
440
441 if (size < (bufsize)SizeQ((8 > (sizeof(qlinks_t))) ? 8 : (sizeof(qlinks_t)))) { // Need at least room for the queue links.
442 size = SizeQ((8 > (sizeof(qlinks_t))) ? 8 : (sizeof(qlinks_t)));
443 }
444#if defined(SizeQuant8) && (SizeQuant8 > 1)
445 size = (size + (SizeQuant8 - 1)) & (~(SizeQuant8 - 1));
446#endif
447
448 size += sizeof(bhead_t); // Add overhead in allocated buffer to size required.
449 KMP_DEBUG_ASSERT(size >= 0)if (!(size >= 0)) { __kmp_debug_assert("size >= 0", "openmp/runtime/src/kmp_alloc.cpp"
, 449); }
;
450 KMP_DEBUG_ASSERT(size % SizeQuant == 0)if (!(size % 8 == 0)) { __kmp_debug_assert("size % 8 == 0", "openmp/runtime/src/kmp_alloc.cpp"
, 450); }
;
451
452 use_blink = (thr->mode == bget_mode_lifo);
453
454 /* If a compact function was provided in the call to bectl(), wrap
455 a loop around the allocation process to allow compaction to
456 intervene in case we don't find a suitable buffer in the chain. */
457
458 for (;;) {
459 int bin;
460
461 for (bin = bget_get_bin(size); bin < MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize)); ++bin) {
462 /* Link to next buffer */
463 b = (use_blink ? thr->freelist[bin].ql.blink
464 : thr->freelist[bin].ql.flink);
465
466 if (thr->mode == bget_mode_best) {
467 best = &thr->freelist[bin];
468
469 /* Scan the free list searching for the first buffer big enough
470 to hold the requested size buffer. */
471 while (b != &thr->freelist[bin]) {
472 if (b->bh.bb.bsize >= (bufsize)size) {
473 if ((best == &thr->freelist[bin]) ||
474 (b->bh.bb.bsize < best->bh.bb.bsize)) {
475 best = b;
476 }
477 }
478
479 /* Link to next buffer */
480 b = (use_blink ? b->ql.blink : b->ql.flink);
481 }
482 b = best;
483 }
484
485 while (b != &thr->freelist[bin]) {
486 if ((bufsize)b->bh.bb.bsize >= (bufsize)size) {
487
488 // Buffer is big enough to satisfy the request. Allocate it to the
489 // caller. We must decide whether the buffer is large enough to split
490 // into the part given to the caller and a free buffer that remains
491 // on the free list, or whether the entire buffer should be removed
492 // from the free list and given to the caller in its entirety. We
493 // only split the buffer if enough room remains for a header plus the
494 // minimum quantum of allocation.
495 if ((b->bh.bb.bsize - (bufsize)size) >
496 (bufsize)(SizeQ((8 > (sizeof(qlinks_t))) ? 8 : (sizeof(qlinks_t))) + (sizeof(bhead_t)))) {
497 bhead_t *ba, *bn;
498
499 ba = BH(((char *)b) + (b->bh.bb.bsize - (bufsize)size))((bhead_t *)(((char *)b) + (b->bh.bb.bsize - (bufsize)size
)))
;
500 bn = BH(((char *)ba) + size)((bhead_t *)(((char *)ba) + size));
501
502 KMP_DEBUG_ASSERT(bn->bb.prevfree == b->bh.bb.bsize)if (!(bn->bb.prevfree == b->bh.bb.bsize)) { __kmp_debug_assert
("bn->bb.prevfree == b->bh.bb.bsize", "openmp/runtime/src/kmp_alloc.cpp"
, 502); }
;
503
504 /* Subtract size from length of free block. */
505 b->bh.bb.bsize -= (bufsize)size;
506
507 /* Link allocated buffer to the previous free buffer. */
508 ba->bb.prevfree = b->bh.bb.bsize;
509
510 /* Plug negative size into user buffer. */
511 ba->bb.bsize = -size;
512
513 /* Mark this buffer as owned by this thread. */
514 TCW_PTR(ba->bb.bthr,((ba->bb.bthr)) = ((th))
515 th)((ba->bb.bthr)) = ((th)); // not an allocated address (do not mark it)
516 /* Mark buffer after this one not preceded by free block. */
517 bn->bb.prevfree = 0;
518
519 // unlink buffer from old freelist, and reinsert into new freelist
520 __kmp_bget_remove_from_freelist(b);
521 __kmp_bget_insert_into_freelist(thr, b);
522#if BufStats1
523 thr->totalloc += (size_t)size;
524 thr->numget++; /* Increment number of bget() calls */
525#endif
526 buf = (void *)((((char *)ba) + sizeof(bhead_t)));
527 KMP_DEBUG_ASSERT(((size_t)buf) % SizeQuant == 0)if (!(((size_t)buf) % 8 == 0)) { __kmp_debug_assert("((size_t)buf) % 8 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 527); }
;
528 return buf;
529 } else {
530 bhead_t *ba;
531
532 ba = BH(((char *)b) + b->bh.bb.bsize)((bhead_t *)(((char *)b) + b->bh.bb.bsize));
533
534 KMP_DEBUG_ASSERT(ba->bb.prevfree == b->bh.bb.bsize)if (!(ba->bb.prevfree == b->bh.bb.bsize)) { __kmp_debug_assert
("ba->bb.prevfree == b->bh.bb.bsize", "openmp/runtime/src/kmp_alloc.cpp"
, 534); }
;
535
536 /* The buffer isn't big enough to split. Give the whole
537 shebang to the caller and remove it from the free list. */
538
539 __kmp_bget_remove_from_freelist(b);
540#if BufStats1
541 thr->totalloc += (size_t)b->bh.bb.bsize;
542 thr->numget++; /* Increment number of bget() calls */
543#endif
544 /* Negate size to mark buffer allocated. */
545 b->bh.bb.bsize = -(b->bh.bb.bsize);
546
547 /* Mark this buffer as owned by this thread. */
548 TCW_PTR(ba->bb.bthr, th)((ba->bb.bthr)) = ((th)); // not an allocated address (do not mark)
549 /* Zero the back pointer in the next buffer in memory
550 to indicate that this buffer is allocated. */
551 ba->bb.prevfree = 0;
552
553 /* Give user buffer starting at queue links. */
554 buf = (void *)&(b->ql);
555 KMP_DEBUG_ASSERT(((size_t)buf) % SizeQuant == 0)if (!(((size_t)buf) % 8 == 0)) { __kmp_debug_assert("((size_t)buf) % 8 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 555); }
;
556 return buf;
557 }
558 }
559
560 /* Link to next buffer */
561 b = (use_blink ? b->ql.blink : b->ql.flink);
562 }
563 }
564
565 /* We failed to find a buffer. If there's a compact function defined,
566 notify it of the size requested. If it returns TRUE, try the allocation
567 again. */
568
569 if ((thr->compfcn == 0) || (!(*thr->compfcn)(size, ++compactseq))) {
570 break;
571 }
572 }
573
574 /* No buffer available with requested size free. */
575
576 /* Don't give up yet -- look in the reserve supply. */
577 if (thr->acqfcn != 0) {
578 if (size > (bufsize)(thr->exp_incr - sizeof(bhead_t))) {
579 /* Request is too large to fit in a single expansion block.
580 Try to satisfy it by a direct buffer acquisition. */
581 bdhead_t *bdh;
582
583 size += sizeof(bdhead_t) - sizeof(bhead_t);
584
585 KE_TRACE(10, ("%%%%%% MALLOC( %d )\n", (int)size))if (kmp_e_debug >= 10) { __kmp_debug_printf ("%%%%%% MALLOC( %d )\n"
, (int)size); }
;
586
587 /* richryan */
588 bdh = BDH((*thr->acqfcn)((bufsize)size))((bdhead_t *)((*thr->acqfcn)((bufsize)size)));
589 if (bdh != NULL__null) {
590
591 // Mark the buffer special by setting size field of its header to zero.
592 bdh->bh.bb.bsize = 0;
593
594 /* Mark this buffer as owned by this thread. */
595 TCW_PTR(bdh->bh.bb.bthr, th)((bdh->bh.bb.bthr)) = ((th)); // don't mark buffer as allocated,
596 // because direct buffer never goes to free list
597 bdh->bh.bb.prevfree = 0;
598 bdh->tsize = size;
599#if BufStats1
600 thr->totalloc += (size_t)size;
601 thr->numget++; /* Increment number of bget() calls */
602 thr->numdget++; /* Direct bget() call count */
603#endif
604 buf = (void *)(bdh + 1);
605 KMP_DEBUG_ASSERT(((size_t)buf) % SizeQuant == 0)if (!(((size_t)buf) % 8 == 0)) { __kmp_debug_assert("((size_t)buf) % 8 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 605); }
;
606 return buf;
607 }
608
609 } else {
610
611 /* Try to obtain a new expansion block */
612 void *newpool;
613
614 KE_TRACE(10, ("%%%%%% MALLOCB( %d )\n", (int)thr->exp_incr))if (kmp_e_debug >= 10) { __kmp_debug_printf ("%%%%%% MALLOCB( %d )\n"
, (int)thr->exp_incr); }
;
615
616 /* richryan */
617 newpool = (*thr->acqfcn)((bufsize)thr->exp_incr);
618 KMP_DEBUG_ASSERT(((size_t)newpool) % SizeQuant == 0)if (!(((size_t)newpool) % 8 == 0)) { __kmp_debug_assert("((size_t)newpool) % 8 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 618); }
;
619 if (newpool != NULL__null) {
620 bpool(th, newpool, thr->exp_incr);
621 buf = bget(
622 th, requested_size); /* This can't, I say, can't get into a loop. */
623 return buf;
624 }
625 }
626 }
627
628 /* Still no buffer available */
629
630 return NULL__null;
631}
632
633/* BGETZ -- Allocate a buffer and clear its contents to zero. We clear
634 the entire contents of the buffer to zero, not just the
635 region requested by the caller. */
636
637static void *bgetz(kmp_info_t *th, bufsize size) {
638 char *buf = (char *)bget(th, size);
639
640 if (buf != NULL__null) {
641 bhead_t *b;
642 bufsize rsize;
643
644 b = BH(buf - sizeof(bhead_t))((bhead_t *)(buf - sizeof(bhead_t)));
645 rsize = -(b->bb.bsize);
646 if (rsize == 0) {
647 bdhead_t *bd;
648
649 bd = BDH(buf - sizeof(bdhead_t))((bdhead_t *)(buf - sizeof(bdhead_t)));
650 rsize = bd->tsize - (bufsize)sizeof(bdhead_t);
651 } else {
652 rsize -= sizeof(bhead_t);
653 }
654
655 KMP_DEBUG_ASSERT(rsize >= size)if (!(rsize >= size)) { __kmp_debug_assert("rsize >= size"
, "openmp/runtime/src/kmp_alloc.cpp", 655); }
;
656
657 (void)memset(buf, 0, (bufsize)rsize);
658 }
659 return ((void *)buf);
660}
661
662/* BGETR -- Reallocate a buffer. This is a minimal implementation,
663 simply in terms of brel() and bget(). It could be
664 enhanced to allow the buffer to grow into adjacent free
665 blocks and to avoid moving data unnecessarily. */
666
667static void *bgetr(kmp_info_t *th, void *buf, bufsize size) {
668 void *nbuf;
669 bufsize osize; /* Old size of buffer */
670 bhead_t *b;
671
672 nbuf = bget(th, size);
673 if (nbuf == NULL__null) { /* Acquire new buffer */
674 return NULL__null;
675 }
676 if (buf == NULL__null) {
677 return nbuf;
678 }
679 b = BH(((char *)buf) - sizeof(bhead_t))((bhead_t *)(((char *)buf) - sizeof(bhead_t)));
680 osize = -b->bb.bsize;
681 if (osize == 0) {
682 /* Buffer acquired directly through acqfcn. */
683 bdhead_t *bd;
684
685 bd = BDH(((char *)buf) - sizeof(bdhead_t))((bdhead_t *)(((char *)buf) - sizeof(bdhead_t)));
686 osize = bd->tsize - (bufsize)sizeof(bdhead_t);
687 } else {
688 osize -= sizeof(bhead_t);
689 }
690
691 KMP_DEBUG_ASSERT(osize > 0)if (!(osize > 0)) { __kmp_debug_assert("osize > 0", "openmp/runtime/src/kmp_alloc.cpp"
, 691); }
;
692
693 (void)KMP_MEMCPYmemcpy((char *)nbuf, (char *)buf, /* Copy the data */
694 (size_t)((size < osize) ? size : osize));
695 brel(th, buf);
696
697 return nbuf;
698}
699
700/* BREL -- Release a buffer. */
701static void brel(kmp_info_t *th, void *buf) {
702 thr_data_t *thr = get_thr_data(th);
703 bfhead_t *b, *bn;
704 kmp_info_t *bth;
705
706 KMP_DEBUG_ASSERT(buf != NULL)if (!(buf != __null)) { __kmp_debug_assert("buf != __null", "openmp/runtime/src/kmp_alloc.cpp"
, 706); }
;
707 KMP_DEBUG_ASSERT(((size_t)buf) % SizeQuant == 0)if (!(((size_t)buf) % 8 == 0)) { __kmp_debug_assert("((size_t)buf) % 8 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 707); }
;
708
709 b = BFH(((char *)buf) - sizeof(bhead_t))((bfhead_t *)(((char *)buf) - sizeof(bhead_t)));
710
711 if (b->bh.bb.bsize == 0) { /* Directly-acquired buffer? */
712 bdhead_t *bdh;
713
714 bdh = BDH(((char *)buf) - sizeof(bdhead_t))((bdhead_t *)(((char *)buf) - sizeof(bdhead_t)));
715 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0)if (!(b->bh.bb.prevfree == 0)) { __kmp_debug_assert("b->bh.bb.prevfree == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 715); }
;
716#if BufStats1
717 thr->totalloc -= (size_t)bdh->tsize;
718 thr->numdrel++; /* Number of direct releases */
719 thr->numrel++; /* Increment number of brel() calls */
720#endif /* BufStats */
721#ifdef FreeWipe
722 (void)memset((char *)buf, 0x55, (size_t)(bdh->tsize - sizeof(bdhead_t)));
723#endif /* FreeWipe */
724
725 KE_TRACE(10, ("%%%%%% FREE( %p )\n", (void *)bdh))if (kmp_e_debug >= 10) { __kmp_debug_printf ("%%%%%% FREE( %p )\n"
, (void *)bdh); }
;
726
727 KMP_DEBUG_ASSERT(thr->relfcn != 0)if (!(thr->relfcn != 0)) { __kmp_debug_assert("thr->relfcn != 0"
, "openmp/runtime/src/kmp_alloc.cpp", 727); }
;
728 (*thr->relfcn)((void *)bdh); /* Release it directly. */
729 return;
730 }
731
732 bth = (kmp_info_t *)((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr)((void *)(b->bh.bb.bthr)) &
733 ~1); // clear possible mark before comparison
734 if (bth != th) {
735 /* Add this buffer to be released by the owning thread later */
736 __kmp_bget_enqueue(bth, buf
737#ifdef USE_QUEUING_LOCK_FOR_BGET
738 ,
739 __kmp_gtid_from_thread(th)
740#endif
741 );
742 return;
743 }
744
745 /* Buffer size must be negative, indicating that the buffer is allocated. */
746 if (b->bh.bb.bsize >= 0) {
747 bn = NULL__null;
748 }
749 KMP_DEBUG_ASSERT(b->bh.bb.bsize < 0)if (!(b->bh.bb.bsize < 0)) { __kmp_debug_assert("b->bh.bb.bsize < 0"
, "openmp/runtime/src/kmp_alloc.cpp", 749); }
;
750
751 /* Back pointer in next buffer must be zero, indicating the same thing: */
752
753 KMP_DEBUG_ASSERT(BH((char *)b - b->bh.bb.bsize)->bb.prevfree == 0)if (!(((bhead_t *)((char *)b - b->bh.bb.bsize))->bb.prevfree
== 0)) { __kmp_debug_assert("((bhead_t *)((char *)b - b->bh.bb.bsize))->bb.prevfree == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 753); }
;
754
755#if BufStats1
756 thr->numrel++; /* Increment number of brel() calls */
757 thr->totalloc += (size_t)b->bh.bb.bsize;
758#endif
759
760 /* If the back link is nonzero, the previous buffer is free. */
761
762 if (b->bh.bb.prevfree != 0) {
763 /* The previous buffer is free. Consolidate this buffer with it by adding
764 the length of this buffer to the previous free buffer. Note that we
765 subtract the size in the buffer being released, since it's negative to
766 indicate that the buffer is allocated. */
767 bufsize size = b->bh.bb.bsize;
768
769 /* Make the previous buffer the one we're working on. */
770 KMP_DEBUG_ASSERT(BH((char *)b - b->bh.bb.prevfree)->bb.bsize ==if (!(((bhead_t *)((char *)b - b->bh.bb.prevfree))->bb.
bsize == b->bh.bb.prevfree)) { __kmp_debug_assert("((bhead_t *)((char *)b - b->bh.bb.prevfree))->bb.bsize == b->bh.bb.prevfree"
, "openmp/runtime/src/kmp_alloc.cpp", 771); }
771 b->bh.bb.prevfree)if (!(((bhead_t *)((char *)b - b->bh.bb.prevfree))->bb.
bsize == b->bh.bb.prevfree)) { __kmp_debug_assert("((bhead_t *)((char *)b - b->bh.bb.prevfree))->bb.bsize == b->bh.bb.prevfree"
, "openmp/runtime/src/kmp_alloc.cpp", 771); }
;
772 b = BFH(((char *)b) - b->bh.bb.prevfree)((bfhead_t *)(((char *)b) - b->bh.bb.prevfree));
773 b->bh.bb.bsize -= size;
774
775 /* unlink the buffer from the old freelist */
776 __kmp_bget_remove_from_freelist(b);
777 } else {
778 /* The previous buffer isn't allocated. Mark this buffer size as positive
779 (i.e. free) and fall through to place the buffer on the free list as an
780 isolated free block. */
781 b->bh.bb.bsize = -b->bh.bb.bsize;
782 }
783
784 /* insert buffer back onto a new freelist */
785 __kmp_bget_insert_into_freelist(thr, b);
786
787 /* Now we look at the next buffer in memory, located by advancing from
788 the start of this buffer by its size, to see if that buffer is
789 free. If it is, we combine this buffer with the next one in
790 memory, dechaining the second buffer from the free list. */
791 bn = BFH(((char *)b) + b->bh.bb.bsize)((bfhead_t *)(((char *)b) + b->bh.bb.bsize));
792 if (bn->bh.bb.bsize > 0) {
793
794 /* The buffer is free. Remove it from the free list and add
795 its size to that of our buffer. */
796 KMP_DEBUG_ASSERT(BH((char *)bn + bn->bh.bb.bsize)->bb.prevfree ==if (!(((bhead_t *)((char *)bn + bn->bh.bb.bsize))->bb.prevfree
== bn->bh.bb.bsize)) { __kmp_debug_assert("((bhead_t *)((char *)bn + bn->bh.bb.bsize))->bb.prevfree == bn->bh.bb.bsize"
, "openmp/runtime/src/kmp_alloc.cpp", 797); }
797 bn->bh.bb.bsize)if (!(((bhead_t *)((char *)bn + bn->bh.bb.bsize))->bb.prevfree
== bn->bh.bb.bsize)) { __kmp_debug_assert("((bhead_t *)((char *)bn + bn->bh.bb.bsize))->bb.prevfree == bn->bh.bb.bsize"
, "openmp/runtime/src/kmp_alloc.cpp", 797); }
;
798
799 __kmp_bget_remove_from_freelist(bn);
800
801 b->bh.bb.bsize += bn->bh.bb.bsize;
802
803 /* unlink the buffer from the old freelist, and reinsert it into the new
804 * freelist */
805 __kmp_bget_remove_from_freelist(b);
806 __kmp_bget_insert_into_freelist(thr, b);
807
808 /* Finally, advance to the buffer that follows the newly
809 consolidated free block. We must set its backpointer to the
810 head of the consolidated free block. We know the next block
811 must be an allocated block because the process of recombination
812 guarantees that two free blocks will never be contiguous in
813 memory. */
814 bn = BFH(((char *)b) + b->bh.bb.bsize)((bfhead_t *)(((char *)b) + b->bh.bb.bsize));
815 }
816#ifdef FreeWipe
817 (void)memset(((char *)b) + sizeof(bfhead_t), 0x55,
818 (size_t)(b->bh.bb.bsize - sizeof(bfhead_t)));
819#endif
820 KMP_DEBUG_ASSERT(bn->bh.bb.bsize < 0)if (!(bn->bh.bb.bsize < 0)) { __kmp_debug_assert("bn->bh.bb.bsize < 0"
, "openmp/runtime/src/kmp_alloc.cpp", 820); }
;
821
822 /* The next buffer is allocated. Set the backpointer in it to point
823 to this buffer; the previous free buffer in memory. */
824
825 bn->bh.bb.prevfree = b->bh.bb.bsize;
826
827 /* If a block-release function is defined, and this free buffer
828 constitutes the entire block, release it. Note that pool_len
829 is defined in such a way that the test will fail unless all
830 pool blocks are the same size. */
831 if (thr->relfcn != 0 &&
832 b->bh.bb.bsize == (bufsize)(thr->pool_len - sizeof(bhead_t))) {
833#if BufStats1
834 if (thr->numpblk !=
835 1) { /* Do not release the last buffer until finalization time */
836#endif
837
838 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0)if (!(b->bh.bb.prevfree == 0)) { __kmp_debug_assert("b->bh.bb.prevfree == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 838); }
;
839 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.bsize == ESent)if (!(((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.bsize
== ((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize
) * 8 - 2)) - 1) * 2) - 2)))) { __kmp_debug_assert("((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.bsize == ((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2))"
, "openmp/runtime/src/kmp_alloc.cpp", 839); }
;
840 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.prevfree ==if (!(((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree
== b->bh.bb.bsize)) { __kmp_debug_assert("((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree == b->bh.bb.bsize"
, "openmp/runtime/src/kmp_alloc.cpp", 841); }
841 b->bh.bb.bsize)if (!(((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree
== b->bh.bb.bsize)) { __kmp_debug_assert("((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree == b->bh.bb.bsize"
, "openmp/runtime/src/kmp_alloc.cpp", 841); }
;
842
843 /* Unlink the buffer from the free list */
844 __kmp_bget_remove_from_freelist(b);
845
846 KE_TRACE(10, ("%%%%%% FREE( %p )\n", (void *)b))if (kmp_e_debug >= 10) { __kmp_debug_printf ("%%%%%% FREE( %p )\n"
, (void *)b); }
;
847
848 (*thr->relfcn)(b);
849#if BufStats1
850 thr->numprel++; /* Nr of expansion block releases */
851 thr->numpblk--; /* Total number of blocks */
852 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel)if (!(thr->numpblk == thr->numpget - thr->numprel)) {
__kmp_debug_assert("thr->numpblk == thr->numpget - thr->numprel"
, "openmp/runtime/src/kmp_alloc.cpp", 852); }
;
853
854 // avoid leaving stale last_pool pointer around if it is being dealloced
855 if (thr->last_pool == b)
856 thr->last_pool = 0;
857 } else {
858 thr->last_pool = b;
859 }
860#endif /* BufStats */
861 }
862}
863
864/* BECTL -- Establish automatic pool expansion control */
865static void bectl(kmp_info_t *th, bget_compact_t compact,
866 bget_acquire_t acquire, bget_release_t release,
867 bufsize pool_incr) {
868 thr_data_t *thr = get_thr_data(th);
869
870 thr->compfcn = compact;
871 thr->acqfcn = acquire;
872 thr->relfcn = release;
873 thr->exp_incr = pool_incr;
874}
875
876/* BPOOL -- Add a region of memory to the buffer pool. */
877static void bpool(kmp_info_t *th, void *buf, bufsize len) {
878 /* int bin = 0; */
879 thr_data_t *thr = get_thr_data(th);
880 bfhead_t *b = BFH(buf)((bfhead_t *)(buf));
881 bhead_t *bn;
882
883 __kmp_bget_dequeue(th); /* Release any queued buffers */
884
885#ifdef SizeQuant8
886 len &= ~((bufsize)(SizeQuant8 - 1));
887#endif
888 if (thr->pool_len == 0) {
889 thr->pool_len = len;
890 } else if (len != thr->pool_len) {
891 thr->pool_len = -1;
892 }
893#if BufStats1
894 thr->numpget++; /* Number of block acquisitions */
895 thr->numpblk++; /* Number of blocks total */
896 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel)if (!(thr->numpblk == thr->numpget - thr->numprel)) {
__kmp_debug_assert("thr->numpblk == thr->numpget - thr->numprel"
, "openmp/runtime/src/kmp_alloc.cpp", 896); }
;
897#endif /* BufStats */
898
899 /* Since the block is initially occupied by a single free buffer,
900 it had better not be (much) larger than the largest buffer
901 whose size we can store in bhead.bb.bsize. */
902 KMP_DEBUG_ASSERT(len - sizeof(bhead_t) <= -((bufsize)ESent + 1))if (!(len - sizeof(bhead_t) <= -((bufsize)((bufsize)(-((((
(bufsize)1) << ((int)sizeof(bufsize) * 8 - 2)) - 1) * 2
) - 2)) + 1))) { __kmp_debug_assert("len - sizeof(bhead_t) <= -((bufsize)((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2)) + 1)"
, "openmp/runtime/src/kmp_alloc.cpp", 902); }
;
903
904 /* Clear the backpointer at the start of the block to indicate that
905 there is no free block prior to this one. That blocks
906 recombination when the first block in memory is released. */
907 b->bh.bb.prevfree = 0;
908
909 /* Create a dummy allocated buffer at the end of the pool. This dummy
910 buffer is seen when a buffer at the end of the pool is released and
911 blocks recombination of the last buffer with the dummy buffer at
912 the end. The length in the dummy buffer is set to the largest
913 negative number to denote the end of the pool for diagnostic
914 routines (this specific value is not counted on by the actual
915 allocation and release functions). */
916 len -= sizeof(bhead_t);
917 b->bh.bb.bsize = (bufsize)len;
918 /* Set the owner of this buffer */
919 TCW_PTR(b->bh.bb.bthr,((b->bh.bb.bthr)) = (((kmp_info_t *)((kmp_uintptr_t)th | 1
)))
920 (kmp_info_t *)((kmp_uintptr_t)th |((b->bh.bb.bthr)) = (((kmp_info_t *)((kmp_uintptr_t)th | 1
)))
921 1))((b->bh.bb.bthr)) = (((kmp_info_t *)((kmp_uintptr_t)th | 1
)))
; // mark the buffer as allocated address
922
923 /* Chain the new block to the free list. */
924 __kmp_bget_insert_into_freelist(thr, b);
925
926#ifdef FreeWipe
927 (void)memset(((char *)b) + sizeof(bfhead_t), 0x55,
928 (size_t)(len - sizeof(bfhead_t)));
929#endif
930 bn = BH(((char *)b) + len)((bhead_t *)(((char *)b) + len));
931 bn->bb.prevfree = (bufsize)len;
932 /* Definition of ESent assumes two's complement! */
933 KMP_DEBUG_ASSERT((~0) == -1 && (bn != 0))if (!((~0) == -1 && (bn != 0))) { __kmp_debug_assert(
"(~0) == -1 && (bn != 0)", "openmp/runtime/src/kmp_alloc.cpp"
, 933); }
;
934
935 bn->bb.bsize = ESent((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8
- 2)) - 1) * 2) - 2))
;
936}
937
938/* BFREED -- Dump the free lists for this thread. */
939static void bfreed(kmp_info_t *th) {
940 int bin = 0, count = 0;
941 int gtid = __kmp_gtid_from_thread(th);
942 thr_data_t *thr = get_thr_data(th);
943
944#if BufStats1
945 __kmp_printf_no_lock("__kmp_printpool: T#%d total=%" KMP_UINT64_SPEC"llu"
946 " get=%" KMP_INT64_SPEC"lld" " rel=%" KMP_INT64_SPEC"lld"
947 " pblk=%" KMP_INT64_SPEC"lld" " pget=%" KMP_INT64_SPEC"lld"
948 " prel=%" KMP_INT64_SPEC"lld" " dget=%" KMP_INT64_SPEC"lld"
949 " drel=%" KMP_INT64_SPEC"lld" "\n",
950 gtid, (kmp_uint64)thr->totalloc, (kmp_int64)thr->numget,
951 (kmp_int64)thr->numrel, (kmp_int64)thr->numpblk,
952 (kmp_int64)thr->numpget, (kmp_int64)thr->numprel,
953 (kmp_int64)thr->numdget, (kmp_int64)thr->numdrel);
954#endif
955
956 for (bin = 0; bin < MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize)); ++bin) {
957 bfhead_t *b;
958
959 for (b = thr->freelist[bin].ql.flink; b != &thr->freelist[bin];
960 b = b->ql.flink) {
961 bufsize bs = b->bh.bb.bsize;
962
963 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b)if (!(b->ql.blink->ql.flink == b)) { __kmp_debug_assert
("b->ql.blink->ql.flink == b", "openmp/runtime/src/kmp_alloc.cpp"
, 963); }
;
964 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b)if (!(b->ql.flink->ql.blink == b)) { __kmp_debug_assert
("b->ql.flink->ql.blink == b", "openmp/runtime/src/kmp_alloc.cpp"
, 964); }
;
965 KMP_DEBUG_ASSERT(bs > 0)if (!(bs > 0)) { __kmp_debug_assert("bs > 0", "openmp/runtime/src/kmp_alloc.cpp"
, 965); }
;
966
967 count += 1;
968
969 __kmp_printf_no_lock(
970 "__kmp_printpool: T#%d Free block: 0x%p size %6ld bytes.\n", gtid, b,
971 (long)bs);
972#ifdef FreeWipe
973 {
974 char *lerr = ((char *)b) + sizeof(bfhead_t);
975 if ((bs > sizeof(bfhead_t)) &&
976 ((*lerr != 0x55) ||
977 (memcmp(lerr, lerr + 1, (size_t)(bs - (sizeof(bfhead_t) + 1))) !=
978 0))) {
979 __kmp_printf_no_lock("__kmp_printpool: T#%d (Contents of above "
980 "free block have been overstored.)\n",
981 gtid);
982 }
983 }
984#endif
985 }
986 }
987
988 if (count == 0)
989 __kmp_printf_no_lock("__kmp_printpool: T#%d No free blocks\n", gtid);
990}
991
992void __kmp_initialize_bget(kmp_info_t *th) {
993 KMP_DEBUG_ASSERT(SizeQuant >= sizeof(void *) && (th != 0))if (!(8 >= sizeof(void *) && (th != 0))) { __kmp_debug_assert
("8 >= sizeof(void *) && (th != 0)", "openmp/runtime/src/kmp_alloc.cpp"
, 993); }
;
994
995 set_thr_data(th);
996
997 bectl(th, (bget_compact_t)0, (bget_acquire_t)malloc, (bget_release_t)free,
998 (bufsize)__kmp_malloc_pool_incr);
999}
1000
1001void __kmp_finalize_bget(kmp_info_t *th) {
1002 thr_data_t *thr;
1003 bfhead_t *b;
1004
1005 KMP_DEBUG_ASSERT(th != 0)if (!(th != 0)) { __kmp_debug_assert("th != 0", "openmp/runtime/src/kmp_alloc.cpp"
, 1005); }
;
1006
1007#if BufStats1
1008 thr = (thr_data_t *)th->th.th_local.bget_data;
1009 KMP_DEBUG_ASSERT(thr != NULL)if (!(thr != __null)) { __kmp_debug_assert("thr != __null", "openmp/runtime/src/kmp_alloc.cpp"
, 1009); }
;
1010 b = thr->last_pool;
1011
1012 /* If a block-release function is defined, and this free buffer constitutes
1013 the entire block, release it. Note that pool_len is defined in such a way
1014 that the test will fail unless all pool blocks are the same size. */
1015
1016 // Deallocate the last pool if one exists because we no longer do it in brel()
1017 if (thr->relfcn != 0 && b != 0 && thr->numpblk != 0 &&
1018 b->bh.bb.bsize == (bufsize)(thr->pool_len - sizeof(bhead_t))) {
1019 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0)if (!(b->bh.bb.prevfree == 0)) { __kmp_debug_assert("b->bh.bb.prevfree == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 1019); }
;
1020 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.bsize == ESent)if (!(((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.bsize
== ((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize
) * 8 - 2)) - 1) * 2) - 2)))) { __kmp_debug_assert("((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.bsize == ((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2))"
, "openmp/runtime/src/kmp_alloc.cpp", 1020); }
;
1021 KMP_DEBUG_ASSERT(BH((char *)b + b->bh.bb.bsize)->bb.prevfree ==if (!(((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree
== b->bh.bb.bsize)) { __kmp_debug_assert("((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree == b->bh.bb.bsize"
, "openmp/runtime/src/kmp_alloc.cpp", 1022); }
1022 b->bh.bb.bsize)if (!(((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree
== b->bh.bb.bsize)) { __kmp_debug_assert("((bhead_t *)((char *)b + b->bh.bb.bsize))->bb.prevfree == b->bh.bb.bsize"
, "openmp/runtime/src/kmp_alloc.cpp", 1022); }
;
1023
1024 /* Unlink the buffer from the free list */
1025 __kmp_bget_remove_from_freelist(b);
1026
1027 KE_TRACE(10, ("%%%%%% FREE( %p )\n", (void *)b))if (kmp_e_debug >= 10) { __kmp_debug_printf ("%%%%%% FREE( %p )\n"
, (void *)b); }
;
1028
1029 (*thr->relfcn)(b);
1030 thr->numprel++; /* Nr of expansion block releases */
1031 thr->numpblk--; /* Total number of blocks */
1032 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel)if (!(thr->numpblk == thr->numpget - thr->numprel)) {
__kmp_debug_assert("thr->numpblk == thr->numpget - thr->numprel"
, "openmp/runtime/src/kmp_alloc.cpp", 1032); }
;
1033 }
1034#endif /* BufStats */
1035
1036 /* Deallocate bget_data */
1037 if (th->th.th_local.bget_data != NULL__null) {
1038 __kmp_free(th->th.th_local.bget_data)___kmp_free((th->th.th_local.bget_data), "openmp/runtime/src/kmp_alloc.cpp"
, 1038)
;
1039 th->th.th_local.bget_data = NULL__null;
1040 }
1041}
1042
1043void kmpc_set_poolsize(size_t size) {
1044 bectl(__kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id())), (bget_compact_t)0, (bget_acquire_t)malloc,
1045 (bget_release_t)free, (bufsize)size);
1046}
1047
1048size_t kmpc_get_poolsize(void) {
1049 thr_data_t *p;
1050
1051 p = get_thr_data(__kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id())));
1052
1053 return p->exp_incr;
1054}
1055
1056void kmpc_set_poolmode(int mode) {
1057 thr_data_t *p;
1058
1059 if (mode == bget_mode_fifo || mode == bget_mode_lifo ||
1060 mode == bget_mode_best) {
1061 p = get_thr_data(__kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id())));
1062 p->mode = (bget_mode_t)mode;
1063 }
1064}
1065
1066int kmpc_get_poolmode(void) {
1067 thr_data_t *p;
1068
1069 p = get_thr_data(__kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id())));
1070
1071 return p->mode;
1072}
1073
1074void kmpc_get_poolstat(size_t *maxmem, size_t *allmem) {
1075 kmp_info_t *th = __kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id()));
1076 bufsize a, b;
1077
1078 __kmp_bget_dequeue(th); /* Release any queued buffers */
1079
1080 bcheck(th, &a, &b);
1081
1082 *maxmem = a;
1083 *allmem = b;
1084}
1085
1086void kmpc_poolprint(void) {
1087 kmp_info_t *th = __kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id()));
1088
1089 __kmp_bget_dequeue(th); /* Release any queued buffers */
1090
1091 bfreed(th);
1092}
1093
1094#endif // #if KMP_USE_BGET
1095
1096void *kmpc_malloc(size_t size) {
1097 void *ptr;
1098 ptr = bget(__kmp_entry_thread(), (bufsize)(size + sizeof(ptr)));
1099 if (ptr != NULL__null) {
1100 // save allocated pointer just before one returned to user
1101 *(void **)ptr = ptr;
1102 ptr = (void **)ptr + 1;
1103 }
1104 return ptr;
1105}
1106
1107#define IS_POWER_OF_TWO(n)(((n) & ((n)-1)) == 0) (((n) & ((n)-1)) == 0)
1108
1109void *kmpc_aligned_malloc(size_t size, size_t alignment) {
1110 void *ptr;
1111 void *ptr_allocated;
1112 KMP_DEBUG_ASSERT(alignment < 32 * 1024)if (!(alignment < 32 * 1024)) { __kmp_debug_assert("alignment < 32 * 1024"
, "openmp/runtime/src/kmp_alloc.cpp", 1112); }
; // Alignment should not be too big
1113 if (!IS_POWER_OF_TWO(alignment)(((alignment) & ((alignment)-1)) == 0)) {
1114 // AC: do we need to issue a warning here?
1115 errno(*__errno_location ()) = EINVAL22;
1116 return NULL__null;
1117 }
1118 size = size + sizeof(void *) + alignment;
1119 ptr_allocated = bget(__kmp_entry_thread(), (bufsize)size);
1120 if (ptr_allocated != NULL__null) {
1121 // save allocated pointer just before one returned to user
1122 ptr = (void *)(((kmp_uintptr_t)ptr_allocated + sizeof(void *) + alignment) &
1123 ~(alignment - 1));
1124 *((void **)ptr - 1) = ptr_allocated;
1125 } else {
1126 ptr = NULL__null;
1127 }
1128 return ptr;
1129}
1130
1131void *kmpc_calloc(size_t nelem, size_t elsize) {
1132 void *ptr;
1133 ptr = bgetz(__kmp_entry_thread(), (bufsize)(nelem * elsize + sizeof(ptr)));
1134 if (ptr != NULL__null) {
1135 // save allocated pointer just before one returned to user
1136 *(void **)ptr = ptr;
1137 ptr = (void **)ptr + 1;
1138 }
1139 return ptr;
1140}
1141
1142void *kmpc_realloc(void *ptr, size_t size) {
1143 void *result = NULL__null;
1144 if (ptr == NULL__null) {
1145 // If pointer is NULL, realloc behaves like malloc.
1146 result = bget(__kmp_entry_thread(), (bufsize)(size + sizeof(ptr)));
1147 // save allocated pointer just before one returned to user
1148 if (result != NULL__null) {
1149 *(void **)result = result;
1150 result = (void **)result + 1;
1151 }
1152 } else if (size == 0) {
1153 // If size is 0, realloc behaves like free.
1154 // The thread must be registered by the call to kmpc_malloc() or
1155 // kmpc_calloc() before.
1156 // So it should be safe to call __kmp_get_thread(), not
1157 // __kmp_entry_thread().
1158 KMP_ASSERT(*((void **)ptr - 1))if (!(*((void **)ptr - 1))) { __kmp_debug_assert("*((void **)ptr - 1)"
, "openmp/runtime/src/kmp_alloc.cpp", 1158); }
;
1159 brel(__kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id())), *((void **)ptr - 1));
1160 } else {
1161 result = bgetr(__kmp_entry_thread(), *((void **)ptr - 1),
1162 (bufsize)(size + sizeof(ptr)));
1163 if (result != NULL__null) {
1164 *(void **)result = result;
1165 result = (void **)result + 1;
1166 }
1167 }
1168 return result;
1169}
1170
1171// NOTE: the library must have already been initialized by a previous allocate
1172void kmpc_free(void *ptr) {
1173 if (!__kmp_init_serial) {
1174 return;
1175 }
1176 if (ptr != NULL__null) {
1177 kmp_info_t *th = __kmp_get_thread()(__kmp_thread_from_gtid(__kmp_get_global_thread_id()));
1178 __kmp_bget_dequeue(th); /* Release any queued buffers */
1179 // extract allocated pointer and free it
1180 KMP_ASSERT(*((void **)ptr - 1))if (!(*((void **)ptr - 1))) { __kmp_debug_assert("*((void **)ptr - 1)"
, "openmp/runtime/src/kmp_alloc.cpp", 1180); }
;
1181 brel(th, *((void **)ptr - 1));
1182 }
1183}
1184
1185void *___kmp_thread_malloc(kmp_info_t *th, size_t size KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1186 void *ptr;
1187 KE_TRACE(30, ("-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n", th,if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n"
, th, (int)size , _file_, _line_); }
1188 (int)size KMP_SRC_LOC_PARM))if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n"
, th, (int)size , _file_, _line_); }
;
1189 ptr = bget(th, (bufsize)size);
1190 KE_TRACE(30, ("<- __kmp_thread_malloc() returns %p\n", ptr))if (kmp_e_debug >= 30) { __kmp_debug_printf ("<- __kmp_thread_malloc() returns %p\n"
, ptr); }
;
1191 return ptr;
1192}
1193
1194void *___kmp_thread_calloc(kmp_info_t *th, size_t nelem,
1195 size_t elsize KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1196 void *ptr;
1197 KE_TRACE(30, ("-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n", th,if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n"
, th, (int)nelem, (int)elsize , _file_, _line_); }
1198 (int)nelem, (int)elsize KMP_SRC_LOC_PARM))if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n"
, th, (int)nelem, (int)elsize , _file_, _line_); }
;
1199 ptr = bgetz(th, (bufsize)(nelem * elsize));
1200 KE_TRACE(30, ("<- __kmp_thread_calloc() returns %p\n", ptr))if (kmp_e_debug >= 30) { __kmp_debug_printf ("<- __kmp_thread_calloc() returns %p\n"
, ptr); }
;
1201 return ptr;
1202}
1203
1204void *___kmp_thread_realloc(kmp_info_t *th, void *ptr,
1205 size_t size KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1206 KE_TRACE(30, ("-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n", th,if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n"
, th, ptr, (int)size , _file_, _line_); }
1207 ptr, (int)size KMP_SRC_LOC_PARM))if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n"
, th, ptr, (int)size , _file_, _line_); }
;
1208 ptr = bgetr(th, ptr, (bufsize)size);
1209 KE_TRACE(30, ("<- __kmp_thread_realloc() returns %p\n", ptr))if (kmp_e_debug >= 30) { __kmp_debug_printf ("<- __kmp_thread_realloc() returns %p\n"
, ptr); }
;
1210 return ptr;
1211}
1212
1213void ___kmp_thread_free(kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1214 KE_TRACE(30, ("-> __kmp_thread_free( %p, %p ) called from %s:%d\n", th,if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_free( %p, %p ) called from %s:%d\n"
, th, ptr , _file_, _line_); }
1215 ptr KMP_SRC_LOC_PARM))if (kmp_e_debug >= 30) { __kmp_debug_printf ("-> __kmp_thread_free( %p, %p ) called from %s:%d\n"
, th, ptr , _file_, _line_); }
;
1216 if (ptr != NULL__null) {
1217 __kmp_bget_dequeue(th); /* Release any queued buffers */
1218 brel(th, ptr);
1219 }
1220 KE_TRACE(30, ("<- __kmp_thread_free()\n"))if (kmp_e_debug >= 30) { __kmp_debug_printf ("<- __kmp_thread_free()\n"
); }
;
1221}
1222
1223/* OMP 5.0 Memory Management support */
1224static const char *kmp_mk_lib_name;
1225static void *h_memkind;
1226/* memkind experimental API: */
1227// memkind_alloc
1228static void *(*kmp_mk_alloc)(void *k, size_t sz);
1229// memkind_free
1230static void (*kmp_mk_free)(void *kind, void *ptr);
1231// memkind_check_available
1232static int (*kmp_mk_check)(void *kind);
1233// kinds we are going to use
1234static void **mk_default;
1235static void **mk_interleave;
1236static void **mk_hbw;
1237static void **mk_hbw_interleave;
1238static void **mk_hbw_preferred;
1239static void **mk_hugetlb;
1240static void **mk_hbw_hugetlb;
1241static void **mk_hbw_preferred_hugetlb;
1242static void **mk_dax_kmem;
1243static void **mk_dax_kmem_all;
1244static void **mk_dax_kmem_preferred;
1245static void *(*kmp_target_alloc_host)(size_t size, int device);
1246static void *(*kmp_target_alloc_shared)(size_t size, int device);
1247static void *(*kmp_target_alloc_device)(size_t size, int device);
1248static void *(*kmp_target_free_host)(void *ptr, int device);
1249static void *(*kmp_target_free_shared)(void *ptr, int device);
1250static void *(*kmp_target_free_device)(void *ptr, int device);
1251static bool __kmp_target_mem_available;
1252#define KMP_IS_TARGET_MEM_SPACE(MS)(MS == llvm_omp_target_host_mem_space || MS == llvm_omp_target_shared_mem_space
|| MS == llvm_omp_target_device_mem_space)
\
1253 (MS == llvm_omp_target_host_mem_space || \
1254 MS == llvm_omp_target_shared_mem_space || \
1255 MS == llvm_omp_target_device_mem_space)
1256#define KMP_IS_TARGET_MEM_ALLOC(MA)(MA == llvm_omp_target_host_mem_alloc || MA == llvm_omp_target_shared_mem_alloc
|| MA == llvm_omp_target_device_mem_alloc)
\
1257 (MA == llvm_omp_target_host_mem_alloc || \
1258 MA == llvm_omp_target_shared_mem_alloc || \
1259 MA == llvm_omp_target_device_mem_alloc)
1260
1261#if KMP_OS_UNIX1 && KMP_DYNAMIC_LIB1 && !KMP_OS_DARWIN0
1262static inline void chk_kind(void ***pkind) {
1263 KMP_DEBUG_ASSERT(pkind)if (!(pkind)) { __kmp_debug_assert("pkind", "openmp/runtime/src/kmp_alloc.cpp"
, 1263); }
;
1264 if (*pkind) // symbol found
1265 if (kmp_mk_check(**pkind)) // kind not available or error
1266 *pkind = NULL__null;
1267}
1268#endif
1269
1270void __kmp_init_memkind() {
1271// as of 2018-07-31 memkind does not support Windows*, exclude it for now
1272#if KMP_OS_UNIX1 && KMP_DYNAMIC_LIB1 && !KMP_OS_DARWIN0
1273 // use of statically linked memkind is problematic, as it depends on libnuma
1274 kmp_mk_lib_name = "libmemkind.so";
1275 h_memkind = dlopen(kmp_mk_lib_name, RTLD_LAZY0x00001);
1276 if (h_memkind) {
1277 kmp_mk_check = (int (*)(void *))dlsym(h_memkind, "memkind_check_available");
1278 kmp_mk_alloc =
1279 (void *(*)(void *, size_t))dlsym(h_memkind, "memkind_malloc");
1280 kmp_mk_free = (void (*)(void *, void *))dlsym(h_memkind, "memkind_free");
1281 mk_default = (void **)dlsym(h_memkind, "MEMKIND_DEFAULT");
1282 if (kmp_mk_check && kmp_mk_alloc && kmp_mk_free && mk_default &&
1283 !kmp_mk_check(*mk_default)) {
1284 __kmp_memkind_available = 1;
1285 mk_interleave = (void **)dlsym(h_memkind, "MEMKIND_INTERLEAVE");
1286 chk_kind(&mk_interleave);
1287 mk_hbw = (void **)dlsym(h_memkind, "MEMKIND_HBW");
1288 chk_kind(&mk_hbw);
1289 mk_hbw_interleave = (void **)dlsym(h_memkind, "MEMKIND_HBW_INTERLEAVE");
1290 chk_kind(&mk_hbw_interleave);
1291 mk_hbw_preferred = (void **)dlsym(h_memkind, "MEMKIND_HBW_PREFERRED");
1292 chk_kind(&mk_hbw_preferred);
1293 mk_hugetlb = (void **)dlsym(h_memkind, "MEMKIND_HUGETLB");
1294 chk_kind(&mk_hugetlb);
1295 mk_hbw_hugetlb = (void **)dlsym(h_memkind, "MEMKIND_HBW_HUGETLB");
1296 chk_kind(&mk_hbw_hugetlb);
1297 mk_hbw_preferred_hugetlb =
1298 (void **)dlsym(h_memkind, "MEMKIND_HBW_PREFERRED_HUGETLB");
1299 chk_kind(&mk_hbw_preferred_hugetlb);
1300 mk_dax_kmem = (void **)dlsym(h_memkind, "MEMKIND_DAX_KMEM");
1301 chk_kind(&mk_dax_kmem);
1302 mk_dax_kmem_all = (void **)dlsym(h_memkind, "MEMKIND_DAX_KMEM_ALL");
1303 chk_kind(&mk_dax_kmem_all);
1304 mk_dax_kmem_preferred =
1305 (void **)dlsym(h_memkind, "MEMKIND_DAX_KMEM_PREFERRED");
1306 chk_kind(&mk_dax_kmem_preferred);
1307 KE_TRACE(25, ("__kmp_init_memkind: memkind library initialized\n"))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmp_init_memkind: memkind library initialized\n"
); }
;
1308 return; // success
1309 }
1310 dlclose(h_memkind); // failure
1311 }
1312#else // !(KMP_OS_UNIX && KMP_DYNAMIC_LIB)
1313 kmp_mk_lib_name = "";
1314#endif // !(KMP_OS_UNIX && KMP_DYNAMIC_LIB)
1315 h_memkind = NULL__null;
1316 kmp_mk_check = NULL__null;
1317 kmp_mk_alloc = NULL__null;
1318 kmp_mk_free = NULL__null;
1319 mk_default = NULL__null;
1320 mk_interleave = NULL__null;
1321 mk_hbw = NULL__null;
1322 mk_hbw_interleave = NULL__null;
1323 mk_hbw_preferred = NULL__null;
1324 mk_hugetlb = NULL__null;
1325 mk_hbw_hugetlb = NULL__null;
1326 mk_hbw_preferred_hugetlb = NULL__null;
1327 mk_dax_kmem = NULL__null;
1328 mk_dax_kmem_all = NULL__null;
1329 mk_dax_kmem_preferred = NULL__null;
1330}
1331
1332void __kmp_fini_memkind() {
1333#if KMP_OS_UNIX1 && KMP_DYNAMIC_LIB1
1334 if (__kmp_memkind_available)
1335 KE_TRACE(25, ("__kmp_fini_memkind: finalize memkind library\n"))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmp_fini_memkind: finalize memkind library\n"
); }
;
1336 if (h_memkind) {
1337 dlclose(h_memkind);
1338 h_memkind = NULL__null;
1339 }
1340 kmp_mk_check = NULL__null;
1341 kmp_mk_alloc = NULL__null;
1342 kmp_mk_free = NULL__null;
1343 mk_default = NULL__null;
1344 mk_interleave = NULL__null;
1345 mk_hbw = NULL__null;
1346 mk_hbw_interleave = NULL__null;
1347 mk_hbw_preferred = NULL__null;
1348 mk_hugetlb = NULL__null;
1349 mk_hbw_hugetlb = NULL__null;
1350 mk_hbw_preferred_hugetlb = NULL__null;
1351 mk_dax_kmem = NULL__null;
1352 mk_dax_kmem_all = NULL__null;
1353 mk_dax_kmem_preferred = NULL__null;
1354#endif
1355}
1356
1357void __kmp_init_target_mem() {
1358 *(void **)(&kmp_target_alloc_host) = KMP_DLSYM("llvm_omp_target_alloc_host")dlsym(((void *) 0), "llvm_omp_target_alloc_host");
1359 *(void **)(&kmp_target_alloc_shared) =
1360 KMP_DLSYM("llvm_omp_target_alloc_shared")dlsym(((void *) 0), "llvm_omp_target_alloc_shared");
1361 *(void **)(&kmp_target_alloc_device) =
1362 KMP_DLSYM("llvm_omp_target_alloc_device")dlsym(((void *) 0), "llvm_omp_target_alloc_device");
1363 *(void **)(&kmp_target_free_host) = KMP_DLSYM("llvm_omp_target_free_host")dlsym(((void *) 0), "llvm_omp_target_free_host");
1364 *(void **)(&kmp_target_free_shared) =
1365 KMP_DLSYM("llvm_omp_target_free_shared")dlsym(((void *) 0), "llvm_omp_target_free_shared");
1366 *(void **)(&kmp_target_free_device) =
1367 KMP_DLSYM("llvm_omp_target_free_device")dlsym(((void *) 0), "llvm_omp_target_free_device");
1368 __kmp_target_mem_available =
1369 kmp_target_alloc_host && kmp_target_alloc_shared &&
1370 kmp_target_alloc_device && kmp_target_free_host &&
1371 kmp_target_free_shared && kmp_target_free_device;
1372}
1373
1374omp_allocator_handle_t __kmpc_init_allocator(int gtid, omp_memspace_handle_t ms,
1375 int ntraits,
1376 omp_alloctrait_t traits[]) {
1377 // OpenMP 5.0 only allows predefined memspaces
1378 KMP_DEBUG_ASSERT(ms == omp_default_mem_space || ms == omp_low_lat_mem_space ||if (!(ms == omp_default_mem_space || ms == omp_low_lat_mem_space
|| ms == omp_large_cap_mem_space || ms == omp_const_mem_space
|| ms == omp_high_bw_mem_space || (ms == llvm_omp_target_host_mem_space
|| ms == llvm_omp_target_shared_mem_space || ms == llvm_omp_target_device_mem_space
))) { __kmp_debug_assert("ms == omp_default_mem_space || ms == omp_low_lat_mem_space || ms == omp_large_cap_mem_space || ms == omp_const_mem_space || ms == omp_high_bw_mem_space || (ms == llvm_omp_target_host_mem_space || ms == llvm_omp_target_shared_mem_space || ms == llvm_omp_target_device_mem_space)"
, "openmp/runtime/src/kmp_alloc.cpp", 1380); }
1379 ms == omp_large_cap_mem_space || ms == omp_const_mem_space ||if (!(ms == omp_default_mem_space || ms == omp_low_lat_mem_space
|| ms == omp_large_cap_mem_space || ms == omp_const_mem_space
|| ms == omp_high_bw_mem_space || (ms == llvm_omp_target_host_mem_space
|| ms == llvm_omp_target_shared_mem_space || ms == llvm_omp_target_device_mem_space
))) { __kmp_debug_assert("ms == omp_default_mem_space || ms == omp_low_lat_mem_space || ms == omp_large_cap_mem_space || ms == omp_const_mem_space || ms == omp_high_bw_mem_space || (ms == llvm_omp_target_host_mem_space || ms == llvm_omp_target_shared_mem_space || ms == llvm_omp_target_device_mem_space)"
, "openmp/runtime/src/kmp_alloc.cpp", 1380); }
1380 ms == omp_high_bw_mem_space || KMP_IS_TARGET_MEM_SPACE(ms))if (!(ms == omp_default_mem_space || ms == omp_low_lat_mem_space
|| ms == omp_large_cap_mem_space || ms == omp_const_mem_space
|| ms == omp_high_bw_mem_space || (ms == llvm_omp_target_host_mem_space
|| ms == llvm_omp_target_shared_mem_space || ms == llvm_omp_target_device_mem_space
))) { __kmp_debug_assert("ms == omp_default_mem_space || ms == omp_low_lat_mem_space || ms == omp_large_cap_mem_space || ms == omp_const_mem_space || ms == omp_high_bw_mem_space || (ms == llvm_omp_target_host_mem_space || ms == llvm_omp_target_shared_mem_space || ms == llvm_omp_target_device_mem_space)"
, "openmp/runtime/src/kmp_alloc.cpp", 1380); }
;
1381 kmp_allocator_t *al;
1382 int i;
1383 al = (kmp_allocator_t *)__kmp_allocate(sizeof(kmp_allocator_t))___kmp_allocate((sizeof(kmp_allocator_t)), "openmp/runtime/src/kmp_alloc.cpp"
, 1383)
; // zeroed
1384 al->memspace = ms; // not used currently
1385 for (i = 0; i < ntraits; ++i) {
1386 switch (traits[i].key) {
1387 case omp_atk_sync_hint:
1388 case omp_atk_access:
1389 case omp_atk_pinned:
1390 break;
1391 case omp_atk_alignment:
1392 __kmp_type_convert(traits[i].value, &(al->alignment));
1393 KMP_ASSERT(IS_POWER_OF_TWO(al->alignment))if (!((((al->alignment) & ((al->alignment)-1)) == 0
))) { __kmp_debug_assert("IS_POWER_OF_TWO(al->alignment)",
"openmp/runtime/src/kmp_alloc.cpp", 1393); }
;
1394 break;
1395 case omp_atk_pool_size:
1396 al->pool_size = traits[i].value;
1397 break;
1398 case omp_atk_fallback:
1399 al->fb = (omp_alloctrait_value_t)traits[i].value;
1400 KMP_DEBUG_ASSERT(if (!(al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb
|| al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb
)) { __kmp_debug_assert("al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb || al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb"
, "openmp/runtime/src/kmp_alloc.cpp", 1402); }
1401 al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb ||if (!(al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb
|| al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb
)) { __kmp_debug_assert("al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb || al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb"
, "openmp/runtime/src/kmp_alloc.cpp", 1402); }
1402 al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb)if (!(al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb
|| al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb
)) { __kmp_debug_assert("al->fb == omp_atv_default_mem_fb || al->fb == omp_atv_null_fb || al->fb == omp_atv_abort_fb || al->fb == omp_atv_allocator_fb"
, "openmp/runtime/src/kmp_alloc.cpp", 1402); }
;
1403 break;
1404 case omp_atk_fb_data:
1405 al->fb_data = RCAST(kmp_allocator_t *, traits[i].value)reinterpret_cast<kmp_allocator_t *>(traits[i].value);
1406 break;
1407 case omp_atk_partition:
1408 al->memkind = RCAST(void **, traits[i].value)reinterpret_cast<void **>(traits[i].value);
1409 break;
1410 default:
1411 KMP_ASSERT2(0, "Unexpected allocator trait")if (!(0)) { __kmp_debug_assert(("Unexpected allocator trait")
, "openmp/runtime/src/kmp_alloc.cpp", 1411); }
;
1412 }
1413 }
1414 if (al->fb == 0) {
1415 // set default allocator
1416 al->fb = omp_atv_default_mem_fb;
1417 al->fb_data = (kmp_allocator_t *)omp_default_mem_alloc;
1418 } else if (al->fb == omp_atv_allocator_fb) {
1419 KMP_ASSERT(al->fb_data != NULL)if (!(al->fb_data != __null)) { __kmp_debug_assert("al->fb_data != NULL"
, "openmp/runtime/src/kmp_alloc.cpp", 1419); }
;
1420 } else if (al->fb == omp_atv_default_mem_fb) {
1421 al->fb_data = (kmp_allocator_t *)omp_default_mem_alloc;
1422 }
1423 if (__kmp_memkind_available) {
1424 // Let's use memkind library if available
1425 if (ms == omp_high_bw_mem_space) {
1426 if (al->memkind == (void *)omp_atv_interleaved && mk_hbw_interleave) {
1427 al->memkind = mk_hbw_interleave;
1428 } else if (mk_hbw_preferred) {
1429 // AC: do not try to use MEMKIND_HBW for now, because memkind library
1430 // cannot reliably detect exhaustion of HBW memory.
1431 // It could be possible using hbw_verify_memory_region() but memkind
1432 // manual says: "Using this function in production code may result in
1433 // serious performance penalty".
1434 al->memkind = mk_hbw_preferred;
1435 } else {
1436 // HBW is requested but not available --> return NULL allocator
1437 __kmp_free(al)___kmp_free((al), "openmp/runtime/src/kmp_alloc.cpp", 1437);
1438 return omp_null_allocator;
1439 }
1440 } else if (ms == omp_large_cap_mem_space) {
1441 if (mk_dax_kmem_all) {
1442 // All pmem nodes are visited
1443 al->memkind = mk_dax_kmem_all;
1444 } else if (mk_dax_kmem) {
1445 // Only closest pmem node is visited
1446 al->memkind = mk_dax_kmem;
1447 } else {
1448 __kmp_free(al)___kmp_free((al), "openmp/runtime/src/kmp_alloc.cpp", 1448);
1449 return omp_null_allocator;
1450 }
1451 } else {
1452 if (al->memkind == (void *)omp_atv_interleaved && mk_interleave) {
1453 al->memkind = mk_interleave;
1454 } else {
1455 al->memkind = mk_default;
1456 }
1457 }
1458 } else if (KMP_IS_TARGET_MEM_SPACE(ms)(ms == llvm_omp_target_host_mem_space || ms == llvm_omp_target_shared_mem_space
|| ms == llvm_omp_target_device_mem_space)
&& !__kmp_target_mem_available) {
1459 __kmp_free(al)___kmp_free((al), "openmp/runtime/src/kmp_alloc.cpp", 1459);
1460 return omp_null_allocator;
1461 } else {
1462 if (ms == omp_high_bw_mem_space) {
1463 // cannot detect HBW memory presence without memkind library
1464 __kmp_free(al)___kmp_free((al), "openmp/runtime/src/kmp_alloc.cpp", 1464);
1465 return omp_null_allocator;
1466 }
1467 }
1468 return (omp_allocator_handle_t)al;
1469}
1470
1471void __kmpc_destroy_allocator(int gtid, omp_allocator_handle_t allocator) {
1472 if (allocator > kmp_max_mem_alloc)
1473 __kmp_free(allocator)___kmp_free((allocator), "openmp/runtime/src/kmp_alloc.cpp", 1473
)
;
1474}
1475
1476void __kmpc_set_default_allocator(int gtid, omp_allocator_handle_t allocator) {
1477 if (allocator == omp_null_allocator)
1478 allocator = omp_default_mem_alloc;
1479 __kmp_threads[gtid]->th.th_def_allocator = allocator;
1480}
1481
1482omp_allocator_handle_t __kmpc_get_default_allocator(int gtid) {
1483 return __kmp_threads[gtid]->th.th_def_allocator;
1484}
1485
1486typedef struct kmp_mem_desc { // Memory block descriptor
1487 void *ptr_alloc; // Pointer returned by allocator
1488 size_t size_a; // Size of allocated memory block (initial+descriptor+align)
1489 size_t size_orig; // Original size requested
1490 void *ptr_align; // Pointer to aligned memory, returned
1491 kmp_allocator_t *allocator; // allocator
1492} kmp_mem_desc_t;
1493static int alignment = sizeof(void *); // align to pointer size by default
1494
1495// external interfaces are wrappers over internal implementation
1496void *__kmpc_alloc(int gtid, size_t size, omp_allocator_handle_t allocator) {
1497 KE_TRACE(25, ("__kmpc_alloc: T#%d (%d, %p)\n", gtid, (int)size, allocator))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_alloc: T#%d (%d, %p)\n"
, gtid, (int)size, allocator); }
;
1498 void *ptr = __kmp_alloc(gtid, 0, size, allocator);
1499 KE_TRACE(25, ("__kmpc_alloc returns %p, T#%d\n", ptr, gtid))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_alloc returns %p, T#%d\n"
, ptr, gtid); }
;
1500 return ptr;
1501}
1502
1503void *__kmpc_aligned_alloc(int gtid, size_t algn, size_t size,
1504 omp_allocator_handle_t allocator) {
1505 KE_TRACE(25, ("__kmpc_aligned_alloc: T#%d (%d, %d, %p)\n", gtid, (int)algn,if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_aligned_alloc: T#%d (%d, %d, %p)\n"
, gtid, (int)algn, (int)size, allocator); }
1506 (int)size, allocator))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_aligned_alloc: T#%d (%d, %d, %p)\n"
, gtid, (int)algn, (int)size, allocator); }
;
1507 void *ptr = __kmp_alloc(gtid, algn, size, allocator);
1508 KE_TRACE(25, ("__kmpc_aligned_alloc returns %p, T#%d\n", ptr, gtid))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_aligned_alloc returns %p, T#%d\n"
, ptr, gtid); }
;
1509 return ptr;
1510}
1511
1512void *__kmpc_calloc(int gtid, size_t nmemb, size_t size,
1513 omp_allocator_handle_t allocator) {
1514 KE_TRACE(25, ("__kmpc_calloc: T#%d (%d, %d, %p)\n", gtid, (int)nmemb,if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_calloc: T#%d (%d, %d, %p)\n"
, gtid, (int)nmemb, (int)size, allocator); }
1515 (int)size, allocator))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_calloc: T#%d (%d, %d, %p)\n"
, gtid, (int)nmemb, (int)size, allocator); }
;
1516 void *ptr = __kmp_calloc(gtid, 0, nmemb, size, allocator);
1517 KE_TRACE(25, ("__kmpc_calloc returns %p, T#%d\n", ptr, gtid))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_calloc returns %p, T#%d\n"
, ptr, gtid); }
;
1518 return ptr;
1519}
1520
1521void *__kmpc_realloc(int gtid, void *ptr, size_t size,
1522 omp_allocator_handle_t allocator,
1523 omp_allocator_handle_t free_allocator) {
1524 KE_TRACE(25, ("__kmpc_realloc: T#%d (%p, %d, %p, %p)\n", gtid, ptr, (int)size,if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_realloc: T#%d (%p, %d, %p, %p)\n"
, gtid, ptr, (int)size, allocator, free_allocator); }
1525 allocator, free_allocator))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_realloc: T#%d (%p, %d, %p, %p)\n"
, gtid, ptr, (int)size, allocator, free_allocator); }
;
1526 void *nptr = __kmp_realloc(gtid, ptr, size, allocator, free_allocator);
1527 KE_TRACE(25, ("__kmpc_realloc returns %p, T#%d\n", nptr, gtid))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_realloc returns %p, T#%d\n"
, nptr, gtid); }
;
1528 return nptr;
1529}
1530
1531void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t allocator) {
1532 KE_TRACE(25, ("__kmpc_free: T#%d free(%p,%p)\n", gtid, ptr, allocator))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmpc_free: T#%d free(%p,%p)\n"
, gtid, ptr, allocator); }
;
1533 ___kmpc_free(gtid, ptr, allocator);
1534 KE_TRACE(10, ("__kmpc_free: T#%d freed %p (%p)\n", gtid, ptr, allocator))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmpc_free: T#%d freed %p (%p)\n"
, gtid, ptr, allocator); }
;
1535 return;
1536}
1537
1538// internal implementation, called from inside the library
1539void *__kmp_alloc(int gtid, size_t algn, size_t size,
1540 omp_allocator_handle_t allocator) {
1541 void *ptr = NULL__null;
1542 kmp_allocator_t *al;
1543 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_alloc.cpp", 1543); }
;
1544 if (size == 0)
1545 return NULL__null;
1546 if (allocator == omp_null_allocator)
1547 allocator = __kmp_threads[gtid]->th.th_def_allocator;
1548
1549 al = RCAST(kmp_allocator_t *, allocator)reinterpret_cast<kmp_allocator_t *>(allocator);
1550
1551 int sz_desc = sizeof(kmp_mem_desc_t);
1552 kmp_mem_desc_t desc;
1553 kmp_uintptr_t addr; // address returned by allocator
1554 kmp_uintptr_t addr_align; // address to return to caller
1555 kmp_uintptr_t addr_descr; // address of memory block descriptor
1556 size_t align = alignment; // default alignment
1557 if (allocator > kmp_max_mem_alloc && al->alignment > align)
1558 align = al->alignment; // alignment required by allocator trait
1559 if (align < algn)
1560 align = algn; // max of allocator trait, parameter and sizeof(void*)
1561 desc.size_orig = size;
1562 desc.size_a = size + sz_desc + align;
1563
1564 if (__kmp_memkind_available) {
1565 if (allocator < kmp_max_mem_alloc) {
1566 // pre-defined allocator
1567 if (allocator == omp_high_bw_mem_alloc && mk_hbw_preferred) {
1568 ptr = kmp_mk_alloc(*mk_hbw_preferred, desc.size_a);
1569 } else if (allocator == omp_large_cap_mem_alloc && mk_dax_kmem_all) {
1570 ptr = kmp_mk_alloc(*mk_dax_kmem_all, desc.size_a);
1571 } else {
1572 ptr = kmp_mk_alloc(*mk_default, desc.size_a);
1573 }
1574 } else if (al->pool_size > 0) {
1575 // custom allocator with pool size requested
1576 kmp_uint64 used =
1577 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, desc.size_a)__sync_fetch_and_add((volatile kmp_int64 *)((kmp_int64 *)&
al->pool_used), (kmp_int64)(desc.size_a))
;
1578 if (used + desc.size_a > al->pool_size) {
1579 // not enough space, need to go fallback path
1580 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a)__sync_fetch_and_add((volatile kmp_int64 *)((kmp_int64 *)&
al->pool_used), (kmp_int64)(-desc.size_a))
;
1581 if (al->fb == omp_atv_default_mem_fb) {
1582 al = (kmp_allocator_t *)omp_default_mem_alloc;
1583 ptr = kmp_mk_alloc(*mk_default, desc.size_a);
1584 } else if (al->fb == omp_atv_abort_fb) {
1585 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_alloc.cpp"
, 1585); }
; // abort fallback requested
1586 } else if (al->fb == omp_atv_allocator_fb) {
1587 KMP_ASSERT(al != al->fb_data)if (!(al != al->fb_data)) { __kmp_debug_assert("al != al->fb_data"
, "openmp/runtime/src/kmp_alloc.cpp", 1587); }
;
1588 al = al->fb_data;
1589 return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
1590 } // else ptr == NULL;
1591 } else {
1592 // pool has enough space
1593 ptr = kmp_mk_alloc(*al->memkind, desc.size_a);
1594 if (ptr == NULL__null) {
1595 if (al->fb == omp_atv_default_mem_fb) {
1596 al = (kmp_allocator_t *)omp_default_mem_alloc;
1597 ptr = kmp_mk_alloc(*mk_default, desc.size_a);
1598 } else if (al->fb == omp_atv_abort_fb) {
1599 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_alloc.cpp"
, 1599); }
; // abort fallback requested
1600 } else if (al->fb == omp_atv_allocator_fb) {
1601 KMP_ASSERT(al != al->fb_data)if (!(al != al->fb_data)) { __kmp_debug_assert("al != al->fb_data"
, "openmp/runtime/src/kmp_alloc.cpp", 1601); }
;
1602 al = al->fb_data;
1603 return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
1604 }
1605 }
1606 }
1607 } else {
1608 // custom allocator, pool size not requested
1609 ptr = kmp_mk_alloc(*al->memkind, desc.size_a);
1610 if (ptr == NULL__null) {
1611 if (al->fb == omp_atv_default_mem_fb) {
1612 al = (kmp_allocator_t *)omp_default_mem_alloc;
1613 ptr = kmp_mk_alloc(*mk_default, desc.size_a);
1614 } else if (al->fb == omp_atv_abort_fb) {
1615 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_alloc.cpp"
, 1615); }
; // abort fallback requested
1616 } else if (al->fb == omp_atv_allocator_fb) {
1617 KMP_ASSERT(al != al->fb_data)if (!(al != al->fb_data)) { __kmp_debug_assert("al != al->fb_data"
, "openmp/runtime/src/kmp_alloc.cpp", 1617); }
;
1618 al = al->fb_data;
1619 return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
1620 }
1621 }
1622 }
1623 } else if (allocator < kmp_max_mem_alloc) {
1624 if (KMP_IS_TARGET_MEM_ALLOC(allocator)(allocator == llvm_omp_target_host_mem_alloc || allocator == llvm_omp_target_shared_mem_alloc
|| allocator == llvm_omp_target_device_mem_alloc)
) {
1625 // Use size input directly as the memory may not be accessible on host.
1626 // Use default device for now.
1627 if (__kmp_target_mem_available) {
1628 kmp_int32 device =
1629 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device;
1630 if (allocator == llvm_omp_target_host_mem_alloc)
1631 ptr = kmp_target_alloc_host(size, device);
1632 else if (allocator == llvm_omp_target_shared_mem_alloc)
1633 ptr = kmp_target_alloc_shared(size, device);
1634 else // allocator == llvm_omp_target_device_mem_alloc
1635 ptr = kmp_target_alloc_device(size, device);
1636 }
1637 return ptr;
1638 }
1639
1640 // pre-defined allocator
1641 if (allocator == omp_high_bw_mem_alloc) {
1642 // ptr = NULL;
1643 } else if (allocator == omp_large_cap_mem_alloc) {
1644 // warnings?
1645 } else {
1646 ptr = __kmp_thread_malloc(__kmp_thread_from_gtid(gtid), desc.size_a)___kmp_thread_malloc((__kmp_thread_from_gtid(gtid)), (desc.size_a
), "openmp/runtime/src/kmp_alloc.cpp", 1646)
;
1647 }
1648 } else if (KMP_IS_TARGET_MEM_SPACE(al->memspace)(al->memspace == llvm_omp_target_host_mem_space || al->
memspace == llvm_omp_target_shared_mem_space || al->memspace
== llvm_omp_target_device_mem_space)
) {
1649 if (__kmp_target_mem_available) {
1650 kmp_int32 device =
1651 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device;
1652 if (al->memspace == llvm_omp_target_host_mem_space)
1653 ptr = kmp_target_alloc_host(size, device);
1654 else if (al->memspace == llvm_omp_target_shared_mem_space)
1655 ptr = kmp_target_alloc_shared(size, device);
1656 else // al->memspace == llvm_omp_target_device_mem_space
1657 ptr = kmp_target_alloc_device(size, device);
1658 }
1659 return ptr;
1660 } else if (al->pool_size > 0) {
1661 // custom allocator with pool size requested
1662 kmp_uint64 used =
1663 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, desc.size_a)__sync_fetch_and_add((volatile kmp_int64 *)((kmp_int64 *)&
al->pool_used), (kmp_int64)(desc.size_a))
;
1664 if (used + desc.size_a > al->pool_size) {
1665 // not enough space, need to go fallback path
1666 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a)__sync_fetch_and_add((volatile kmp_int64 *)((kmp_int64 *)&
al->pool_used), (kmp_int64)(-desc.size_a))
;
1667 if (al->fb == omp_atv_default_mem_fb) {
1668 al = (kmp_allocator_t *)omp_default_mem_alloc;
1669 ptr = __kmp_thread_malloc(__kmp_thread_from_gtid(gtid), desc.size_a)___kmp_thread_malloc((__kmp_thread_from_gtid(gtid)), (desc.size_a
), "openmp/runtime/src/kmp_alloc.cpp", 1669)
;
1670 } else if (al->fb == omp_atv_abort_fb) {
1671 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_alloc.cpp"
, 1671); }
; // abort fallback requested
1672 } else if (al->fb == omp_atv_allocator_fb) {
1673 KMP_ASSERT(al != al->fb_data)if (!(al != al->fb_data)) { __kmp_debug_assert("al != al->fb_data"
, "openmp/runtime/src/kmp_alloc.cpp", 1673); }
;
1674 al = al->fb_data;
1675 return __kmp_alloc(gtid, algn, size, (omp_allocator_handle_t)al);
1676 } // else ptr == NULL;
1677 } else {
1678 // pool has enough space
1679 ptr = __kmp_thread_malloc(__kmp_thread_from_gtid(gtid), desc.size_a)___kmp_thread_malloc((__kmp_thread_from_gtid(gtid)), (desc.size_a
), "openmp/runtime/src/kmp_alloc.cpp", 1679)
;
1680 if (ptr == NULL__null && al->fb == omp_atv_abort_fb) {
1681 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_alloc.cpp"
, 1681); }
; // abort fallback requested
1682 } // no sense to look for another fallback because of same internal alloc
1683 }
1684 } else {
1685 // custom allocator, pool size not requested
1686 ptr = __kmp_thread_malloc(__kmp_thread_from_gtid(gtid), desc.size_a)___kmp_thread_malloc((__kmp_thread_from_gtid(gtid)), (desc.size_a
), "openmp/runtime/src/kmp_alloc.cpp", 1686)
;
1687 if (ptr == NULL__null && al->fb == omp_atv_abort_fb) {
1688 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_alloc.cpp"
, 1688); }
; // abort fallback requested
1689 } // no sense to look for another fallback because of same internal alloc
1690 }
1691 KE_TRACE(10, ("__kmp_alloc: T#%d %p=alloc(%d)\n", gtid, ptr, desc.size_a))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_alloc: T#%d %p=alloc(%d)\n"
, gtid, ptr, desc.size_a); }
;
1692 if (ptr == NULL__null)
1693 return NULL__null;
1694
1695 addr = (kmp_uintptr_t)ptr;
1696 addr_align = (addr + sz_desc + align - 1) & ~(align - 1);
1697 addr_descr = addr_align - sz_desc;
1698
1699 desc.ptr_alloc = ptr;
1700 desc.ptr_align = (void *)addr_align;
1701 desc.allocator = al;
1702 *((kmp_mem_desc_t *)addr_descr) = desc; // save descriptor contents
1703 KMP_MB();
1704
1705 return desc.ptr_align;
1706}
1707
1708void *__kmp_calloc(int gtid, size_t algn, size_t nmemb, size_t size,
1709 omp_allocator_handle_t allocator) {
1710 void *ptr = NULL__null;
1711 kmp_allocator_t *al;
1712 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_alloc.cpp", 1712); }
;
1713
1714 if (allocator == omp_null_allocator)
1715 allocator = __kmp_threads[gtid]->th.th_def_allocator;
1716
1717 al = RCAST(kmp_allocator_t *, allocator)reinterpret_cast<kmp_allocator_t *>(allocator);
1718
1719 if (nmemb == 0 || size == 0)
1720 return ptr;
1721
1722 if ((SIZE_MAX(18446744073709551615UL) - sizeof(kmp_mem_desc_t)) / size < nmemb) {
1723 if (al->fb == omp_atv_abort_fb) {
1724 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_alloc.cpp"
, 1724); }
;
1725 }
1726 return ptr;
1727 }
1728
1729 ptr = __kmp_alloc(gtid, algn, nmemb * size, allocator);
1730
1731 if (ptr) {
1732 memset(ptr, 0x00, nmemb * size);
1733 }
1734 return ptr;
1735}
1736
1737void *__kmp_realloc(int gtid, void *ptr, size_t size,
1738 omp_allocator_handle_t allocator,
1739 omp_allocator_handle_t free_allocator) {
1740 void *nptr = NULL__null;
1741 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_alloc.cpp", 1741); }
;
1742
1743 if (size == 0) {
1744 if (ptr != NULL__null)
1745 ___kmpc_free(gtid, ptr, free_allocator);
1746 return nptr;
1747 }
1748
1749 nptr = __kmp_alloc(gtid, 0, size, allocator);
1750
1751 if (nptr != NULL__null && ptr != NULL__null) {
1752 kmp_mem_desc_t desc;
1753 kmp_uintptr_t addr_align; // address to return to caller
1754 kmp_uintptr_t addr_descr; // address of memory block descriptor
1755
1756 addr_align = (kmp_uintptr_t)ptr;
1757 addr_descr = addr_align - sizeof(kmp_mem_desc_t);
1758 desc = *((kmp_mem_desc_t *)addr_descr); // read descriptor
1759
1760 KMP_DEBUG_ASSERT(desc.ptr_align == ptr)if (!(desc.ptr_align == ptr)) { __kmp_debug_assert("desc.ptr_align == ptr"
, "openmp/runtime/src/kmp_alloc.cpp", 1760); }
;
1761 KMP_DEBUG_ASSERT(desc.size_orig > 0)if (!(desc.size_orig > 0)) { __kmp_debug_assert("desc.size_orig > 0"
, "openmp/runtime/src/kmp_alloc.cpp", 1761); }
;
1762 KMP_DEBUG_ASSERT(desc.size_orig < desc.size_a)if (!(desc.size_orig < desc.size_a)) { __kmp_debug_assert(
"desc.size_orig < desc.size_a", "openmp/runtime/src/kmp_alloc.cpp"
, 1762); }
;
1763 KMP_MEMCPYmemcpy((char *)nptr, (char *)ptr,
1764 (size_t)((size < desc.size_orig) ? size : desc.size_orig));
1765 }
1766
1767 if (nptr != NULL__null) {
1768 ___kmpc_free(gtid, ptr, free_allocator);
1769 }
1770
1771 return nptr;
1772}
1773
1774void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t allocator) {
1775 if (ptr == NULL__null)
1776 return;
1777
1778 kmp_allocator_t *al;
1779 omp_allocator_handle_t oal;
1780 al = RCAST(kmp_allocator_t *, CCAST(omp_allocator_handle_t, allocator))reinterpret_cast<kmp_allocator_t *>(const_cast<omp_allocator_handle_t
>(allocator))
;
1781 kmp_mem_desc_t desc;
1782 kmp_uintptr_t addr_align; // address to return to caller
1783 kmp_uintptr_t addr_descr; // address of memory block descriptor
1784 if (__kmp_target_mem_available && (KMP_IS_TARGET_MEM_ALLOC(allocator)(allocator == llvm_omp_target_host_mem_alloc || allocator == llvm_omp_target_shared_mem_alloc
|| allocator == llvm_omp_target_device_mem_alloc)
||
1785 (allocator > kmp_max_mem_alloc &&
1786 KMP_IS_TARGET_MEM_SPACE(al->memspace)(al->memspace == llvm_omp_target_host_mem_space || al->
memspace == llvm_omp_target_shared_mem_space || al->memspace
== llvm_omp_target_device_mem_space)
))) {
1787 kmp_int32 device =
1788 __kmp_threads[gtid]->th.th_current_task->td_icvs.default_device;
1789 if (allocator == llvm_omp_target_host_mem_alloc) {
1790 kmp_target_free_host(ptr, device);
1791 } else if (allocator == llvm_omp_target_shared_mem_alloc) {
1792 kmp_target_free_shared(ptr, device);
1793 } else if (allocator == llvm_omp_target_device_mem_alloc) {
1794 kmp_target_free_device(ptr, device);
1795 }
1796 return;
1797 }
1798
1799 addr_align = (kmp_uintptr_t)ptr;
1800 addr_descr = addr_align - sizeof(kmp_mem_desc_t);
1801 desc = *((kmp_mem_desc_t *)addr_descr); // read descriptor
1802
1803 KMP_DEBUG_ASSERT(desc.ptr_align == ptr)if (!(desc.ptr_align == ptr)) { __kmp_debug_assert("desc.ptr_align == ptr"
, "openmp/runtime/src/kmp_alloc.cpp", 1803); }
;
1804 if (allocator) {
1805 KMP_DEBUG_ASSERT(desc.allocator == al || desc.allocator == al->fb_data)if (!(desc.allocator == al || desc.allocator == al->fb_data
)) { __kmp_debug_assert("desc.allocator == al || desc.allocator == al->fb_data"
, "openmp/runtime/src/kmp_alloc.cpp", 1805); }
;
1806 }
1807 al = desc.allocator;
1808 oal = (omp_allocator_handle_t)al; // cast to void* for comparisons
1809 KMP_DEBUG_ASSERT(al)if (!(al)) { __kmp_debug_assert("al", "openmp/runtime/src/kmp_alloc.cpp"
, 1809); }
;
1810
1811 if (__kmp_memkind_available) {
1812 if (oal < kmp_max_mem_alloc) {
1813 // pre-defined allocator
1814 if (oal == omp_high_bw_mem_alloc && mk_hbw_preferred) {
1815 kmp_mk_free(*mk_hbw_preferred, desc.ptr_alloc);
1816 } else if (oal == omp_large_cap_mem_alloc && mk_dax_kmem_all) {
1817 kmp_mk_free(*mk_dax_kmem_all, desc.ptr_alloc);
1818 } else {
1819 kmp_mk_free(*mk_default, desc.ptr_alloc);
1820 }
1821 } else {
1822 if (al->pool_size > 0) { // custom allocator with pool size requested
1823 kmp_uint64 used =
1824 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a)__sync_fetch_and_add((volatile kmp_int64 *)((kmp_int64 *)&
al->pool_used), (kmp_int64)(-desc.size_a))
;
1825 (void)used; // to suppress compiler warning
1826 KMP_DEBUG_ASSERT(used >= desc.size_a)if (!(used >= desc.size_a)) { __kmp_debug_assert("used >= desc.size_a"
, "openmp/runtime/src/kmp_alloc.cpp", 1826); }
;
1827 }
1828 kmp_mk_free(*al->memkind, desc.ptr_alloc);
1829 }
1830 } else {
1831 if (oal > kmp_max_mem_alloc && al->pool_size > 0) {
1832 kmp_uint64 used =
1833 KMP_TEST_THEN_ADD64((kmp_int64 *)&al->pool_used, -desc.size_a)__sync_fetch_and_add((volatile kmp_int64 *)((kmp_int64 *)&
al->pool_used), (kmp_int64)(-desc.size_a))
;
1834 (void)used; // to suppress compiler warning
1835 KMP_DEBUG_ASSERT(used >= desc.size_a)if (!(used >= desc.size_a)) { __kmp_debug_assert("used >= desc.size_a"
, "openmp/runtime/src/kmp_alloc.cpp", 1835); }
;
1836 }
1837 __kmp_thread_free(__kmp_thread_from_gtid(gtid), desc.ptr_alloc)___kmp_thread_free((__kmp_thread_from_gtid(gtid)), (desc.ptr_alloc
), "openmp/runtime/src/kmp_alloc.cpp", 1837)
;
1838 }
1839}
1840
1841/* If LEAK_MEMORY is defined, __kmp_free() will *not* free memory. It causes
1842 memory leaks, but it may be useful for debugging memory corruptions, used
1843 freed pointers, etc. */
1844/* #define LEAK_MEMORY */
1845struct kmp_mem_descr { // Memory block descriptor.
1846 void *ptr_allocated; // Pointer returned by malloc(), subject for free().
1847 size_t size_allocated; // Size of allocated memory block.
1848 void *ptr_aligned; // Pointer to aligned memory, to be used by client code.
1849 size_t size_aligned; // Size of aligned memory block.
1850};
1851typedef struct kmp_mem_descr kmp_mem_descr_t;
1852
1853/* Allocate memory on requested boundary, fill allocated memory with 0x00.
1854 NULL is NEVER returned, __kmp_abort() is called in case of memory allocation
1855 error. Must use __kmp_free when freeing memory allocated by this routine! */
1856static void *___kmp_allocate_align(size_t size,
1857 size_t alignment KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1858 /* __kmp_allocate() allocates (by call to malloc()) bigger memory block than
1859 requested to return properly aligned pointer. Original pointer returned
1860 by malloc() and size of allocated block is saved in descriptor just
1861 before the aligned pointer. This information used by __kmp_free() -- it
1862 has to pass to free() original pointer, not aligned one.
1863
1864 +---------+------------+-----------------------------------+---------+
1865 | padding | descriptor | aligned block | padding |
1866 +---------+------------+-----------------------------------+---------+
1867 ^ ^
1868 | |
1869 | +- Aligned pointer returned to caller
1870 +- Pointer returned by malloc()
1871
1872 Aligned block is filled with zeros, paddings are filled with 0xEF. */
1873
1874 kmp_mem_descr_t descr;
1875 kmp_uintptr_t addr_allocated; // Address returned by malloc().
1876 kmp_uintptr_t addr_aligned; // Aligned address to return to caller.
1877 kmp_uintptr_t addr_descr; // Address of memory block descriptor.
1878
1879 KE_TRACE(25, ("-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n"
, (int)size, (int)alignment , _file_, _line_); }
1880 (int)size, (int)alignment KMP_SRC_LOC_PARM))if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n"
, (int)size, (int)alignment , _file_, _line_); }
;
1881
1882 KMP_DEBUG_ASSERT(alignment < 32 * 1024)if (!(alignment < 32 * 1024)) { __kmp_debug_assert("alignment < 32 * 1024"
, "openmp/runtime/src/kmp_alloc.cpp", 1882); }
; // Alignment should not be too
1883 KMP_DEBUG_ASSERT(sizeof(void *) <= sizeof(kmp_uintptr_t))if (!(sizeof(void *) <= sizeof(kmp_uintptr_t))) { __kmp_debug_assert
("sizeof(void *) <= sizeof(kmp_uintptr_t)", "openmp/runtime/src/kmp_alloc.cpp"
, 1883); }
;
1884 // Make sure kmp_uintptr_t is enough to store addresses.
1885
1886 descr.size_aligned = size;
1887 descr.size_allocated =
1888 descr.size_aligned + sizeof(kmp_mem_descr_t) + alignment;
1889
1890#if KMP_DEBUG1
1891 descr.ptr_allocated = _malloc_src_loc(descr.size_allocated, _file_, _line_)malloc((descr.size_allocated));
1892#else
1893 descr.ptr_allocated = malloc_src_loc(descr.size_allocated KMP_SRC_LOC_PARM)malloc((descr.size_allocated));
1894#endif
1895 KE_TRACE(10, (" malloc( %d ) returned %p\n", (int)descr.size_allocated,if (kmp_e_debug >= 10) { __kmp_debug_printf (" malloc( %d ) returned %p\n"
, (int)descr.size_allocated, descr.ptr_allocated); }
1896 descr.ptr_allocated))if (kmp_e_debug >= 10) { __kmp_debug_printf (" malloc( %d ) returned %p\n"
, (int)descr.size_allocated, descr.ptr_allocated); }
;
1897 if (descr.ptr_allocated == NULL__null) {
1898 KMP_FATAL(OutOfHeapMemory)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_OutOfHeapMemory), __kmp_msg_null
)
;
1899 }
1900
1901 addr_allocated = (kmp_uintptr_t)descr.ptr_allocated;
1902 addr_aligned =
1903 (addr_allocated + sizeof(kmp_mem_descr_t) + alignment) & ~(alignment - 1);
1904 addr_descr = addr_aligned - sizeof(kmp_mem_descr_t);
1905
1906 descr.ptr_aligned = (void *)addr_aligned;
1907
1908 KE_TRACE(26, (" ___kmp_allocate_align: "if (kmp_e_debug >= 26) { __kmp_debug_printf (" ___kmp_allocate_align: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1909 "ptr_allocated=%p, size_allocated=%d, "if (kmp_e_debug >= 26) { __kmp_debug_printf (" ___kmp_allocate_align: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1910 "ptr_aligned=%p, size_aligned=%d\n",if (kmp_e_debug >= 26) { __kmp_debug_printf (" ___kmp_allocate_align: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1911 descr.ptr_allocated, (int)descr.size_allocated,if (kmp_e_debug >= 26) { __kmp_debug_printf (" ___kmp_allocate_align: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1912 descr.ptr_aligned, (int)descr.size_aligned))if (kmp_e_debug >= 26) { __kmp_debug_printf (" ___kmp_allocate_align: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
;
1913
1914 KMP_DEBUG_ASSERT(addr_allocated <= addr_descr)if (!(addr_allocated <= addr_descr)) { __kmp_debug_assert(
"addr_allocated <= addr_descr", "openmp/runtime/src/kmp_alloc.cpp"
, 1914); }
;
1915 KMP_DEBUG_ASSERT(addr_descr + sizeof(kmp_mem_descr_t) == addr_aligned)if (!(addr_descr + sizeof(kmp_mem_descr_t) == addr_aligned)) {
__kmp_debug_assert("addr_descr + sizeof(kmp_mem_descr_t) == addr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 1915); }
;
1916 KMP_DEBUG_ASSERT(addr_aligned + descr.size_aligned <=if (!(addr_aligned + descr.size_aligned <= addr_allocated +
descr.size_allocated)) { __kmp_debug_assert("addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 1917); }
1917 addr_allocated + descr.size_allocated)if (!(addr_aligned + descr.size_aligned <= addr_allocated +
descr.size_allocated)) { __kmp_debug_assert("addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 1917); }
;
1918 KMP_DEBUG_ASSERT(addr_aligned % alignment == 0)if (!(addr_aligned % alignment == 0)) { __kmp_debug_assert("addr_aligned % alignment == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 1918); }
;
1919#ifdef KMP_DEBUG1
1920 memset(descr.ptr_allocated, 0xEF, descr.size_allocated);
1921// Fill allocated memory block with 0xEF.
1922#endif
1923 memset(descr.ptr_aligned, 0x00, descr.size_aligned);
1924 // Fill the aligned memory block (which is intended for using by caller) with
1925 // 0x00. Do not
1926 // put this filling under KMP_DEBUG condition! Many callers expect zeroed
1927 // memory. (Padding
1928 // bytes remain filled with 0xEF in debugging library.)
1929 *((kmp_mem_descr_t *)addr_descr) = descr;
1930
1931 KMP_MB();
1932
1933 KE_TRACE(25, ("<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned))if (kmp_e_debug >= 25) { __kmp_debug_printf ("<- ___kmp_allocate_align() returns %p\n"
, descr.ptr_aligned); }
;
1934 return descr.ptr_aligned;
1935} // func ___kmp_allocate_align
1936
1937/* Allocate memory on cache line boundary, fill allocated memory with 0x00.
1938 Do not call this func directly! Use __kmp_allocate macro instead.
1939 NULL is NEVER returned, __kmp_abort() is called in case of memory allocation
1940 error. Must use __kmp_free when freeing memory allocated by this routine! */
1941void *___kmp_allocate(size_t size KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1942 void *ptr;
1943 KE_TRACE(25, ("-> __kmp_allocate( %d ) called from %s:%d\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_allocate( %d ) called from %s:%d\n"
, (int)size , _file_, _line_); }
1944 (int)size KMP_SRC_LOC_PARM))if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_allocate( %d ) called from %s:%d\n"
, (int)size , _file_, _line_); }
;
1945 ptr = ___kmp_allocate_align(size, __kmp_align_alloc KMP_SRC_LOC_PARM, _file_, _line_);
1946 KE_TRACE(25, ("<- __kmp_allocate() returns %p\n", ptr))if (kmp_e_debug >= 25) { __kmp_debug_printf ("<- __kmp_allocate() returns %p\n"
, ptr); }
;
1947 return ptr;
1948} // func ___kmp_allocate
1949
1950/* Allocate memory on page boundary, fill allocated memory with 0x00.
1951 Does not call this func directly! Use __kmp_page_allocate macro instead.
1952 NULL is NEVER returned, __kmp_abort() is called in case of memory allocation
1953 error. Must use __kmp_free when freeing memory allocated by this routine! */
1954void *___kmp_page_allocate(size_t size KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1955 int page_size = 8 * 1024;
1956 void *ptr;
1957
1958 KE_TRACE(25, ("-> __kmp_page_allocate( %d ) called from %s:%d\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_page_allocate( %d ) called from %s:%d\n"
, (int)size , _file_, _line_); }
1959 (int)size KMP_SRC_LOC_PARM))if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_page_allocate( %d ) called from %s:%d\n"
, (int)size , _file_, _line_); }
;
1960 ptr = ___kmp_allocate_align(size, page_size KMP_SRC_LOC_PARM, _file_, _line_);
1961 KE_TRACE(25, ("<- __kmp_page_allocate( %d ) returns %p\n", (int)size, ptr))if (kmp_e_debug >= 25) { __kmp_debug_printf ("<- __kmp_page_allocate( %d ) returns %p\n"
, (int)size, ptr); }
;
1962 return ptr;
1963} // ___kmp_page_allocate
1964
1965/* Free memory allocated by __kmp_allocate() and __kmp_page_allocate().
1966 In debug mode, fill the memory block with 0xEF before call to free(). */
1967void ___kmp_free(void *ptr KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
1968 kmp_mem_descr_t descr;
1969#if KMP_DEBUG1
1970 kmp_uintptr_t addr_allocated; // Address returned by malloc().
1971 kmp_uintptr_t addr_aligned; // Aligned address passed by caller.
1972#endif
1973 KE_TRACE(25,if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_free( %p ) called from %s:%d\n"
, ptr , _file_, _line_); }
1974 ("-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM))if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_free( %p ) called from %s:%d\n"
, ptr , _file_, _line_); }
;
1975 KMP_ASSERT(ptr != NULL)if (!(ptr != __null)) { __kmp_debug_assert("ptr != NULL", "openmp/runtime/src/kmp_alloc.cpp"
, 1975); }
;
1976
1977 descr = *(kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t));
1978
1979 KE_TRACE(26, (" __kmp_free: "if (kmp_e_debug >= 26) { __kmp_debug_printf (" __kmp_free: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1980 "ptr_allocated=%p, size_allocated=%d, "if (kmp_e_debug >= 26) { __kmp_debug_printf (" __kmp_free: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1981 "ptr_aligned=%p, size_aligned=%d\n",if (kmp_e_debug >= 26) { __kmp_debug_printf (" __kmp_free: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1982 descr.ptr_allocated, (int)descr.size_allocated,if (kmp_e_debug >= 26) { __kmp_debug_printf (" __kmp_free: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
1983 descr.ptr_aligned, (int)descr.size_aligned))if (kmp_e_debug >= 26) { __kmp_debug_printf (" __kmp_free: "
"ptr_allocated=%p, size_allocated=%d, " "ptr_aligned=%p, size_aligned=%d\n"
, descr.ptr_allocated, (int)descr.size_allocated, descr.ptr_aligned
, (int)descr.size_aligned); }
;
1984#if KMP_DEBUG1
1985 addr_allocated = (kmp_uintptr_t)descr.ptr_allocated;
1986 addr_aligned = (kmp_uintptr_t)descr.ptr_aligned;
1987 KMP_DEBUG_ASSERT(addr_aligned % CACHE_LINE == 0)if (!(addr_aligned % 64 == 0)) { __kmp_debug_assert("addr_aligned % 64 == 0"
, "openmp/runtime/src/kmp_alloc.cpp", 1987); }
;
1988 KMP_DEBUG_ASSERT(descr.ptr_aligned == ptr)if (!(descr.ptr_aligned == ptr)) { __kmp_debug_assert("descr.ptr_aligned == ptr"
, "openmp/runtime/src/kmp_alloc.cpp", 1988); }
;
1989 KMP_DEBUG_ASSERT(addr_allocated + sizeof(kmp_mem_descr_t) <= addr_aligned)if (!(addr_allocated + sizeof(kmp_mem_descr_t) <= addr_aligned
)) { __kmp_debug_assert("addr_allocated + sizeof(kmp_mem_descr_t) <= addr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 1989); }
;
1990 KMP_DEBUG_ASSERT(descr.size_aligned < descr.size_allocated)if (!(descr.size_aligned < descr.size_allocated)) { __kmp_debug_assert
("descr.size_aligned < descr.size_allocated", "openmp/runtime/src/kmp_alloc.cpp"
, 1990); }
;
1991 KMP_DEBUG_ASSERT(addr_aligned + descr.size_aligned <=if (!(addr_aligned + descr.size_aligned <= addr_allocated +
descr.size_allocated)) { __kmp_debug_assert("addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 1992); }
1992 addr_allocated + descr.size_allocated)if (!(addr_aligned + descr.size_aligned <= addr_allocated +
descr.size_allocated)) { __kmp_debug_assert("addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 1992); }
;
1993 memset(descr.ptr_allocated, 0xEF, descr.size_allocated);
1994// Fill memory block with 0xEF, it helps catch using freed memory.
1995#endif
1996
1997#ifndef LEAK_MEMORY
1998 KE_TRACE(10, (" free( %p )\n", descr.ptr_allocated))if (kmp_e_debug >= 10) { __kmp_debug_printf (" free( %p )\n"
, descr.ptr_allocated); }
;
1999#ifdef KMP_DEBUG1
2000 _free_src_loc(descr.ptr_allocated, _file_, _line_)free((descr.ptr_allocated));
2001#else
2002 free_src_loc(descr.ptr_allocated KMP_SRC_LOC_PARM)free((descr.ptr_allocated));
2003#endif
2004#endif
2005 KMP_MB();
2006 KE_TRACE(25, ("<- __kmp_free() returns\n"))if (kmp_e_debug >= 25) { __kmp_debug_printf ("<- __kmp_free() returns\n"
); }
;
2007} // func ___kmp_free
2008
2009#if USE_FAST_MEMORY3 == 3
2010// Allocate fast memory by first scanning the thread's free lists
2011// If a chunk the right size exists, grab it off the free list.
2012// Otherwise allocate normally using kmp_thread_malloc.
2013
2014// AC: How to choose the limit? Just get 16 for now...
2015#define KMP_FREE_LIST_LIMIT16 16
2016
2017// Always use 128 bytes for determining buckets for caching memory blocks
2018#define DCACHE_LINE128 128
2019
2020void *___kmp_fast_allocate(kmp_info_t *this_thr, size_t size KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
2021 void *ptr;
2022 size_t num_lines, idx;
2023 int index;
2024 void *alloc_ptr;
2025 size_t alloc_size;
2026 kmp_mem_descr_t *descr;
2027
2028 KE_TRACE(25, ("-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n"
, __kmp_gtid_from_thread(this_thr), (int)size , _file_, _line_
); }
2029 __kmp_gtid_from_thread(this_thr), (int)size KMP_SRC_LOC_PARM))if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n"
, __kmp_gtid_from_thread(this_thr), (int)size , _file_, _line_
); }
;
2030
2031 num_lines = (size + DCACHE_LINE128 - 1) / DCACHE_LINE128;
2032 idx = num_lines - 1;
2033 KMP_DEBUG_ASSERT(idx >= 0)if (!(idx >= 0)) { __kmp_debug_assert("idx >= 0", "openmp/runtime/src/kmp_alloc.cpp"
, 2033); }
;
2034 if (idx < 2) {
2035 index = 0; // idx is [ 0, 1 ], use first free list
2036 num_lines = 2; // 1, 2 cache lines or less than cache line
2037 } else if ((idx >>= 2) == 0) {
2038 index = 1; // idx is [ 2, 3 ], use second free list
2039 num_lines = 4; // 3, 4 cache lines
2040 } else if ((idx >>= 2) == 0) {
2041 index = 2; // idx is [ 4, 15 ], use third free list
2042 num_lines = 16; // 5, 6, ..., 16 cache lines
2043 } else if ((idx >>= 2) == 0) {
2044 index = 3; // idx is [ 16, 63 ], use fourth free list
2045 num_lines = 64; // 17, 18, ..., 64 cache lines
2046 } else {
2047 goto alloc_call; // 65 or more cache lines ( > 8KB ), don't use free lists
2048 }
2049
2050 ptr = this_thr->th.th_free_lists[index].th_free_list_self;
2051 if (ptr != NULL__null) {
2052 // pop the head of no-sync free list
2053 this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr);
2054 KMP_DEBUG_ASSERT(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr -if (!(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof
(kmp_mem_descr_t))) ->ptr_aligned)) { __kmp_debug_assert("this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t))) ->ptr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 2056); }
2055 sizeof(kmp_mem_descr_t)))if (!(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof
(kmp_mem_descr_t))) ->ptr_aligned)) { __kmp_debug_assert("this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t))) ->ptr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 2056); }
2056 ->ptr_aligned)if (!(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof
(kmp_mem_descr_t))) ->ptr_aligned)) { __kmp_debug_assert("this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t))) ->ptr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 2056); }
;
2057 goto end;
2058 }
2059 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync)((void *)(this_thr->th.th_free_lists[index].th_free_list_sync
))
;
2060 if (ptr != NULL__null) {
2061 // no-sync free list is empty, use sync free list (filled in by other
2062 // threads only)
2063 // pop the head of the sync free list, push NULL instead
2064 while (!KMP_COMPARE_AND_STORE_PTR(__sync_bool_compare_and_swap((void *volatile *)(&this_thr
->th.th_free_lists[index].th_free_list_sync), (void *)(ptr
), (void *)(nullptr))
2065 &this_thr->th.th_free_lists[index].th_free_list_sync, ptr, nullptr)__sync_bool_compare_and_swap((void *volatile *)(&this_thr
->th.th_free_lists[index].th_free_list_sync), (void *)(ptr
), (void *)(nullptr))
) {
2066 KMP_CPU_PAUSE()__kmp_x86_pause();
2067 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync)((void *)(this_thr->th.th_free_lists[index].th_free_list_sync
))
;
2068 }
2069 // push the rest of chain into no-sync free list (can be NULL if there was
2070 // the only block)
2071 this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr);
2072 KMP_DEBUG_ASSERT(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr -if (!(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof
(kmp_mem_descr_t))) ->ptr_aligned)) { __kmp_debug_assert("this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t))) ->ptr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 2074); }
2073 sizeof(kmp_mem_descr_t)))if (!(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof
(kmp_mem_descr_t))) ->ptr_aligned)) { __kmp_debug_assert("this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t))) ->ptr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 2074); }
2074 ->ptr_aligned)if (!(this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof
(kmp_mem_descr_t))) ->ptr_aligned)) { __kmp_debug_assert("this_thr == ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t))) ->ptr_aligned"
, "openmp/runtime/src/kmp_alloc.cpp", 2074); }
;
2075 goto end;
2076 }
2077
2078alloc_call:
2079 // haven't found block in the free lists, thus allocate it
2080 size = num_lines * DCACHE_LINE128;
2081
2082 alloc_size = size + sizeof(kmp_mem_descr_t) + DCACHE_LINE128;
2083 KE_TRACE(25, ("__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with "if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with "
"alloc_size %d\n", __kmp_gtid_from_thread(this_thr), alloc_size
); }
2084 "alloc_size %d\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with "
"alloc_size %d\n", __kmp_gtid_from_thread(this_thr), alloc_size
); }
2085 __kmp_gtid_from_thread(this_thr), alloc_size))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with "
"alloc_size %d\n", __kmp_gtid_from_thread(this_thr), alloc_size
); }
;
2086 alloc_ptr = bget(this_thr, (bufsize)alloc_size);
2087
2088 // align ptr to DCACHE_LINE
2089 ptr = (void *)((((kmp_uintptr_t)alloc_ptr) + sizeof(kmp_mem_descr_t) +
2090 DCACHE_LINE128) &
2091 ~(DCACHE_LINE128 - 1));
2092 descr = (kmp_mem_descr_t *)(((kmp_uintptr_t)ptr) - sizeof(kmp_mem_descr_t));
2093
2094 descr->ptr_allocated = alloc_ptr; // remember allocated pointer
2095 // we don't need size_allocated
2096 descr->ptr_aligned = (void *)this_thr; // remember allocating thread
2097 // (it is already saved in bget buffer,
2098 // but we may want to use another allocator in future)
2099 descr->size_aligned = size;
2100
2101end:
2102 KE_TRACE(25, ("<- __kmp_fast_allocate( T#%d ) returns %p\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("<- __kmp_fast_allocate( T#%d ) returns %p\n"
, __kmp_gtid_from_thread(this_thr), ptr); }
2103 __kmp_gtid_from_thread(this_thr), ptr))if (kmp_e_debug >= 25) { __kmp_debug_printf ("<- __kmp_fast_allocate( T#%d ) returns %p\n"
, __kmp_gtid_from_thread(this_thr), ptr); }
;
2104 return ptr;
2105} // func __kmp_fast_allocate
2106
2107// Free fast memory and place it on the thread's free list if it is of
2108// the correct size.
2109void ___kmp_fast_free(kmp_info_t *this_thr, void *ptr KMP_SRC_LOC_DECL, char const *_file_, int _line_) {
2110 kmp_mem_descr_t *descr;
2111 kmp_info_t *alloc_thr;
2112 size_t size;
2113 size_t idx;
2114 int index;
2115
2116 KE_TRACE(25, ("-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n"
, __kmp_gtid_from_thread(this_thr), ptr , _file_, _line_); }
2117 __kmp_gtid_from_thread(this_thr), ptr KMP_SRC_LOC_PARM))if (kmp_e_debug >= 25) { __kmp_debug_printf ("-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n"
, __kmp_gtid_from_thread(this_thr), ptr , _file_, _line_); }
;
2118 KMP_ASSERT(ptr != NULL)if (!(ptr != __null)) { __kmp_debug_assert("ptr != NULL", "openmp/runtime/src/kmp_alloc.cpp"
, 2118); }
;
2119
2120 descr = (kmp_mem_descr_t *)(((kmp_uintptr_t)ptr) - sizeof(kmp_mem_descr_t));
2121
2122 KE_TRACE(26, (" __kmp_fast_free: size_aligned=%d\n",if (kmp_e_debug >= 26) { __kmp_debug_printf (" __kmp_fast_free: size_aligned=%d\n"
, (int)descr->size_aligned); }
2123 (int)descr->size_aligned))if (kmp_e_debug >= 26) { __kmp_debug_printf (" __kmp_fast_free: size_aligned=%d\n"
, (int)descr->size_aligned); }
;
2124
2125 size = descr->size_aligned; // 2, 4, 16, 64, 65, 66, ... cache lines
2126
2127 idx = DCACHE_LINE128 * 2; // 2 cache lines is minimal size of block
2128 if (idx == size) {
2129 index = 0; // 2 cache lines
2130 } else if ((idx <<= 1) == size) {
2131 index = 1; // 4 cache lines
2132 } else if ((idx <<= 2) == size) {
2133 index = 2; // 16 cache lines
2134 } else if ((idx <<= 2) == size) {
Although the value stored to 'idx' is used in the enclosing expression, the value is never actually read from 'idx'
2135 index = 3; // 64 cache lines
2136 } else {
2137 KMP_DEBUG_ASSERT(size > DCACHE_LINE * 64)if (!(size > 128 * 64)) { __kmp_debug_assert("size > 128 * 64"
, "openmp/runtime/src/kmp_alloc.cpp", 2137); }
;
2138 goto free_call; // 65 or more cache lines ( > 8KB )
2139 }
2140
2141 alloc_thr = (kmp_info_t *)descr->ptr_aligned; // get thread owning the block
2142 if (alloc_thr == this_thr) {
2143 // push block to self no-sync free list, linking previous head (LIFO)
2144 *((void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self;
2145 this_thr->th.th_free_lists[index].th_free_list_self = ptr;
2146 } else {
2147 void *head = this_thr->th.th_free_lists[index].th_free_list_other;
2148 if (head == NULL__null) {
2149 // Create new free list
2150 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
2151 *((void **)ptr) = NULL__null; // mark the tail of the list
2152 descr->size_allocated = (size_t)1; // head of the list keeps its length
2153 } else {
2154 // need to check existed "other" list's owner thread and size of queue
2155 kmp_mem_descr_t *dsc =
2156 (kmp_mem_descr_t *)((char *)head - sizeof(kmp_mem_descr_t));
2157 // allocating thread, same for all queue nodes
2158 kmp_info_t *q_th = (kmp_info_t *)(dsc->ptr_aligned);
2159 size_t q_sz =
2160 dsc->size_allocated + 1; // new size in case we add current task
2161 if (q_th == alloc_thr && q_sz <= KMP_FREE_LIST_LIMIT16) {
2162 // we can add current task to "other" list, no sync needed
2163 *((void **)ptr) = head;
2164 descr->size_allocated = q_sz;
2165 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
2166 } else {
2167 // either queue blocks owner is changing or size limit exceeded
2168 // return old queue to allocating thread (q_th) synchronously,
2169 // and start new list for alloc_thr's tasks
2170 void *old_ptr;
2171 void *tail = head;
2172 void *next = *((void **)head);
2173 while (next != NULL__null) {
2174 KMP_DEBUG_ASSERT(if (!(((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t
))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail
- sizeof(kmp_mem_descr_t))) ->size_allocated)) { __kmp_debug_assert
("((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) ->size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 2180); }
2175 // queue size should decrease by 1 each step through the listif (!(((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t
))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail
- sizeof(kmp_mem_descr_t))) ->size_allocated)) { __kmp_debug_assert
("((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) ->size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 2180); }
2176 ((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t)))if (!(((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t
))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail
- sizeof(kmp_mem_descr_t))) ->size_allocated)) { __kmp_debug_assert
("((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) ->size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 2180); }
2177 ->size_allocated +if (!(((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t
))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail
- sizeof(kmp_mem_descr_t))) ->size_allocated)) { __kmp_debug_assert
("((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) ->size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 2180); }
2178 1 ==if (!(((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t
))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail
- sizeof(kmp_mem_descr_t))) ->size_allocated)) { __kmp_debug_assert
("((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) ->size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 2180); }
2179 ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t)))if (!(((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t
))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail
- sizeof(kmp_mem_descr_t))) ->size_allocated)) { __kmp_debug_assert
("((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) ->size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 2180); }
2180 ->size_allocated)if (!(((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t
))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail
- sizeof(kmp_mem_descr_t))) ->size_allocated)) { __kmp_debug_assert
("((kmp_mem_descr_t *)((char *)next - sizeof(kmp_mem_descr_t))) ->size_allocated + 1 == ((kmp_mem_descr_t *)((char *)tail - sizeof(kmp_mem_descr_t))) ->size_allocated"
, "openmp/runtime/src/kmp_alloc.cpp", 2180); }
;
2181 tail = next; // remember tail node
2182 next = *((void **)next);
2183 }
2184 KMP_DEBUG_ASSERT(q_th != NULL)if (!(q_th != __null)) { __kmp_debug_assert("q_th != __null",
"openmp/runtime/src/kmp_alloc.cpp", 2184); }
;
2185 // push block to owner's sync free list
2186 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync)((void *)(q_th->th.th_free_lists[index].th_free_list_sync)
)
;
2187 /* the next pointer must be set before setting free_list to ptr to avoid
2188 exposing a broken list to other threads, even for an instant. */
2189 *((void **)tail) = old_ptr;
2190
2191 while (!KMP_COMPARE_AND_STORE_PTR(__sync_bool_compare_and_swap((void *volatile *)(&q_th->
th.th_free_lists[index].th_free_list_sync), (void *)(old_ptr)
, (void *)(head))
2192 &q_th->th.th_free_lists[index].th_free_list_sync, old_ptr, head)__sync_bool_compare_and_swap((void *volatile *)(&q_th->
th.th_free_lists[index].th_free_list_sync), (void *)(old_ptr)
, (void *)(head))
) {
2193 KMP_CPU_PAUSE()__kmp_x86_pause();
2194 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync)((void *)(q_th->th.th_free_lists[index].th_free_list_sync)
)
;
2195 *((void **)tail) = old_ptr;
2196 }
2197
2198 // start new list of not-selt tasks
2199 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
2200 *((void **)ptr) = NULL__null;
2201 descr->size_allocated = (size_t)1; // head of queue keeps its length
2202 }
2203 }
2204 }
2205 goto end;
2206
2207free_call:
2208 KE_TRACE(25, ("__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n",if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n"
, __kmp_gtid_from_thread(this_thr), size); }
2209 __kmp_gtid_from_thread(this_thr), size))if (kmp_e_debug >= 25) { __kmp_debug_printf ("__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n"
, __kmp_gtid_from_thread(this_thr), size); }
;
2210 __kmp_bget_dequeue(this_thr); /* Release any queued buffers */
2211 brel(this_thr, descr->ptr_allocated);
2212
2213end:
2214 KE_TRACE(25, ("<- __kmp_fast_free() returns\n"))if (kmp_e_debug >= 25) { __kmp_debug_printf ("<- __kmp_fast_free() returns\n"
); }
;
2215
2216} // func __kmp_fast_free
2217
2218// Initialize the thread free lists related to fast memory
2219// Only do this when a thread is initially created.
2220void __kmp_initialize_fast_memory(kmp_info_t *this_thr) {
2221 KE_TRACE(10, ("__kmp_initialize_fast_memory: Called from th %p\n", this_thr))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_fast_memory: Called from th %p\n"
, this_thr); }
;
2222
2223 memset(this_thr->th.th_free_lists, 0, NUM_LISTS4 * sizeof(kmp_free_list_t));
2224}
2225
2226// Free the memory in the thread free lists related to fast memory
2227// Only do this when a thread is being reaped (destroyed).
2228void __kmp_free_fast_memory(kmp_info_t *th) {
2229 // Suppose we use BGET underlying allocator, walk through its structures...
2230 int bin;
2231 thr_data_t *thr = get_thr_data(th);
2232 void **lst = NULL__null;
2233
2234 KE_TRACE(if (kmp_e_debug >= 5) { __kmp_debug_printf ("__kmp_free_fast_memory: Called T#%d\n"
, __kmp_gtid_from_thread(th)); }
2235 5, ("__kmp_free_fast_memory: Called T#%d\n", __kmp_gtid_from_thread(th)))if (kmp_e_debug >= 5) { __kmp_debug_printf ("__kmp_free_fast_memory: Called T#%d\n"
, __kmp_gtid_from_thread(th)); }
;
2236
2237 __kmp_bget_dequeue(th); // Release any queued buffers
2238
2239 // Dig through free lists and extract all allocated blocks
2240 for (bin = 0; bin < MAX_BGET_BINS(int)(sizeof(bget_bin_size) / sizeof(bufsize)); ++bin) {
2241 bfhead_t *b = thr->freelist[bin].ql.flink;
2242 while (b != &thr->freelist[bin]) {
2243 if ((kmp_uintptr_t)b->bh.bb.bthr & 1) { // the buffer is allocated address
2244 *((void **)b) =
2245 lst; // link the list (override bthr, but keep flink yet)
2246 lst = (void **)b; // push b into lst
2247 }
2248 b = b->ql.flink; // get next buffer
2249 }
2250 }
2251 while (lst != NULL__null) {
2252 void *next = *lst;
2253 KE_TRACE(10, ("__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n",if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n"
, lst, next, th, __kmp_gtid_from_thread(th)); }
2254 lst, next, th, __kmp_gtid_from_thread(th)))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n"
, lst, next, th, __kmp_gtid_from_thread(th)); }
;
2255 (*thr->relfcn)(lst);
2256#if BufStats1
2257 // count blocks to prevent problems in __kmp_finalize_bget()
2258 thr->numprel++; /* Nr of expansion block releases */
2259 thr->numpblk--; /* Total number of blocks */
2260#endif
2261 lst = (void **)next;
2262 }
2263
2264 KE_TRACE(if (kmp_e_debug >= 5) { __kmp_debug_printf ("__kmp_free_fast_memory: Freed T#%d\n"
, __kmp_gtid_from_thread(th)); }
2265 5, ("__kmp_free_fast_memory: Freed T#%d\n", __kmp_gtid_from_thread(th)))if (kmp_e_debug >= 5) { __kmp_debug_printf ("__kmp_free_fast_memory: Freed T#%d\n"
, __kmp_gtid_from_thread(th)); }
;
2266}
2267
2268#endif // USE_FAST_MEMORY