Bug Summary

File:build/source/openmp/runtime/src/kmp_runtime.cpp
Warning:line 4191, column 3
Dereference of null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name kmp_runtime.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D omp_EXPORTS -I projects/openmp/runtime/src -I /build/source/openmp/runtime/src -I include -I /build/source/llvm/include -I /build/source/openmp/runtime/src/i18n -I /build/source/openmp/runtime/src/include -I /build/source/openmp/runtime/src/thirdparty/ittnotify -D _FORTIFY_SOURCE=2 -D NDEBUG -D _GNU_SOURCE -D _REENTRANT -D _FORTIFY_SOURCE=2 -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1668078801 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -Wno-enum-constexpr-conversion -Wno-extra -Wno-pedantic -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-covered-switch-default -Wno-frame-address -Wno-strict-aliasing -Wno-stringop-truncation -Wno-switch -Wno-uninitialized -Wno-return-type-c-linkage -Wno-cast-qual -Wno-int-to-void-pointer-cast -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-11-10-135928-647445-1 -x c++ /build/source/openmp/runtime/src/kmp_runtime.cpp
1/*
2 * kmp_runtime.cpp -- KPTS runtime support library
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//
12
13#include "kmp.h"
14#include "kmp_affinity.h"
15#include "kmp_atomic.h"
16#include "kmp_environment.h"
17#include "kmp_error.h"
18#include "kmp_i18n.h"
19#include "kmp_io.h"
20#include "kmp_itt.h"
21#include "kmp_settings.h"
22#include "kmp_stats.h"
23#include "kmp_str.h"
24#include "kmp_wait_release.h"
25#include "kmp_wrapper_getpid.h"
26#include "kmp_dispatch.h"
27#if KMP_USE_HIER_SCHED0
28#include "kmp_dispatch_hier.h"
29#endif
30
31#if OMPT_SUPPORT1
32#include "ompt-specific.h"
33#endif
34#if OMPD_SUPPORT1
35#include "ompd-specific.h"
36#endif
37
38#if OMP_PROFILING_SUPPORT0
39#include "llvm/Support/TimeProfiler.h"
40static char *ProfileTraceFile = nullptr;
41#endif
42
43/* these are temporary issues to be dealt with */
44#define KMP_USE_PRCTL0 0
45
46#if KMP_OS_WINDOWS0
47#include <process.h>
48#endif
49
50#if KMP_OS_WINDOWS0
51// windows does not need include files as it doesn't use shared memory
52#else
53#include <sys/mman.h>
54#include <sys/stat.h>
55#include <fcntl.h>
56#define SHM_SIZE1024 1024
57#endif
58
59#if defined(KMP_GOMP_COMPAT)
60char const __kmp_version_alt_comp[] =
61 KMP_VERSION_PREFIX"\x00@(#) " "LLVM OMP " "alternative compiler support: yes";
62#endif /* defined(KMP_GOMP_COMPAT) */
63
64char const __kmp_version_omp_api[] =
65 KMP_VERSION_PREFIX"\x00@(#) " "LLVM OMP " "API version: 5.0 (201611)";
66
67#ifdef KMP_DEBUG1
68char const __kmp_version_lock[] =
69 KMP_VERSION_PREFIX"\x00@(#) " "LLVM OMP " "lock type: run time selectable";
70#endif /* KMP_DEBUG */
71
72#define KMP_MIN(x, y)((x) < (y) ? (x) : (y)) ((x) < (y) ? (x) : (y))
73
74/* ------------------------------------------------------------------------ */
75
76#if KMP_USE_MONITOR
77kmp_info_t __kmp_monitor;
78#endif
79
80/* Forward declarations */
81
82void __kmp_cleanup(void);
83
84static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
85 int gtid);
86static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
87 kmp_internal_control_t *new_icvs,
88 ident_t *loc);
89#if KMP_AFFINITY_SUPPORTED1
90static void __kmp_partition_places(kmp_team_t *team,
91 int update_master_only = 0);
92#endif
93static void __kmp_do_serial_initialize(void);
94void __kmp_fork_barrier(int gtid, int tid);
95void __kmp_join_barrier(int gtid);
96void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
97 kmp_internal_control_t *new_icvs, ident_t *loc);
98
99#ifdef USE_LOAD_BALANCE1
100static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
101#endif
102
103static int __kmp_expand_threads(int nNeed);
104#if KMP_OS_WINDOWS0
105static int __kmp_unregister_root_other_thread(int gtid);
106#endif
107static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
108kmp_info_t *__kmp_thread_pool_insert_pt = NULL__null;
109
110void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads,
111 int new_nthreads);
112void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads);
113
114/* Calculate the identifier of the current thread */
115/* fast (and somewhat portable) way to get unique identifier of executing
116 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
117int __kmp_get_global_thread_id() {
118 int i;
119 kmp_info_t **other_threads;
120 size_t stack_data;
121 char *stack_addr;
122 size_t stack_size;
123 char *stack_base;
124
125 KA_TRACE(if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
126 1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
127 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
128 __kmp_nth, __kmp_all_nth))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
;
129
130 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
131 a parallel region, made it return KMP_GTID_DNE to force serial_initialize
132 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
133 __kmp_init_gtid for this to work. */
134
135 if (!TCR_4(__kmp_init_gtid)(__kmp_init_gtid))
136 return KMP_GTID_DNE(-2);
137
138#ifdef KMP_TDATA_GTID1
139 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
140 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using TDATA\n"
); }
;
141 return __kmp_gtid;
142 }
143#endif
144 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
145 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using keyed TLS\n"
); }
;
146 return __kmp_gtid_get_specific();
147 }
148 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using internal alg.\n"
); }
;
149
150 stack_addr = (char *)&stack_data;
151 other_threads = __kmp_threads;
152
153 /* ATT: The code below is a source of potential bugs due to unsynchronized
154 access to __kmp_threads array. For example:
155 1. Current thread loads other_threads[i] to thr and checks it, it is
156 non-NULL.
157 2. Current thread is suspended by OS.
158 3. Another thread unregisters and finishes (debug versions of free()
159 may fill memory with something like 0xEF).
160 4. Current thread is resumed.
161 5. Current thread reads junk from *thr.
162 TODO: Fix it. --ln */
163
164 for (i = 0; i < __kmp_threads_capacity; i++) {
165
166 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i])((void *)(other_threads[i]));
167 if (!thr)
168 continue;
169
170 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize)((void *)(thr->th.th_info.ds.ds_stacksize));
171 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase)((void *)(thr->th.th_info.ds.ds_stackbase));
172
173 /* stack grows down -- search through all of the active threads */
174
175 if (stack_addr <= stack_base) {
176 size_t stack_diff = stack_base - stack_addr;
177
178 if (stack_diff <= stack_size) {
179 /* The only way we can be closer than the allocated */
180 /* stack size is if we are running on this thread. */
181 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i)if (!(__kmp_gtid_get_specific() == i)) { __kmp_debug_assert("__kmp_gtid_get_specific() == i"
, "openmp/runtime/src/kmp_runtime.cpp", 181); }
;
182 return i;
183 }
184 }
185 }
186
187 /* get specific to try and determine our gtid */
188 KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
189 ("*** __kmp_get_global_thread_id: internal alg. failed to find "if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
190 "thread, using TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
;
191 i = __kmp_gtid_get_specific();
192
193 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */
194
195 /* if we havn't been assigned a gtid, then return code */
196 if (i < 0)
197 return i;
198
199 /* dynamically updated stack window for uber threads to avoid get_specific
200 call */
201 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
202 KMP_FATAL(StackOverflow, i)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_StackOverflow, i), __kmp_msg_null
)
;
203 }
204
205 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
206 if (stack_addr > stack_base) {
207 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr)((other_threads[i]->th.th_info.ds.ds_stackbase)) = ((stack_addr
))
;
208 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
209 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
210 stack_base)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
;
211 } else {
212 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
- stack_addr))
213 stack_base - stack_addr)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
- stack_addr))
;
214 }
215
216 /* Reprint stack bounds for ubermaster since they have been refined */
217 if (__kmp_storage_map) {
218 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
219 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
220 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
221 other_threads[i]->th.th_info.ds.ds_stacksize,
222 "th_%d stack (refinement)", i);
223 }
224 return i;
225}
226
227int __kmp_get_global_thread_id_reg() {
228 int gtid;
229
230 if (!__kmp_init_serial) {
231 gtid = KMP_GTID_DNE(-2);
232 } else
233#ifdef KMP_TDATA_GTID1
234 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
235 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using TDATA\n"
); }
;
236 gtid = __kmp_gtid;
237 } else
238#endif
239 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
240 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"
); }
;
241 gtid = __kmp_gtid_get_specific();
242 } else {
243 KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); }
244 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); }
;
245 gtid = __kmp_get_global_thread_id();
246 }
247
248 /* we must be a new uber master sibling thread */
249 if (gtid == KMP_GTID_DNE(-2)) {
250 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
251 ("__kmp_get_global_thread_id_reg: Encountered new root thread. "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
252 "Registering a new gtid.\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
;
253 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
254 if (!__kmp_init_serial) {
255 __kmp_do_serial_initialize();
256 gtid = __kmp_gtid_get_specific();
257 } else {
258 gtid = __kmp_register_root(FALSE0);
259 }
260 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
261 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
262 }
263
264 KMP_DEBUG_ASSERT(gtid >= 0)if (!(gtid >= 0)) { __kmp_debug_assert("gtid >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 264); }
;
265
266 return gtid;
267}
268
269/* caller must hold forkjoin_lock */
270void __kmp_check_stack_overlap(kmp_info_t *th) {
271 int f;
272 char *stack_beg = NULL__null;
273 char *stack_end = NULL__null;
274 int gtid;
275
276 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: called\n"
); }
;
277 if (__kmp_storage_map) {
278 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
279 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
280
281 gtid = __kmp_gtid_from_thread(th);
282
283 if (gtid == KMP_GTID_MONITOR(-4)) {
284 __kmp_print_storage_map_gtid(
285 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
286 "th_%s stack (%s)", "mon",
287 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
288 } else {
289 __kmp_print_storage_map_gtid(
290 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
291 "th_%d stack (%s)", gtid,
292 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
293 }
294 }
295
296 /* No point in checking ubermaster threads since they use refinement and
297 * cannot overlap */
298 gtid = __kmp_gtid_from_thread(th);
299 if (__kmp_env_checks == TRUE(!0) && !KMP_UBER_GTID(gtid)) {
300 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); }
301 ("__kmp_check_stack_overlap: performing extensive checking\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); }
;
302 if (stack_beg == NULL__null) {
303 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
304 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
305 }
306
307 for (f = 0; f < __kmp_threads_capacity; f++) {
308 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f])((void *)(__kmp_threads[f]));
309
310 if (f_th && f_th != th) {
311 char *other_stack_end =
312 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase)((void *)(f_th->th.th_info.ds.ds_stackbase));
313 char *other_stack_beg =
314 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize));
315 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
316 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
317
318 /* Print the other stack values before the abort */
319 if (__kmp_storage_map)
320 __kmp_print_storage_map_gtid(
321 -1, other_stack_beg, other_stack_end,
322 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize)),
323 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
324
325 __kmp_fatal(KMP_MSG(StackOverlap)__kmp_msg_format(kmp_i18n_msg_StackOverlap), KMP_HNT(ChangeStackLimit)__kmp_msg_format(kmp_i18n_hnt_ChangeStackLimit),
326 __kmp_msg_null);
327 }
328 }
329 }
330 }
331 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: returning\n"
); }
;
332}
333
334/* ------------------------------------------------------------------------ */
335
336void __kmp_infinite_loop(void) {
337 static int done = FALSE0;
338
339 while (!done) {
340 KMP_YIELD(TRUE){ __kmp_x86_pause(); if (((!0)) && (((__kmp_use_yield
== 1) || (__kmp_use_yield == 2 && (((__kmp_nth) >
(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc))))))) __kmp_yield
(); }
;
341 }
342}
343
344#define MAX_MESSAGE512 512
345
346void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
347 char const *format, ...) {
348 char buffer[MAX_MESSAGE512];
349 va_list ap;
350
351 va_start(ap, format)__builtin_va_start(ap, format);
352 KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
353 p2, (unsigned long)size, format);
354 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
355 __kmp_vprintf(kmp_err, buffer, ap);
356#if KMP_PRINT_DATA_PLACEMENT
357 int node;
358 if (gtid >= 0) {
359 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
360 if (__kmp_storage_map_verbose) {
361 node = __kmp_get_host_node(p1);
362 if (node < 0) /* doesn't work, so don't try this next time */
363 __kmp_storage_map_verbose = FALSE0;
364 else {
365 char *last;
366 int lastNode;
367 int localProc = __kmp_get_cpu_from_gtid(gtid);
368
369 const int page_size = KMP_GET_PAGE_SIZE()getpagesize();
370
371 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
372 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
373 if (localProc >= 0)
374 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid,
375 localProc >> 1);
376 else
377 __kmp_printf_no_lock(" GTID %d\n", gtid);
378#if KMP_USE_PRCTL0
379 /* The more elaborate format is disabled for now because of the prctl
380 * hanging bug. */
381 do {
382 last = p1;
383 lastNode = node;
384 /* This loop collates adjacent pages with the same host node. */
385 do {
386 (char *)p1 += page_size;
387 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
388 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1,
389 lastNode);
390 } while (p1 <= p2);
391#else
392 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1,
393 (char *)p1 + (page_size - 1),
394 __kmp_get_host_node(p1));
395 if (p1 < p2) {
396 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2,
397 (char *)p2 + (page_size - 1),
398 __kmp_get_host_node(p2));
399 }
400#endif
401 }
402 }
403 } else
404 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)__kmp_i18n_catgets(kmp_i18n_str_StorageMapWarning));
405 }
406#endif /* KMP_PRINT_DATA_PLACEMENT */
407 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
408}
409
410void __kmp_warn(char const *format, ...) {
411 char buffer[MAX_MESSAGE512];
412 va_list ap;
413
414 if (__kmp_generate_warnings == kmp_warnings_off) {
415 return;
416 }
417
418 va_start(ap, format)__builtin_va_start(ap, format);
419
420 KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP warning: %s\n", format);
421 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
422 __kmp_vprintf(kmp_err, buffer, ap);
423 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
424
425 va_end(ap)__builtin_va_end(ap);
426}
427
428void __kmp_abort_process() {
429 // Later threads may stall here, but that's ok because abort() will kill them.
430 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
431
432 if (__kmp_debug_buf) {
433 __kmp_dump_debug_buffer();
434 }
435
436 if (KMP_OS_WINDOWS0) {
437 // Let other threads know of abnormal termination and prevent deadlock
438 // if abort happened during library initialization or shutdown
439 __kmp_global.g.g_abort = SIGABRT6;
440
441 /* On Windows* OS by default abort() causes pop-up error box, which stalls
442 nightly testing. Unfortunately, we cannot reliably suppress pop-up error
443 boxes. _set_abort_behavior() works well, but this function is not
444 available in VS7 (this is not problem for DLL, but it is a problem for
445 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
446 help, at least in some versions of MS C RTL.
447
448 It seems following sequence is the only way to simulate abort() and
449 avoid pop-up error box. */
450 raise(SIGABRT6);
451 _exit(3); // Just in case, if signal ignored, exit anyway.
452 } else {
453 __kmp_unregister_library();
454 abort();
455 }
456
457 __kmp_infinite_loop();
458 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
459
460} // __kmp_abort_process
461
462void __kmp_abort_thread(void) {
463 // TODO: Eliminate g_abort global variable and this function.
464 // In case of abort just call abort(), it will kill all the threads.
465 __kmp_infinite_loop();
466} // __kmp_abort_thread
467
468/* Print out the storage map for the major kmp_info_t thread data structures
469 that are allocated together. */
470
471static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
472 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
473 gtid);
474
475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
476 sizeof(kmp_desc_t), "th_%d.th_info", gtid);
477
478 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
479 sizeof(kmp_local_t), "th_%d.th_local", gtid);
480
481 __kmp_print_storage_map_gtid(
482 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
483 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);
484
485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
486 &thr->th.th_bar[bs_plain_barrier + 1],
487 sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
488 gtid);
489
490 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
491 &thr->th.th_bar[bs_forkjoin_barrier + 1],
492 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
493 gtid);
494
495#if KMP_FAST_REDUCTION_BARRIER1
496 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
497 &thr->th.th_bar[bs_reduction_barrier + 1],
498 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
499 gtid);
500#endif // KMP_FAST_REDUCTION_BARRIER
501}
502
503/* Print out the storage map for the major kmp_team_t team data structures
504 that are allocated together. */
505
506static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
507 int team_id, int num_thr) {
508 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
509 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
510 header, team_id);
511
512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
513 &team->t.t_bar[bs_last_barrier],
514 sizeof(kmp_balign_team_t) * bs_last_barrier,
515 "%s_%d.t_bar", header, team_id);
516
517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
518 &team->t.t_bar[bs_plain_barrier + 1],
519 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
520 header, team_id);
521
522 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
523 &team->t.t_bar[bs_forkjoin_barrier + 1],
524 sizeof(kmp_balign_team_t),
525 "%s_%d.t_bar[forkjoin]", header, team_id);
526
527#if KMP_FAST_REDUCTION_BARRIER1
528 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
529 &team->t.t_bar[bs_reduction_barrier + 1],
530 sizeof(kmp_balign_team_t),
531 "%s_%d.t_bar[reduction]", header, team_id);
532#endif // KMP_FAST_REDUCTION_BARRIER
533
534 __kmp_print_storage_map_gtid(
535 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
536 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);
537
538 __kmp_print_storage_map_gtid(
539 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
540 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);
541
542 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
543 &team->t.t_disp_buffer[num_disp_buff],
544 sizeof(dispatch_shared_info_t) * num_disp_buff,
545 "%s_%d.t_disp_buffer", header, team_id);
546}
547
548static void __kmp_init_allocator() {
549 __kmp_init_memkind();
550 __kmp_init_target_mem();
551}
552static void __kmp_fini_allocator() { __kmp_fini_memkind(); }
553
554/* ------------------------------------------------------------------------ */
555
556#if KMP_DYNAMIC_LIB1
557#if KMP_OS_WINDOWS0
558
559BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
560 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
561
562 switch (fdwReason) {
563
564 case DLL_PROCESS_ATTACH:
565 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_ATTACH\n"
); }
;
566
567 return TRUE(!0);
568
569 case DLL_PROCESS_DETACH:
570 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_DETACH T#%d\n"
, __kmp_gtid_get_specific()); }
;
571
572 // According to Windows* documentation for DllMain entry point:
573 // for DLL_PROCESS_DETACH, lpReserved is used for telling the difference:
574 // lpReserved == NULL when FreeLibrary() is called,
575 // lpReserved != NULL when the process is terminated.
576 // When FreeLibrary() is called, worker threads remain alive. So the
577 // runtime's state is consistent and executing proper shutdown is OK.
578 // When the process is terminated, worker threads have exited or been
579 // forcefully terminated by the OS and only the shutdown thread remains.
580 // This can leave the runtime in an inconsistent state.
581 // Hence, only attempt proper cleanup when FreeLibrary() is called.
582 // Otherwise, rely on OS to reclaim resources.
583 if (lpReserved == NULL__null)
584 __kmp_internal_end_library(__kmp_gtid_get_specific());
585
586 return TRUE(!0);
587
588 case DLL_THREAD_ATTACH:
589 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_ATTACH\n"
); }
;
590
591 /* if we want to register new siblings all the time here call
592 * __kmp_get_gtid(); */
593 return TRUE(!0);
594
595 case DLL_THREAD_DETACH:
596 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_DETACH T#%d\n"
, __kmp_gtid_get_specific()); }
;
597
598 __kmp_internal_end_thread(__kmp_gtid_get_specific());
599 return TRUE(!0);
600 }
601
602 return TRUE(!0);
603}
604
605#endif /* KMP_OS_WINDOWS */
606#endif /* KMP_DYNAMIC_LIB */
607
608/* __kmp_parallel_deo -- Wait until it's our turn. */
609void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
610 int gtid = *gtid_ref;
611#ifdef BUILD_PARALLEL_ORDERED1
612 kmp_team_t *team = __kmp_team_from_gtid(gtid);
613#endif /* BUILD_PARALLEL_ORDERED */
614
615 if (__kmp_env_consistency_check) {
616 if (__kmp_threads[gtid]->th.th_root->r.r_active)
617#if KMP_USE_DYNAMIC_LOCK1
618 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null, 0);
619#else
620 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null);
621#endif
622 }
623#ifdef BUILD_PARALLEL_ORDERED1
624 if (!team->t.t_serialized) {
625 KMP_MB();
626 KMP_WAIT__kmp_wait_4(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ__kmp_eq_4,
627 NULL__null);
628 KMP_MB();
629 }
630#endif /* BUILD_PARALLEL_ORDERED */
631}
632
633/* __kmp_parallel_dxo -- Signal the next task. */
634void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
635 int gtid = *gtid_ref;
636#ifdef BUILD_PARALLEL_ORDERED1
637 int tid = __kmp_tid_from_gtid(gtid);
638 kmp_team_t *team = __kmp_team_from_gtid(gtid);
639#endif /* BUILD_PARALLEL_ORDERED */
640
641 if (__kmp_env_consistency_check) {
642 if (__kmp_threads[gtid]->th.th_root->r.r_active)
643 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
644 }
645#ifdef BUILD_PARALLEL_ORDERED1
646 if (!team->t.t_serialized) {
647 KMP_MB(); /* Flush all pending memory write invalidates. */
648
649 /* use the tid of the next thread in this team */
650 /* TODO replace with general release procedure */
651 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
652
653 KMP_MB(); /* Flush all pending memory write invalidates. */
654 }
655#endif /* BUILD_PARALLEL_ORDERED */
656}
657
658/* ------------------------------------------------------------------------ */
659/* The BARRIER for a SINGLE process section is always explicit */
660
661int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
662 int status;
663 kmp_info_t *th;
664 kmp_team_t *team;
665
666 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
667 __kmp_parallel_initialize();
668 __kmp_resume_if_soft_paused();
669
670 th = __kmp_threads[gtid];
671 team = th->th.th_team;
672 status = 0;
673
674 th->th.th_ident = id_ref;
675
676 if (team->t.t_serialized) {
677 status = 1;
678 } else {
679 kmp_int32 old_this = th->th.th_local.this_construct;
680
681 ++th->th.th_local.this_construct;
682 /* try to set team count to thread count--success means thread got the
683 single block */
684 /* TODO: Should this be acquire or release? */
685 if (team->t.t_construct == old_this) {
686 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
687 th->th.th_local.this_construct);
688 }
689#if USE_ITT_BUILD1
690 if (__itt_metadata_add_ptr__kmp_itt_metadata_add_ptr__3_0 && __kmp_forkjoin_frames_mode == 3 &&
691 KMP_MASTER_GTID(gtid)(0 == __kmp_tid_from_gtid((gtid))) && th->th.th_teams_microtask == NULL__null &&
692 team->t.t_active_level == 1) {
693 // Only report metadata by primary thread of active team at level 1
694 __kmp_itt_metadata_single(id_ref);
695 }
696#endif /* USE_ITT_BUILD */
697 }
698
699 if (__kmp_env_consistency_check) {
700 if (status && push_ws) {
701 __kmp_push_workshare(gtid, ct_psingle, id_ref);
702 } else {
703 __kmp_check_workshare(gtid, ct_psingle, id_ref);
704 }
705 }
706#if USE_ITT_BUILD1
707 if (status) {
708 __kmp_itt_single_start(gtid);
709 }
710#endif /* USE_ITT_BUILD */
711 return status;
712}
713
714void __kmp_exit_single(int gtid) {
715#if USE_ITT_BUILD1
716 __kmp_itt_single_end(gtid);
717#endif /* USE_ITT_BUILD */
718 if (__kmp_env_consistency_check)
719 __kmp_pop_workshare(gtid, ct_psingle, NULL__null);
720}
721
722/* determine if we can go parallel or must use a serialized parallel region and
723 * how many threads we can use
724 * set_nproc is the number of threads requested for the team
725 * returns 0 if we should serialize or only use one thread,
726 * otherwise the number of threads to use
727 * The forkjoin lock is held by the caller. */
728static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
729 int master_tid, int set_nthreads,
730 int enter_teams) {
731 int capacity;
732 int new_nthreads;
733 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 733); }
;
734 KMP_DEBUG_ASSERT(root && parent_team)if (!(root && parent_team)) { __kmp_debug_assert("root && parent_team"
, "openmp/runtime/src/kmp_runtime.cpp", 734); }
;
735 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];
736
737 // If dyn-var is set, dynamically adjust the number of desired threads,
738 // according to the method specified by dynamic_mode.
739 new_nthreads = set_nthreads;
740 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
) {
741 ;
742 }
743#ifdef USE_LOAD_BALANCE1
744 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
745 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
746 if (new_nthreads == 1) {
747 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
748 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
749 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
;
750 return 1;
751 }
752 if (new_nthreads < set_nthreads) {
753 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
754 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
755 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
756 }
757 }
758#endif /* USE_LOAD_BALANCE */
759 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
760 new_nthreads = __kmp_avail_proc - __kmp_nth +
761 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
762 if (new_nthreads <= 1) {
763 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
764 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
765 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
;
766 return 1;
767 }
768 if (new_nthreads < set_nthreads) {
769 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
770 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
771 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
772 } else {
773 new_nthreads = set_nthreads;
774 }
775 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
776 if (set_nthreads > 2) {
777 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
778 new_nthreads = (new_nthreads % set_nthreads) + 1;
779 if (new_nthreads == 1) {
780 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
781 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
782 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
;
783 return 1;
784 }
785 if (new_nthreads < set_nthreads) {
786 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
787 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
788 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
789 }
790 }
791 } else {
792 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_runtime.cpp"
, 792); }
;
793 }
794
795 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
796 if (__kmp_nth + new_nthreads -
797 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
798 __kmp_max_nth) {
799 int tl_nthreads = __kmp_max_nth - __kmp_nth +
800 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
801 if (tl_nthreads <= 0) {
802 tl_nthreads = 1;
803 }
804
805 // If dyn-var is false, emit a 1-time warning.
806 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
807 __kmp_reserve_warn = 1;
808 __kmp_msg(kmp_ms_warning,
809 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
)
,
810 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
811 }
812 if (tl_nthreads == 1) {
813 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
814 "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
815 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
;
816 return 1;
817 }
818 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
819 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
820 master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
;
821 new_nthreads = tl_nthreads;
822 }
823
824 // Respect OMP_THREAD_LIMIT
825 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
826 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
827 if (cg_nthreads + new_nthreads -
828 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
829 max_cg_threads) {
830 int tl_nthreads = max_cg_threads - cg_nthreads +
831 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
832 if (tl_nthreads <= 0) {
833 tl_nthreads = 1;
834 }
835
836 // If dyn-var is false, emit a 1-time warning.
837 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
838 __kmp_reserve_warn = 1;
839 __kmp_msg(kmp_ms_warning,
840 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
)
,
841 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
842 }
843 if (tl_nthreads == 1) {
844 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
845 "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
846 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
;
847 return 1;
848 }
849 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
850 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
851 master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
;
852 new_nthreads = tl_nthreads;
853 }
854
855 // Check if the threads array is large enough, or needs expanding.
856 // See comment in __kmp_register_root() about the adjustment if
857 // __kmp_threads[0] == NULL.
858 capacity = __kmp_threads_capacity;
859 if (TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
860 --capacity;
861 }
862 // If it is not for initializing the hidden helper team, we need to take
863 // __kmp_hidden_helper_threads_num out of the capacity because it is included
864 // in __kmp_threads_capacity.
865 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
866 capacity -= __kmp_hidden_helper_threads_num;
867 }
868 if (__kmp_nth + new_nthreads -
869 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
870 capacity) {
871 // Expand the threads array.
872 int slotsRequired = __kmp_nth + new_nthreads -
873 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
874 capacity;
875 int slotsAdded = __kmp_expand_threads(slotsRequired);
876 if (slotsAdded < slotsRequired) {
877 // The threads array was not expanded enough.
878 new_nthreads -= (slotsRequired - slotsAdded);
879 KMP_ASSERT(new_nthreads >= 1)if (!(new_nthreads >= 1)) { __kmp_debug_assert("new_nthreads >= 1"
, "openmp/runtime/src/kmp_runtime.cpp", 879); }
;
880
881 // If dyn-var is false, emit a 1-time warning.
882 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
883 __kmp_reserve_warn = 1;
884 if (__kmp_tp_cached) {
885 __kmp_msg(kmp_ms_warning,
886 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
)
,
887 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
)
,
888 KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
889 } else {
890 __kmp_msg(kmp_ms_warning,
891 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
)
,
892 KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads), __kmp_msg_null);
893 }
894 }
895 }
896 }
897
898#ifdef KMP_DEBUG1
899 if (new_nthreads == 1) {
900 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
901 ("__kmp_reserve_threads: T#%d serializing team after reclaiming "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
902 "dead roots and rechecking; requested %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
903 __kmp_get_gtid(), set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
;
904 } else {
905 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
906 " %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
907 __kmp_get_gtid(), new_nthreads, set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
;
908 }
909#endif // KMP_DEBUG
910 return new_nthreads;
911}
912
913/* Allocate threads from the thread pool and assign them to the new team. We are
914 assured that there are enough threads available, because we checked on that
915 earlier within critical section forkjoin */
916static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
917 kmp_info_t *master_th, int master_gtid,
918 int fork_teams_workers) {
919 int i;
920 int use_hot_team;
921
922 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_fork_team_threads: new_nprocs = %d\n"
, team->t.t_nproc); }
;
923 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid())if (!(master_gtid == __kmp_get_global_thread_id())) { __kmp_debug_assert
("master_gtid == __kmp_get_global_thread_id()", "openmp/runtime/src/kmp_runtime.cpp"
, 923); }
;
924 KMP_MB();
925
926 /* first, let's setup the primary thread */
927 master_th->th.th_info.ds.ds_tid = 0;
928 master_th->th.th_team = team;
929 master_th->th.th_team_nproc = team->t.t_nproc;
930 master_th->th.th_team_master = master_th;
931 master_th->th.th_team_serialized = FALSE0;
932 master_th->th.th_dispatch = &team->t.t_dispatch[0];
933
934/* make sure we are not the optimized hot team */
935#if KMP_NESTED_HOT_TEAMS1
936 use_hot_team = 0;
937 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
938 if (hot_teams) { // hot teams array is not allocated if
939 // KMP_HOT_TEAMS_MAX_LEVEL=0
940 int level = team->t.t_active_level - 1; // index in array of hot teams
941 if (master_th->th.th_teams_microtask) { // are we inside the teams?
942 if (master_th->th.th_teams_size.nteams > 1) {
943 ++level; // level was not increased in teams construct for
944 // team_of_masters
945 }
946 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
947 master_th->th.th_teams_level == team->t.t_level) {
948 ++level; // level was not increased in teams construct for
949 // team_of_workers before the parallel
950 } // team->t.t_level will be increased inside parallel
951 }
952 if (level < __kmp_hot_teams_max_level) {
953 if (hot_teams[level].hot_team) {
954 // hot team has already been allocated for given level
955 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team)if (!(hot_teams[level].hot_team == team)) { __kmp_debug_assert
("hot_teams[level].hot_team == team", "openmp/runtime/src/kmp_runtime.cpp"
, 955); }
;
956 use_hot_team = 1; // the team is ready to use
957 } else {
958 use_hot_team = 0; // AC: threads are not allocated yet
959 hot_teams[level].hot_team = team; // remember new hot team
960 hot_teams[level].hot_team_nth = team->t.t_nproc;
961 }
962 } else {
963 use_hot_team = 0;
964 }
965 }
966#else
967 use_hot_team = team == root->r.r_hot_team;
968#endif
969 if (!use_hot_team) {
970
971 /* install the primary thread */
972 team->t.t_threads[0] = master_th;
973 __kmp_initialize_info(master_th, team, 0, master_gtid);
974
975 /* now, install the worker threads */
976 for (i = 1; i < team->t.t_nproc; i++) {
977
978 /* fork or reallocate a new thread and install it in team */
979 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
980 team->t.t_threads[i] = thr;
981 KMP_DEBUG_ASSERT(thr)if (!(thr)) { __kmp_debug_assert("thr", "openmp/runtime/src/kmp_runtime.cpp"
, 981); }
;
982 KMP_DEBUG_ASSERT(thr->th.th_team == team)if (!(thr->th.th_team == team)) { __kmp_debug_assert("thr->th.th_team == team"
, "openmp/runtime/src/kmp_runtime.cpp", 982); }
;
983 /* align team and thread arrived states */
984 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
985 "T#%d(%d:%d) join =%llu, plain=%llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
986 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
987 __kmp_gtid_from_tid(i, team), team->t.t_id, i,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
988 team->t.t_bar[bs_forkjoin_barrier].b_arrived,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
989 team->t.t_bar[bs_plain_barrier].b_arrived))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
;
990 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
991 thr->th.th_teams_level = master_th->th.th_teams_level;
992 thr->th.th_teams_size = master_th->th.th_teams_size;
993 { // Initialize threads' barrier data.
994 int b;
995 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
996 for (b = 0; b < bs_last_barrier; ++b) {
997 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
998 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 998); }
;
999#if USE_DEBUGGER0
1000 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1001#endif
1002 }
1003 }
1004 }
1005
1006#if KMP_AFFINITY_SUPPORTED1
1007 // Do not partition the places list for teams construct workers who
1008 // haven't actually been forked to do real work yet. This partitioning
1009 // will take place in the parallel region nested within the teams construct.
1010 if (!fork_teams_workers) {
1011 __kmp_partition_places(team);
1012 }
1013#endif
1014 }
1015
1016 if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
1017 for (i = 0; i < team->t.t_nproc; i++) {
1018 kmp_info_t *thr = team->t.t_threads[i];
1019 if (thr->th.th_prev_num_threads != team->t.t_nproc ||
1020 thr->th.th_prev_level != team->t.t_level) {
1021 team->t.t_display_affinity = 1;
1022 break;
1023 }
1024 }
1025 }
1026
1027 KMP_MB();
1028}
1029
1030#if KMP_ARCH_X860 || KMP_ARCH_X86_641
1031// Propagate any changes to the floating point control registers out to the team
1032// We try to avoid unnecessary writes to the relevant cache line in the team
1033// structure, so we don't make changes unless they are needed.
1034inline static void propagateFPControl(kmp_team_t *team) {
1035 if (__kmp_inherit_fp_control) {
1036 kmp_int16 x87_fpu_control_word;
1037 kmp_uint32 mxcsr;
1038
1039 // Get primary thread's values of FPU control flags (both X87 and vector)
1040 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1041 __kmp_store_mxcsr(&mxcsr);
1042 mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
1043
1044 // There is no point looking at t_fp_control_saved here.
1045 // If it is TRUE, we still have to update the values if they are different
1046 // from those we now have. If it is FALSE we didn't save anything yet, but
1047 // our objective is the same. We have to ensure that the values in the team
1048 // are the same as those we have.
1049 // So, this code achieves what we need whether or not t_fp_control_saved is
1050 // true. By checking whether the value needs updating we avoid unnecessary
1051 // writes that would put the cache-line into a written state, causing all
1052 // threads in the team to have to read it again.
1053 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word)if ((team->t.t_x87_fpu_control_word) != (x87_fpu_control_word
)) (team->t.t_x87_fpu_control_word) = (x87_fpu_control_word
)
;
1054 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr)if ((team->t.t_mxcsr) != (mxcsr)) (team->t.t_mxcsr) = (
mxcsr)
;
1055 // Although we don't use this value, other code in the runtime wants to know
1056 // whether it should restore them. So we must ensure it is correct.
1057 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE)if ((team->t.t_fp_control_saved) != ((!0))) (team->t.t_fp_control_saved
) = ((!0))
;
1058 } else {
1059 // Similarly here. Don't write to this cache-line in the team structure
1060 // unless we have to.
1061 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE)if ((team->t.t_fp_control_saved) != (0)) (team->t.t_fp_control_saved
) = (0)
;
1062 }
1063}
1064
1065// Do the opposite, setting the hardware registers to the updated values from
1066// the team.
1067inline static void updateHWFPControl(kmp_team_t *team) {
1068 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1069 // Only reset the fp control regs if they have been changed in the team.
1070 // the parallel region that we are exiting.
1071 kmp_int16 x87_fpu_control_word;
1072 kmp_uint32 mxcsr;
1073 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1074 __kmp_store_mxcsr(&mxcsr);
1075 mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
1076
1077 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1078 __kmp_clear_x87_fpu_status_word();
1079 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1080 }
1081
1082 if (team->t.t_mxcsr != mxcsr) {
1083 __kmp_load_mxcsr(&team->t.t_mxcsr);
1084 }
1085 }
1086}
1087#else
1088#define propagateFPControl(x) ((void)0)
1089#define updateHWFPControl(x) ((void)0)
1090#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1091
1092static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1093 int realloc); // forward declaration
1094
1095/* Run a parallel region that has been serialized, so runs only in a team of the
1096 single primary thread. */
1097void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
1098 kmp_info_t *this_thr;
1099 kmp_team_t *serial_team;
1100
1101 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: called by T#%d\n"
, global_tid); }
;
1102
1103 /* Skip all this code for autopar serialized loops since it results in
1104 unacceptable overhead */
1105 if (loc != NULL__null && (loc->flags & KMP_IDENT_AUTOPAR))
1106 return;
1107
1108 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
1109 __kmp_parallel_initialize();
1110 __kmp_resume_if_soft_paused();
1111
1112 this_thr = __kmp_threads[global_tid];
1113 serial_team = this_thr->th.th_serial_team;
1114
1115 /* utilize the serialized team held by this thread */
1116 KMP_DEBUG_ASSERT(serial_team)if (!(serial_team)) { __kmp_debug_assert("serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1116); }
;
1117 KMP_MB();
1118
1119 if (__kmp_tasking_mode != tskm_immediate_exec) {
1120 KMP_DEBUG_ASSERT(if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1122); }
1121 this_thr->th.th_task_team ==if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1122); }
1122 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state])if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1122); }
;
1123 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==if (!(serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null)) { __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 1124); }
1124 NULL)if (!(serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null)) { __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 1124); }
;
1125 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
1126 "team %p, new task_team = NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
1127 global_tid, this_thr->th.th_task_team, this_thr->th.th_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
;
1128 this_thr->th.th_task_team = NULL__null;
1129 }
1130
1131 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1132 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1133 proc_bind = proc_bind_false;
1134 } else if (proc_bind == proc_bind_default) {
1135 // No proc_bind clause was specified, so use the current value
1136 // of proc-bind-var for this parallel region.
1137 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1138 }
1139 // Reset for next parallel region
1140 this_thr->th.th_set_proc_bind = proc_bind_default;
1141
1142#if OMPT_SUPPORT1
1143 ompt_data_t ompt_parallel_data = ompt_data_none{0};
1144 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid)__ompt_load_return_address(global_tid);
1145 if (ompt_enabled.enabled &&
1146 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1147
1148 ompt_task_info_t *parent_task_info;
1149 parent_task_info = OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info));
1150
1151 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
1152 if (ompt_enabled.ompt_callback_parallel_begin) {
1153 int team_size = 1;
1154
1155 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
1156 &(parent_task_info->task_data), &(parent_task_info->frame),
1157 &ompt_parallel_data, team_size,
1158 ompt_parallel_invoker_program | ompt_parallel_team, codeptr);
1159 }
1160 }
1161#endif // OMPT_SUPPORT
1162
1163 if (this_thr->th.th_team != serial_team) {
1164 // Nested level will be an index in the nested nthreads array
1165 int level = this_thr->th.th_team->t.t_level;
1166
1167 if (serial_team->t.t_serialized) {
1168 /* this serial team was already used
1169 TODO increase performance by making this locks more specific */
1170 kmp_team_t *new_team;
1171
1172 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1173
1174 new_team =
1175 __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1176#if OMPT_SUPPORT1
1177 ompt_parallel_data,
1178#endif
1179 proc_bind, &this_thr->th.th_current_task->td_icvs,
1180 0 USE_NESTED_HOT_ARG(NULL), __null);
1181 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1182 KMP_ASSERT(new_team)if (!(new_team)) { __kmp_debug_assert("new_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1182); }
;
1183
1184 /* setup new serialized team and install it */
1185 new_team->t.t_threads[0] = this_thr;
1186 new_team->t.t_parent = this_thr->th.th_team;
1187 serial_team = new_team;
1188 this_thr->th.th_serial_team = serial_team;
1189
1190 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1191 10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1192 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1193 global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
;
1194
1195 /* TODO the above breaks the requirement that if we run out of resources,
1196 then we can still guarantee that serialized teams are ok, since we may
1197 need to allocate a new one */
1198 } else {
1199 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1200 10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1201 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1202 global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
;
1203 }
1204
1205 /* we have to initialize this serial team */
1206 KMP_DEBUG_ASSERT(serial_team->t.t_threads)if (!(serial_team->t.t_threads)) { __kmp_debug_assert("serial_team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 1206); }
;
1207 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)if (!(serial_team->t.t_threads[0] == this_thr)) { __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "openmp/runtime/src/kmp_runtime.cpp"
, 1207); }
;
1208 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team)if (!(this_thr->th.th_team != serial_team)) { __kmp_debug_assert
("this_thr->th.th_team != serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1208); }
;
1209 serial_team->t.t_ident = loc;
1210 serial_team->t.t_serialized = 1;
1211 serial_team->t.t_nproc = 1;
1212 serial_team->t.t_parent = this_thr->th.th_team;
1213 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1214 this_thr->th.th_team = serial_team;
1215 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1216
1217 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d curtask=%p\n", global_tid,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); }
1218 this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); }
;
1219 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1)if (!(this_thr->th.th_current_task->td_flags.executing ==
1)) { __kmp_debug_assert("this_thr->th.th_current_task->td_flags.executing == 1"
, "openmp/runtime/src/kmp_runtime.cpp", 1219); }
;
1220 this_thr->th.th_current_task->td_flags.executing = 0;
1221
1222 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1223
1224 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
1225 implicit task for each serialized task represented by
1226 team->t.t_serialized? */
1227 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1228 &this_thr->th.th_current_task->td_parent->td_icvs);
1229
1230 // Thread value exists in the nested nthreads array for the next nested
1231 // level
1232 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1233 this_thr->th.th_current_task->td_icvs.nproc =
1234 __kmp_nested_nth.nth[level + 1];
1235 }
1236
1237 if (__kmp_nested_proc_bind.used &&
1238 (level + 1 < __kmp_nested_proc_bind.used)) {
1239 this_thr->th.th_current_task->td_icvs.proc_bind =
1240 __kmp_nested_proc_bind.bind_types[level + 1];
1241 }
1242
1243#if USE_DEBUGGER0
1244 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
1245#endif
1246 this_thr->th.th_info.ds.ds_tid = 0;
1247
1248 /* set thread cache values */
1249 this_thr->th.th_team_nproc = 1;
1250 this_thr->th.th_team_master = this_thr;
1251 this_thr->th.th_team_serialized = 1;
1252
1253 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1254 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1255 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save
1256
1257 propagateFPControl(serial_team);
1258
1259 /* check if we need to allocate dispatch buffers stack */
1260 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)if (!(serial_team->t.t_dispatch)) { __kmp_debug_assert("serial_team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 1260); }
;
1261 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1262 serial_team->t.t_dispatch->th_disp_buffer =
1263 (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1264)
1264 sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1264)
;
1265 }
1266 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1267
1268 KMP_MB();
1269
1270 } else {
1271 /* this serialized team is already being used,
1272 * that's fine, just add another nested level */
1273 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team)if (!(this_thr->th.th_team == serial_team)) { __kmp_debug_assert
("this_thr->th.th_team == serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1273); }
;
1274 KMP_DEBUG_ASSERT(serial_team->t.t_threads)if (!(serial_team->t.t_threads)) { __kmp_debug_assert("serial_team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 1274); }
;
1275 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)if (!(serial_team->t.t_threads[0] == this_thr)) { __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "openmp/runtime/src/kmp_runtime.cpp"
, 1275); }
;
1276 ++serial_team->t.t_serialized;
1277 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1278
1279 // Nested level will be an index in the nested nthreads array
1280 int level = this_thr->th.th_team->t.t_level;
1281 // Thread value exists in the nested nthreads array for the next nested
1282 // level
1283 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1284 this_thr->th.th_current_task->td_icvs.nproc =
1285 __kmp_nested_nth.nth[level + 1];
1286 }
1287 serial_team->t.t_level++;
1288 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
1289 "of serial team %p to %d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
1290 global_tid, serial_team, serial_team->t.t_level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
;
1291
1292 /* allocate/push dispatch buffers stack */
1293 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)if (!(serial_team->t.t_dispatch)) { __kmp_debug_assert("serial_team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 1293); }
;
1294 {
1295 dispatch_private_info_t *disp_buffer =
1296 (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1297)
1297 sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1297)
;
1298 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1299 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1300 }
1301 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1302
1303 KMP_MB();
1304 }
1305 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq)if ((serial_team->t.t_cancel_request) != (cancel_noreq)) (
serial_team->t.t_cancel_request) = (cancel_noreq)
;
1306
1307 // Perform the display affinity functionality for
1308 // serialized parallel regions
1309 if (__kmp_display_affinity) {
1310 if (this_thr->th.th_prev_level != serial_team->t.t_level ||
1311 this_thr->th.th_prev_num_threads != 1) {
1312 // NULL means use the affinity-format-var ICV
1313 __kmp_aux_display_affinity(global_tid, NULL__null);
1314 this_thr->th.th_prev_level = serial_team->t.t_level;
1315 this_thr->th.th_prev_num_threads = 1;
1316 }
1317 }
1318
1319 if (__kmp_env_consistency_check)
1320 __kmp_push_parallel(global_tid, NULL__null);
1321#if OMPT_SUPPORT1
1322 serial_team->t.ompt_team_info.master_return_address = codeptr;
1323 if (ompt_enabled.enabled &&
1324 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1325 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame.ptr =
1326 OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
1327
1328 ompt_lw_taskteam_t lw_taskteam;
1329 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1330 &ompt_parallel_data, codeptr);
1331
1332 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1333 // don't use lw_taskteam after linking. content was swaped
1334
1335 /* OMPT implicit task begin */
1336 if (ompt_enabled.ompt_callback_implicit_task) {
1337 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1338 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr)(&(this_thr->th.th_team->t.ompt_team_info.parallel_data
))
,
1339 OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
))
, 1, __kmp_tid_from_gtid(global_tid),
1340 ompt_task_implicit); // TODO: Can this be ompt_task_initial?
1341 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->thread_num =
1342 __kmp_tid_from_gtid(global_tid);
1343 }
1344
1345 /* OMPT state */
1346 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1347 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame.ptr =
1348 OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
1349 }
1350#endif
1351}
1352
1353/* most of the work for a fork */
1354/* return true if we really went parallel, false if serialized */
1355int __kmp_fork_call(ident_t *loc, int gtid,
1356 enum fork_context_e call_context, // Intel, GNU, ...
1357 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1358 kmp_va_list ap) {
1359 void **argv;
1360 int i;
1361 int master_tid;
1362 int master_this_cons;
1363 kmp_team_t *team;
1364 kmp_team_t *parent_team;
1365 kmp_info_t *master_th;
1366 kmp_root_t *root;
1367 int nthreads;
1368 int master_active;
1369 int master_set_numthreads;
1370 int level;
1371 int active_level;
1372 int teams_level;
1373#if KMP_NESTED_HOT_TEAMS1
1374 kmp_hot_team_ptr_t **p_hot_teams;
1375#endif
1376 { // KMP_TIME_BLOCK
1377 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call)((void)0);
1378 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc)((void)0);
1379
1380 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: enter T#%d\n"
, gtid); }
;
1381 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL__null) {
1382 /* Some systems prefer the stack for the root thread(s) to start with */
1383 /* some gap from the parent stack to prevent false sharing. */
1384 void *dummy = KMP_ALLOCA(__kmp_stkpadding)__builtin_alloca (__kmp_stkpadding);
1385 /* These 2 lines below are so this does not get optimized out */
1386 if (__kmp_stkpadding > KMP_MAX_STKPADDING(2 * 1024 * 1024))
1387 __kmp_stkpadding += (short)((kmp_int64)dummy);
1388 }
1389
1390 /* initialize if needed */
1391 KMP_DEBUG_ASSERT(if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 1392); }
1392 __kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 1392); }
; // AC: potentially unsafe, not in sync with shutdown
1393 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
1394 __kmp_parallel_initialize();
1395 __kmp_resume_if_soft_paused();
1396
1397 /* setup current data */
1398 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
1399 // shutdown
1400 parent_team = master_th->th.th_team;
1401 master_tid = master_th->th.th_info.ds.ds_tid;
1402 master_this_cons = master_th->th.th_local.this_construct;
1403 root = master_th->th.th_root;
1404 master_active = root->r.r_active;
1405 master_set_numthreads = master_th->th.th_set_nproc;
1406
1407#if OMPT_SUPPORT1
1408 ompt_data_t ompt_parallel_data = ompt_data_none{0};
1409 ompt_data_t *parent_task_data;
1410 ompt_frame_t *ompt_frame;
1411 ompt_data_t *implicit_task_data;
1412 void *return_address = NULL__null;
1413
1414 if (ompt_enabled.enabled) {
1415 __ompt_get_task_info_internal(0, NULL__null, &parent_task_data, &ompt_frame,
1416 NULL__null, NULL__null);
1417 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid);
1418 }
1419#endif
1420
1421 // Assign affinity to root thread if it hasn't happened yet
1422 __kmp_assign_root_init_mask();
1423
1424 // Nested level will be an index in the nested nthreads array
1425 level = parent_team->t.t_level;
1426 // used to launch non-serial teams even if nested is not allowed
1427 active_level = parent_team->t.t_active_level;
1428 // needed to check nesting inside the teams
1429 teams_level = master_th->th.th_teams_level;
1430#if KMP_NESTED_HOT_TEAMS1
1431 p_hot_teams = &master_th->th.th_hot_teams;
1432 if (*p_hot_teams == NULL__null && __kmp_hot_teams_max_level > 0) {
1433 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "openmp/runtime/src/kmp_runtime.cpp", 1434)
1434 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level)___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "openmp/runtime/src/kmp_runtime.cpp", 1434)
;
1435 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1436 // it is either actual or not needed (when active_level > 0)
1437 (*p_hot_teams)[0].hot_team_nth = 1;
1438 }
1439#endif
1440
1441#if OMPT_SUPPORT1
1442 if (ompt_enabled.enabled) {
1443 if (ompt_enabled.ompt_callback_parallel_begin) {
1444 int team_size = master_set_numthreads
1445 ? master_set_numthreads
1446 : get__nproc_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
;
1447 int flags = OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
|
1448 ((microtask == (microtask_t)__kmp_teams_master)
1449 ? ompt_parallel_league
1450 : ompt_parallel_team);
1451 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
1452 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, flags,
1453 return_address);
1454 }
1455 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1456 }
1457#endif
1458
1459 master_th->th.th_ident = loc;
1460
1461 if (master_th->th.th_teams_microtask && ap &&
1462 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1463 // AC: This is start of parallel that is nested inside teams construct.
1464 // The team is actual (hot), all workers are ready at the fork barrier.
1465 // No lock needed to initialize the team a bit, then free workers.
1466 parent_team->t.t_ident = loc;
1467 __kmp_alloc_argv_entries(argc, parent_team, TRUE(!0));
1468 parent_team->t.t_argc = argc;
1469 argv = (void **)parent_team->t.t_argv;
1470 for (i = argc - 1; i >= 0; --i)
1471 *argv++ = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
1472 // Increment our nested depth levels, but not increase the serialization
1473 if (parent_team == master_th->th.th_serial_team) {
1474 // AC: we are in serialized parallel
1475 __kmpc_serialized_parallel(loc, gtid);
1476 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1)if (!(parent_team->t.t_serialized > 1)) { __kmp_debug_assert
("parent_team->t.t_serialized > 1", "openmp/runtime/src/kmp_runtime.cpp"
, 1476); }
;
1477
1478 if (call_context == fork_context_gnu) {
1479 // AC: need to decrement t_serialized for enquiry functions to work
1480 // correctly, will restore at join time
1481 parent_team->t.t_serialized--;
1482 return TRUE(!0);
1483 }
1484
1485#if OMPD_SUPPORT1
1486 parent_team->t.t_pkfn = microtask;
1487#endif
1488
1489#if OMPT_SUPPORT1
1490 void *dummy;
1491 void **exit_frame_p;
1492
1493 ompt_lw_taskteam_t lw_taskteam;
1494
1495 if (ompt_enabled.enabled) {
1496 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1497 &ompt_parallel_data, return_address);
1498 exit_frame_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);
1499
1500 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1501 // don't use lw_taskteam after linking. content was swaped
1502
1503 /* OMPT implicit task begin */
1504 implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
;
1505 if (ompt_enabled.ompt_callback_implicit_task) {
1506 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num =
1507 __kmp_tid_from_gtid(gtid);
1508 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1509 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1510 implicit_task_data, 1,
1511 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
1512 }
1513
1514 /* OMPT state */
1515 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1516 } else {
1517 exit_frame_p = &dummy;
1518 }
1519#endif
1520 // AC: need to decrement t_serialized for enquiry functions to work
1521 // correctly, will restore at join time
1522 parent_team->t.t_serialized--;
1523
1524 {
1525 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1526 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1527 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1528#if OMPT_SUPPORT1
1529 ,
1530 exit_frame_p
1531#endif
1532 );
1533 }
1534
1535#if OMPT_SUPPORT1
1536 if (ompt_enabled.enabled) {
1537 *exit_frame_p = NULL__null;
1538 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->frame.exit_frame = ompt_data_none{0};
1539 if (ompt_enabled.ompt_callback_implicit_task) {
1540 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1541 ompt_scope_end, NULL__null, implicit_task_data, 1,
1542 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
1543 }
1544 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
;
1545 __ompt_lw_taskteam_unlink(master_th);
1546 if (ompt_enabled.ompt_callback_parallel_end) {
1547 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1548 &ompt_parallel_data, OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
,
1549 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
| ompt_parallel_team,
1550 return_address);
1551 }
1552 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1553 }
1554#endif
1555 return TRUE(!0);
1556 }
1557
1558 parent_team->t.t_pkfn = microtask;
1559 parent_team->t.t_invoke = invoker;
1560 KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
)
;
1561 parent_team->t.t_active_level++;
1562 parent_team->t.t_level++;
1563 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save
1564
1565#if OMPT_SUPPORT1
1566 if (ompt_enabled.enabled) {
1567 ompt_lw_taskteam_t lw_taskteam;
1568 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1569 &ompt_parallel_data, return_address);
1570 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 1, true);
1571 }
1572#endif
1573
1574 /* Change number of threads in the team if requested */
1575 if (master_set_numthreads) { // The parallel has num_threads clause
1576 if (master_set_numthreads <= master_th->th.th_teams_size.nth) {
1577 // AC: only can reduce number of threads dynamically, can't increase
1578 kmp_info_t **other_threads = parent_team->t.t_threads;
1579 // NOTE: if using distributed barrier, we need to run this code block
1580 // even when the team size appears not to have changed from the max.
1581 int old_proc = master_th->th.th_teams_size.nth;
1582 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] ==
1583 bp_dist_bar) {
1584 __kmp_resize_dist_barrier(parent_team, old_proc,
1585 master_set_numthreads);
1586 __kmp_add_threads_to_team(parent_team, master_set_numthreads);
1587 }
1588 parent_team->t.t_nproc = master_set_numthreads;
1589 for (i = 0; i < master_set_numthreads; ++i) {
1590 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1591 }
1592 }
1593 // Keep extra threads hot in the team for possible next parallels
1594 master_th->th.th_set_nproc = 0;
1595 }
1596
1597#if USE_DEBUGGER0
1598 if (__kmp_debugging) { // Let debugger override number of threads.
1599 int nth = __kmp_omp_num_threads(loc);
1600 if (nth > 0) { // 0 means debugger doesn't want to change num threads
1601 master_set_numthreads = nth;
1602 }
1603 }
1604#endif
1605
1606 // Figure out the proc_bind policy for the nested parallel within teams
1607 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
1608 // proc_bind_default means don't update
1609 kmp_proc_bind_t proc_bind_icv = proc_bind_default;
1610 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1611 proc_bind = proc_bind_false;
1612 } else {
1613 // No proc_bind clause specified; use current proc-bind-var
1614 if (proc_bind == proc_bind_default) {
1615 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
1616 }
1617 /* else: The proc_bind policy was specified explicitly on parallel
1618 clause.
1619 This overrides proc-bind-var for this parallel region, but does not
1620 change proc-bind-var. */
1621 // Figure the value of proc-bind-var for the child threads.
1622 if ((level + 1 < __kmp_nested_proc_bind.used) &&
1623 (__kmp_nested_proc_bind.bind_types[level + 1] !=
1624 master_th->th.th_current_task->td_icvs.proc_bind)) {
1625 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
1626 }
1627 }
1628 KMP_CHECK_UPDATE(parent_team->t.t_proc_bind, proc_bind)if ((parent_team->t.t_proc_bind) != (proc_bind)) (parent_team
->t.t_proc_bind) = (proc_bind)
;
1629 // Need to change the bind-var ICV to correct value for each implicit task
1630 if (proc_bind_icv != proc_bind_default &&
1631 master_th->th.th_current_task->td_icvs.proc_bind != proc_bind_icv) {
1632 kmp_info_t **other_threads = parent_team->t.t_threads;
1633 for (i = 0; i < master_th->th.th_team_nproc; ++i) {
1634 other_threads[i]->th.th_current_task->td_icvs.proc_bind =
1635 proc_bind_icv;
1636 }
1637 }
1638 // Reset for next parallel region
1639 master_th->th.th_set_proc_bind = proc_bind_default;
1640
1641#if USE_ITT_BUILD1 && USE_ITT_NOTIFY1
1642 if (((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 && __itt_get_timestamp_ptr__kmp_itt_get_timestamp_ptr__3_0) ||
1643 KMP_ITT_DEBUG0) &&
1644 __kmp_forkjoin_frames_mode == 3 &&
1645 parent_team->t.t_active_level == 1 // only report frames at level 1
1646 && master_th->th.th_teams_size.nteams == 1) {
1647 kmp_uint64 tmp_time = __itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0();
1648 master_th->th.th_frame_time = tmp_time;
1649 parent_team->t.t_region_time = tmp_time;
1650 }
1651 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
1652 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL)if (!(parent_team->t.t_stack_id == __null)) { __kmp_debug_assert
("parent_team->t.t_stack_id == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 1652); }
;
1653 // create new stack stitching id before entering fork barrier
1654 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
1655 }
1656#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
1657#if KMP_AFFINITY_SUPPORTED1
1658 __kmp_partition_places(parent_team);
1659#endif
1660
1661 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1662 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1663 root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
;
1664 __kmp_internal_fork(loc, gtid, parent_team);
1665 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1666 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1667 root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
;
1668
1669 if (call_context == fork_context_gnu)
1670 return TRUE(!0);
1671
1672 /* Invoke microtask for PRIMARY thread */
1673 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
1674 parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
;
1675
1676 if (!parent_team->t.t_invoke(gtid)) {
1677 KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread")if (!(0)) { __kmp_debug_assert(("cannot invoke microtask for PRIMARY thread"
), "openmp/runtime/src/kmp_runtime.cpp", 1677); }
;
1678 }
1679 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
1680 parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
;
1681 KMP_MB(); /* Flush all pending memory write invalidates. */
1682
1683 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
1684
1685 return TRUE(!0);
1686 } // Parallel closely nested in teams construct
1687
1688#if KMP_DEBUG1
1689 if (__kmp_tasking_mode != tskm_immediate_exec) {
1690 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1691); }
1691 parent_team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1691); }
;
1692 }
1693#endif
1694
1695 // Need this to happen before we determine the number of threads, not while
1696 // we are allocating the team
1697 //__kmp_push_current_task_to_thread(master_th, parent_team, 0);
1698 int enter_teams = 0;
1699 if (parent_team->t.t_active_level >=
1700 master_th->th.th_current_task->td_icvs.max_active_levels) {
1701 nthreads = 1;
1702 } else {
1703 enter_teams = ((ap == NULL__null && active_level == 0) ||
1704 (ap && teams_level > 0 && teams_level == level));
1705 nthreads = master_set_numthreads
1706 ? master_set_numthreads
1707 // TODO: get nproc directly from current task
1708 : get__nproc_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
;
1709 // Check if we need to take forkjoin lock? (no need for serialized
1710 // parallel out of teams construct). This code moved here from
1711 // __kmp_reserve_threads() to speedup nested serialized parallels.
1712 if (nthreads > 1) {
1713 if ((get__max_active_levels(master_th)((master_th)->th.th_current_task->td_icvs.max_active_levels
)
== 1 &&
1714 (root->r.r_in_parallel && !enter_teams)) ||
1715 (__kmp_library == library_serial)) {
1716 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
1717 " threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
1718 gtid, nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
;
1719 nthreads = 1;
1720 }
1721 }
1722 if (nthreads > 1) {
1723 /* determine how many new threads we can use */
1724 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1725 /* AC: If we execute teams from parallel region (on host), then teams
1726 should be created but each can only have 1 thread if nesting is
1727 disabled. If teams called from serial region, then teams and their
1728 threads should be created regardless of the nesting setting. */
1729 nthreads = __kmp_reserve_threads(root, parent_team, master_tid,
1730 nthreads, enter_teams);
1731 if (nthreads == 1) {
1732 // Free lock for single thread execution here; for multi-thread
1733 // execution it will be freed later after team of threads created
1734 // and initialized
1735 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1736 }
1737 }
1738 }
1739 KMP_DEBUG_ASSERT(nthreads > 0)if (!(nthreads > 0)) { __kmp_debug_assert("nthreads > 0"
, "openmp/runtime/src/kmp_runtime.cpp", 1739); }
;
1740
1741 // If we temporarily changed the set number of threads then restore it now
1742 master_th->th.th_set_nproc = 0;
1743
1744 /* create a serialized parallel region? */
1745 if (nthreads == 1) {
1746/* josh todo: hypothetical question: what do we do for OS X*? */
1747#if KMP_OS_LINUX1 && \
1748 (KMP_ARCH_X860 || KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640)
1749 void *args[argc];
1750#else
1751 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *))__builtin_alloca (argc * sizeof(void *));
1752#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \
1753 KMP_ARCH_AARCH64) */
1754
1755 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing parallel region\n"
, gtid); }
1756 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing parallel region\n"
, gtid); }
;
1757
1758 __kmpc_serialized_parallel(loc, gtid);
1759
1760#if OMPD_SUPPORT1
1761 master_th->th.th_serial_team->t.t_pkfn = microtask;
1762#endif
1763
1764 if (call_context == fork_context_intel) {
1765 /* TODO this sucks, use the compiler itself to pass args! :) */
1766 master_th->th.th_serial_team->t.t_ident = loc;
1767 if (!ap) {
1768 // revert change made in __kmpc_serialized_parallel()
1769 master_th->th.th_serial_team->t.t_level--;
1770 // Get args from parent team for teams construct
1771
1772#if OMPT_SUPPORT1
1773 void *dummy;
1774 void **exit_frame_p;
1775 ompt_task_info_t *task_info;
1776
1777 ompt_lw_taskteam_t lw_taskteam;
1778
1779 if (ompt_enabled.enabled) {
1780 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1781 &ompt_parallel_data, return_address);
1782
1783 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1784 // don't use lw_taskteam after linking. content was swaped
1785
1786 task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
1787 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1788 if (ompt_enabled.ompt_callback_implicit_task) {
1789 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num =
1790 __kmp_tid_from_gtid(gtid);
1791 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1792 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1793 &(task_info->task_data), 1,
1794 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num,
1795 ompt_task_implicit);
1796 }
1797
1798 /* OMPT state */
1799 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1800 } else {
1801 exit_frame_p = &dummy;
1802 }
1803#endif
1804
1805 {
1806 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1807 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1808 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1809 parent_team->t.t_argv
1810#if OMPT_SUPPORT1
1811 ,
1812 exit_frame_p
1813#endif
1814 );
1815 }
1816
1817#if OMPT_SUPPORT1
1818 if (ompt_enabled.enabled) {
1819 *exit_frame_p = NULL__null;
1820 if (ompt_enabled.ompt_callback_implicit_task) {
1821 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1822 ompt_scope_end, NULL__null, &(task_info->task_data), 1,
1823 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num,
1824 ompt_task_implicit);
1825 }
1826 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
;
1827 __ompt_lw_taskteam_unlink(master_th);
1828 if (ompt_enabled.ompt_callback_parallel_end) {
1829 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1830 &ompt_parallel_data, parent_task_data,
1831 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
| ompt_parallel_team,
1832 return_address);
1833 }
1834 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1835 }
1836#endif
1837 } else if (microtask == (microtask_t)__kmp_teams_master) {
1838 KMP_DEBUG_ASSERT(master_th->th.th_team ==if (!(master_th->th.th_team == master_th->th.th_serial_team
)) { __kmp_debug_assert("master_th->th.th_team == master_th->th.th_serial_team"
, "openmp/runtime/src/kmp_runtime.cpp", 1839); }
1839 master_th->th.th_serial_team)if (!(master_th->th.th_team == master_th->th.th_serial_team
)) { __kmp_debug_assert("master_th->th.th_team == master_th->th.th_serial_team"
, "openmp/runtime/src/kmp_runtime.cpp", 1839); }
;
1840 team = master_th->th.th_team;
1841 // team->t.t_pkfn = microtask;
1842 team->t.t_invoke = invoker;
1843 __kmp_alloc_argv_entries(argc, team, TRUE(!0));
1844 team->t.t_argc = argc;
1845 argv = (void **)team->t.t_argv;
1846 if (ap) {
1847 for (i = argc - 1; i >= 0; --i)
1848 *argv++ = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
1849 } else {
1850 for (i = 0; i < argc; ++i)
1851 // Get args from parent team for teams construct
1852 argv[i] = parent_team->t.t_argv[i];
1853 }
1854 // AC: revert change made in __kmpc_serialized_parallel()
1855 // because initial code in teams should have level=0
1856 team->t.t_level--;
1857 // AC: call special invoker for outer "parallel" of teams construct
1858 invoker(gtid);
1859#if OMPT_SUPPORT1
1860 if (ompt_enabled.enabled) {
1861 ompt_task_info_t *task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
1862 if (ompt_enabled.ompt_callback_implicit_task) {
1863 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1864 ompt_scope_end, NULL__null, &(task_info->task_data), 0,
1865 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_initial);
1866 }
1867 if (ompt_enabled.ompt_callback_parallel_end) {
1868 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1869 &ompt_parallel_data, parent_task_data,
1870 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
| ompt_parallel_league,
1871 return_address);
1872 }
1873 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1874 }
1875#endif
1876 } else {
1877 argv = args;
1878 for (i = argc - 1; i >= 0; --i)
1879 *argv++ = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
1880 KMP_MB();
1881
1882#if OMPT_SUPPORT1
1883 void *dummy;
1884 void **exit_frame_p;
1885 ompt_task_info_t *task_info;
1886
1887 ompt_lw_taskteam_t lw_taskteam;
1888
1889 if (ompt_enabled.enabled) {
1890 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1891 &ompt_parallel_data, return_address);
1892 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1893 // don't use lw_taskteam after linking. content was swaped
1894 task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
1895 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1896
1897 /* OMPT implicit task begin */
1898 implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
;
1899 if (ompt_enabled.ompt_callback_implicit_task) {
1900 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1901 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1902 implicit_task_data, 1, __kmp_tid_from_gtid(gtid),
1903 ompt_task_implicit);
1904 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num =
1905 __kmp_tid_from_gtid(gtid);
1906 }
1907
1908 /* OMPT state */
1909 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1910 } else {
1911 exit_frame_p = &dummy;
1912 }
1913#endif
1914
1915 {
1916 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1917 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1918 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1919#if OMPT_SUPPORT1
1920 ,
1921 exit_frame_p
1922#endif
1923 );
1924 }
1925
1926#if OMPT_SUPPORT1
1927 if (ompt_enabled.enabled) {
1928 *exit_frame_p = NULL__null;
1929 if (ompt_enabled.ompt_callback_implicit_task) {
1930 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1931 ompt_scope_end, NULL__null, &(task_info->task_data), 1,
1932 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num,
1933 ompt_task_implicit);
1934 }
1935
1936 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
;
1937 __ompt_lw_taskteam_unlink(master_th);
1938 if (ompt_enabled.ompt_callback_parallel_end) {
1939 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1940 &ompt_parallel_data, parent_task_data,
1941 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
| ompt_parallel_team,
1942 return_address);
1943 }
1944 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1945 }
1946#endif
1947 }
1948 } else if (call_context == fork_context_gnu) {
1949#if OMPT_SUPPORT1
1950 if (ompt_enabled.enabled) {
1951 ompt_lw_taskteam_t lwt;
1952 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1953 return_address);
1954
1955 lwt.ompt_task_info.frame.exit_frame = ompt_data_none{0};
1956 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1957 }
1958// don't use lw_taskteam after linking. content was swaped
1959#endif
1960
1961 // we were called from GNU native code
1962 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serial exit\n"
, gtid); }
;
1963 return FALSE0;
1964 } else {
1965 KMP_ASSERT2(call_context < fork_context_last,if (!(call_context < fork_context_last)) { __kmp_debug_assert
(("__kmp_fork_call: unknown fork_context parameter"), "openmp/runtime/src/kmp_runtime.cpp"
, 1966); }
1966 "__kmp_fork_call: unknown fork_context parameter")if (!(call_context < fork_context_last)) { __kmp_debug_assert
(("__kmp_fork_call: unknown fork_context parameter"), "openmp/runtime/src/kmp_runtime.cpp"
, 1966); }
;
1967 }
1968
1969 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serial exit\n"
, gtid); }
;
1970 KMP_MB();
1971 return FALSE0;
1972 } // if (nthreads == 1)
1973
1974 // GEH: only modify the executing flag in the case when not serialized
1975 // serialized case is handled in kmpc_serialized_parallel
1976 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1977 "curtask=%p, curtask_max_aclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1978 parent_team->t.t_active_level, master_th,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1979 master_th->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1980 master_th->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
;
1981 // TODO: GEH - cannot do this assertion because root thread not set up as
1982 // executing
1983 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
1984 master_th->th.th_current_task->td_flags.executing = 0;
1985
1986 if (!master_th->th.th_teams_microtask || level > teams_level) {
1987 /* Increment our nested depth level */
1988 KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
)
;
1989 }
1990
1991 // See if we need to make a copy of the ICVs.
1992 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
1993 if ((level + 1 < __kmp_nested_nth.used) &&
1994 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1995 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1996 } else {
1997 nthreads_icv = 0; // don't update
1998 }
1999
2000 // Figure out the proc_bind_policy for the new team.
2001 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
2002 // proc_bind_default means don't update
2003 kmp_proc_bind_t proc_bind_icv = proc_bind_default;
2004 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
2005 proc_bind = proc_bind_false;
2006 } else {
2007 // No proc_bind clause specified; use current proc-bind-var for this
2008 // parallel region
2009 if (proc_bind == proc_bind_default) {
2010 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
2011 }
2012 // Have teams construct take proc_bind value from KMP_TEAMS_PROC_BIND
2013 if (master_th->th.th_teams_microtask &&
2014 microtask == (microtask_t)__kmp_teams_master) {
2015 proc_bind = __kmp_teams_proc_bind;
2016 }
2017 /* else: The proc_bind policy was specified explicitly on parallel clause.
2018 This overrides proc-bind-var for this parallel region, but does not
2019 change proc-bind-var. */
2020 // Figure the value of proc-bind-var for the child threads.
2021 if ((level + 1 < __kmp_nested_proc_bind.used) &&
2022 (__kmp_nested_proc_bind.bind_types[level + 1] !=
2023 master_th->th.th_current_task->td_icvs.proc_bind)) {
2024 // Do not modify the proc bind icv for the two teams construct forks
2025 // They just let the proc bind icv pass through
2026 if (!master_th->th.th_teams_microtask ||
2027 !(microtask == (microtask_t)__kmp_teams_master || ap == NULL__null))
2028 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
2029 }
2030 }
2031
2032 // Reset for next parallel region
2033 master_th->th.th_set_proc_bind = proc_bind_default;
2034
2035 if ((nthreads_icv > 0) || (proc_bind_icv != proc_bind_default)) {
2036 kmp_internal_control_t new_icvs;
2037 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2038 new_icvs.next = NULL__null;
2039 if (nthreads_icv > 0) {
2040 new_icvs.nproc = nthreads_icv;
2041 }
2042 if (proc_bind_icv != proc_bind_default) {
2043 new_icvs.proc_bind = proc_bind_icv;
2044 }
2045
2046 /* allocate a new parallel team */
2047 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); }
;
2048 team = __kmp_allocate_team(root, nthreads, nthreads,
2049#if OMPT_SUPPORT1
2050 ompt_parallel_data,
2051#endif
2052 proc_bind, &new_icvs,
2053 argc USE_NESTED_HOT_ARG(master_th), master_th);
2054 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar)
2055 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs);
2056 } else {
2057 /* allocate a new parallel team */
2058 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); }
;
2059 team = __kmp_allocate_team(root, nthreads, nthreads,
2060#if OMPT_SUPPORT1
2061 ompt_parallel_data,
2062#endif
2063 proc_bind,
2064 &master_th->th.th_current_task->td_icvs,
2065 argc USE_NESTED_HOT_ARG(master_th), master_th);
2066 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar)
2067 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs,
2068 &master_th->th.th_current_task->td_icvs);
2069 }
2070 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); }
2071 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); }
;
2072
2073 /* setup the new team */
2074 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid)if ((team->t.t_master_tid) != (master_tid)) (team->t.t_master_tid
) = (master_tid)
;
2075 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons)if ((team->t.t_master_this_cons) != (master_this_cons)) (team
->t.t_master_this_cons) = (master_this_cons)
;
2076 KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
)
;
2077 KMP_CHECK_UPDATE(team->t.t_parent, parent_team)if ((team->t.t_parent) != (parent_team)) (team->t.t_parent
) = (parent_team)
;
2078 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask)if ((team->t.t_pkfn) != (microtask)) (((team->t.t_pkfn)
)) = (((microtask)))
;
2079#if OMPT_SUPPORT1
2080 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)))
2081 return_address)if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)))
;
2082#endif
2083 KMP_CHECK_UPDATE(team->t.t_invoke, invoker)if ((team->t.t_invoke) != (invoker)) (team->t.t_invoke)
= (invoker)
; // TODO move to root, maybe
2084 // TODO: parent_team->t.t_level == INT_MAX ???
2085 if (!master_th->th.th_teams_microtask || level > teams_level) {
2086 int new_level = parent_team->t.t_level + 1;
2087 KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
= (new_level)
;
2088 new_level = parent_team->t.t_active_level + 1;
2089 KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level)
;
2090 } else {
2091 // AC: Do not increase parallel level at start of the teams construct
2092 int new_level = parent_team->t.t_level;
2093 KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
= (new_level)
;
2094 new_level = parent_team->t.t_active_level;
2095 KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level)
;
2096 }
2097 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.sched)
;
2098 // set primary thread's schedule as new run-time schedule
2099 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched)if ((team->t.t_sched.sched) != (new_sched.sched)) (team->
t.t_sched.sched) = (new_sched.sched)
;
2100
2101 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq)if ((team->t.t_cancel_request) != (cancel_noreq)) (team->
t.t_cancel_request) = (cancel_noreq)
;
2102 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator)if ((team->t.t_def_allocator) != (master_th->th.th_def_allocator
)) (team->t.t_def_allocator) = (master_th->th.th_def_allocator
)
;
2103
2104 // Update the floating point rounding in the team if required.
2105 propagateFPControl(team);
2106#if OMPD_SUPPORT1
2107 if (ompd_state & OMPD_ENABLE_BP0x1)
2108 ompd_bp_parallel_begin();
2109#endif
2110
2111 if (__kmp_tasking_mode != tskm_immediate_exec) {
2112 // Set primary thread's task team to team's task team. Unless this is hot
2113 // team, it should be NULL.
2114 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2115); }
2115 parent_team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2115); }
;
2116 KA_TRACE(20, ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2117 "%p, new task_team %p / team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2118 __kmp_gtid_from_thread(master_th),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2119 master_th->th.th_task_team, parent_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2120 team->t.t_task_team[master_th->th.th_task_state], team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
;
2121
2122 if (active_level || master_th->th.th_task_team) {
2123 // Take a memo of primary thread's task_state
2124 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)if (!(master_th->th.th_task_state_memo_stack)) { __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "openmp/runtime/src/kmp_runtime.cpp"
, 2124); }
;
2125 if (master_th->th.th_task_state_top >=
2126 master_th->th.th_task_state_stack_sz) { // increase size
2127 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2128 kmp_uint8 *old_stack, *new_stack;
2129 kmp_uint32 i;
2130 new_stack = (kmp_uint8 *)__kmp_allocate(new_size)___kmp_allocate((new_size), "openmp/runtime/src/kmp_runtime.cpp"
, 2130)
;
2131 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2132 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2133 }
2134 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2135 ++i) { // zero-init rest of stack
2136 new_stack[i] = 0;
2137 }
2138 old_stack = master_th->th.th_task_state_memo_stack;
2139 master_th->th.th_task_state_memo_stack = new_stack;
2140 master_th->th.th_task_state_stack_sz = new_size;
2141 __kmp_free(old_stack)___kmp_free((old_stack), "openmp/runtime/src/kmp_runtime.cpp"
, 2141)
;
2142 }
2143 // Store primary thread's task_state on stack
2144 master_th->th
2145 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2146 master_th->th.th_task_state;
2147 master_th->th.th_task_state_top++;
2148#if KMP_NESTED_HOT_TEAMS1
2149 if (master_th->th.th_hot_teams &&
2150 active_level < __kmp_hot_teams_max_level &&
2151 team == master_th->th.th_hot_teams[active_level].hot_team) {
2152 // Restore primary thread's nested state if nested hot team
2153 master_th->th.th_task_state =
2154 master_th->th
2155 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2156 } else {
2157#endif
2158 master_th->th.th_task_state = 0;
2159#if KMP_NESTED_HOT_TEAMS1
2160 }
2161#endif
2162 }
2163#if !KMP_NESTED_HOT_TEAMS1
2164 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||if (!((master_th->th.th_task_team == __null) || (team == root
->r.r_hot_team))) { __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "openmp/runtime/src/kmp_runtime.cpp", 2165); }
2165 (team == root->r.r_hot_team))if (!((master_th->th.th_task_team == __null) || (team == root
->r.r_hot_team))) { __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "openmp/runtime/src/kmp_runtime.cpp", 2165); }
;
2166#endif
2167 }
2168
2169 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2170 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2171 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2172 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2173 team->t.t_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
;
2174 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2177); }
2175 (team->t.t_master_tid == 0 &&if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2177); }
2176 (team->t.t_parent == root->r.r_root_team ||if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2177); }
2177 team->t.t_parent->t.t_serialized)))if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2177); }
;
2178 KMP_MB();
2179
2180 /* now, setup the arguments */
2181 argv = (void **)team->t.t_argv;
2182 if (ap) {
2183 for (i = argc - 1; i >= 0; --i) {
2184 void *new_argv = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
2185 KMP_CHECK_UPDATE(*argv, new_argv)if ((*argv) != (new_argv)) (*argv) = (new_argv);
2186 argv++;
2187 }
2188 } else {
2189 for (i = 0; i < argc; ++i) {
2190 // Get args from parent team for teams construct
2191 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i])if ((argv[i]) != (team->t.t_parent->t.t_argv[i])) (argv
[i]) = (team->t.t_parent->t.t_argv[i])
;
2192 }
2193 }
2194
2195 /* now actually fork the threads */
2196 KMP_CHECK_UPDATE(team->t.t_master_active, master_active)if ((team->t.t_master_active) != (master_active)) (team->
t.t_master_active) = (master_active)
;
2197 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
2198 root->r.r_active = TRUE(!0);
2199
2200 __kmp_fork_team_threads(root, team, master_th, gtid, !ap);
2201 __kmp_setup_icv_copy(team, nthreads,
2202 &master_th->th.th_current_task->td_icvs, loc);
2203
2204#if OMPT_SUPPORT1
2205 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2206#endif
2207
2208 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2209
2210#if USE_ITT_BUILD1
2211 if (team->t.t_active_level == 1 // only report frames at level 1
2212 && !master_th->th.th_teams_microtask) { // not in teams construct
2213#if USE_ITT_NOTIFY1
2214 if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2215 (__kmp_forkjoin_frames_mode == 3 ||
2216 __kmp_forkjoin_frames_mode == 1)) {
2217 kmp_uint64 tmp_time = 0;
2218 if (__itt_get_timestamp_ptr__kmp_itt_get_timestamp_ptr__3_0)
2219 tmp_time = __itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0();
2220 // Internal fork - report frame begin
2221 master_th->th.th_frame_time = tmp_time;
2222 if (__kmp_forkjoin_frames_mode == 3)
2223 team->t.t_region_time = tmp_time;
2224 } else
2225// only one notification scheme (either "submit" or "forking/joined", not both)
2226#endif /* USE_ITT_NOTIFY */
2227 if ((__itt_frame_begin_v3_ptr__kmp_itt_frame_begin_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2228 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2229 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer.
2230 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2231 }
2232 }
2233#endif /* USE_ITT_BUILD */
2234
2235 /* now go on and do the work */
2236 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team)if (!(team == __kmp_threads[gtid]->th.th_team)) { __kmp_debug_assert
("team == __kmp_threads[gtid]->th.th_team", "openmp/runtime/src/kmp_runtime.cpp"
, 2236); }
;
2237 KMP_MB();
2238 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
2239 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
2240 root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
;
2241
2242#if USE_ITT_BUILD1
2243 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
2244 // create new stack stitching id before entering fork barrier
2245 if (!enter_teams) {
2246 KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL)if (!(team->t.t_stack_id == __null)) { __kmp_debug_assert(
"team->t.t_stack_id == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2246); }
;
2247 team->t.t_stack_id = __kmp_itt_stack_caller_create();
2248 } else if (parent_team->t.t_serialized) {
2249 // keep stack stitching id in the serialized parent_team;
2250 // current team will be used for parallel inside the teams;
2251 // if parent_team is active, then it already keeps stack stitching id
2252 // for the league of teams
2253 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL)if (!(parent_team->t.t_stack_id == __null)) { __kmp_debug_assert
("parent_team->t.t_stack_id == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2253); }
;
2254 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
2255 }
2256 }
2257#endif /* USE_ITT_BUILD */
2258
2259 // AC: skip __kmp_internal_fork at teams construct, let only primary
2260 // threads execute
2261 if (ap) {
2262 __kmp_internal_fork(loc, gtid, team);
2263 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
2264 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
2265 root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
;
2266 }
2267
2268 if (call_context == fork_context_gnu) {
2269 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
2270 return TRUE(!0);
2271 }
2272
2273 /* Invoke microtask for PRIMARY thread */
2274 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
2275 team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
;
2276 } // END of timer KMP_fork_call block
2277
2278#if KMP_STATS_ENABLED0
2279 // If beginning a teams construct, then change thread state
2280 stats_state_e previous_state = KMP_GET_THREAD_STATE()((void)0);
2281 if (!ap) {
2282 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION)((void)0);
2283 }
2284#endif
2285
2286 if (!team->t.t_invoke(gtid)) {
2287 KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread")if (!(0)) { __kmp_debug_assert(("cannot invoke microtask for PRIMARY thread"
), "openmp/runtime/src/kmp_runtime.cpp", 2287); }
;
2288 }
2289
2290#if KMP_STATS_ENABLED0
2291 // If was beginning of a teams construct, then reset thread state
2292 if (!ap) {
2293 KMP_SET_THREAD_STATE(previous_state)((void)0);
2294 }
2295#endif
2296
2297 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
2298 team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
;
2299 KMP_MB(); /* Flush all pending memory write invalidates. */
2300
2301 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
2302#if OMPT_SUPPORT1
2303 if (ompt_enabled.enabled) {
2304 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2305 }
2306#endif
2307
2308 return TRUE(!0);
2309}
2310
2311#if OMPT_SUPPORT1
2312static inline void __kmp_join_restore_state(kmp_info_t *thread,
2313 kmp_team_t *team) {
2314 // restore state outside the region
2315 thread->th.ompt_thread_info.state =
2316 ((team->t.t_serialized) ? ompt_state_work_serial
2317 : ompt_state_work_parallel);
2318}
2319
2320static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread,
2321 kmp_team_t *team, ompt_data_t *parallel_data,
2322 int flags, void *codeptr) {
2323 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2324 if (ompt_enabled.ompt_callback_parallel_end) {
2325 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
2326 parallel_data, &(task_info->task_data), flags, codeptr);
2327 }
2328
2329 task_info->frame.enter_frame = ompt_data_none{0};
2330 __kmp_join_restore_state(thread, team);
2331}
2332#endif
2333
2334void __kmp_join_call(ident_t *loc, int gtid
2335#if OMPT_SUPPORT1
2336 ,
2337 enum fork_context_e fork_context
2338#endif
2339 ,
2340 int exit_teams) {
2341 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call)((void)0);
2342 kmp_team_t *team;
2343 kmp_team_t *parent_team;
2344 kmp_info_t *master_th;
2345 kmp_root_t *root;
2346 int master_active;
2347
2348 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: enter T#%d\n"
, gtid); }
;
2349
2350 /* setup current data */
2351 master_th = __kmp_threads[gtid];
2352 root = master_th->th.th_root;
2353 team = master_th->th.th_team;
2354 parent_team = team->t.t_parent;
2355
2356 master_th->th.th_ident = loc;
2357
2358#if OMPT_SUPPORT1
2359 void *team_microtask = (void *)team->t.t_pkfn;
2360 // For GOMP interface with serialized parallel, need the
2361 // __kmpc_end_serialized_parallel to call hooks for OMPT end-implicit-task
2362 // and end-parallel events.
2363 if (ompt_enabled.enabled &&
2364 !(team->t.t_serialized && fork_context == fork_context_gnu)) {
2365 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2366 }
2367#endif
2368
2369#if KMP_DEBUG1
2370 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2371 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2372 "th_task_team = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2373 __kmp_gtid_from_thread(master_th), team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2374 team->t.t_task_team[master_th->th.th_task_state],if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2375 master_th->th.th_task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
;
2376 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2377); }
2377 team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2377); }
;
2378 }
2379#endif
2380
2381 if (team->t.t_serialized) {
2382 if (master_th->th.th_teams_microtask) {
2383 // We are in teams construct
2384 int level = team->t.t_level;
2385 int tlevel = master_th->th.th_teams_level;
2386 if (level == tlevel) {
2387 // AC: we haven't incremented it earlier at start of teams construct,
2388 // so do it here - at the end of teams construct
2389 team->t.t_level++;
2390 } else if (level == tlevel + 1) {
2391 // AC: we are exiting parallel inside teams, need to increment
2392 // serialization in order to restore it in the next call to
2393 // __kmpc_end_serialized_parallel
2394 team->t.t_serialized++;
2395 }
2396 }
2397 __kmpc_end_serialized_parallel(loc, gtid);
2398
2399#if OMPT_SUPPORT1
2400 if (ompt_enabled.enabled) {
2401 if (fork_context == fork_context_gnu) {
2402 __ompt_lw_taskteam_unlink(master_th);
2403 }
2404 __kmp_join_restore_state(master_th, parent_team);
2405 }
2406#endif
2407
2408 return;
2409 }
2410
2411 master_active = team->t.t_master_active;
2412
2413 if (!exit_teams) {
2414 // AC: No barrier for internal teams at exit from teams construct.
2415 // But there is barrier for external team (league).
2416 __kmp_internal_join(loc, gtid, team);
2417#if USE_ITT_BUILD1
2418 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
2419 KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL)if (!(team->t.t_stack_id != __null)) { __kmp_debug_assert(
"team->t.t_stack_id != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2419); }
;
2420 // destroy the stack stitching id after join barrier
2421 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id);
2422 team->t.t_stack_id = NULL__null;
2423 }
2424#endif
2425 } else {
2426 master_th->th.th_task_state =
2427 0; // AC: no tasking in teams (out of any parallel)
2428#if USE_ITT_BUILD1
2429 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0 && parent_team->t.t_serialized) {
2430 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id != NULL)if (!(parent_team->t.t_stack_id != __null)) { __kmp_debug_assert
("parent_team->t.t_stack_id != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2430); }
;
2431 // destroy the stack stitching id on exit from the teams construct
2432 // if parent_team is active, then the id will be destroyed later on
2433 // by master of the league of teams
2434 __kmp_itt_stack_caller_destroy((__itt_caller)parent_team->t.t_stack_id);
2435 parent_team->t.t_stack_id = NULL__null;
2436 }
2437#endif
2438
2439 if (team->t.t_nproc > 1 &&
2440 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
2441 team->t.b->update_num_threads(team->t.t_nproc);
2442 __kmp_add_threads_to_team(team, team->t.t_nproc);
2443 }
2444 }
2445
2446 KMP_MB();
2447
2448#if OMPT_SUPPORT1
2449 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2450 void *codeptr = team->t.ompt_team_info.master_return_address;
2451#endif
2452
2453#if USE_ITT_BUILD1
2454 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer.
2455 if (team->t.t_active_level == 1 &&
2456 (!master_th->th.th_teams_microtask || /* not in teams construct */
2457 master_th->th.th_teams_size.nteams == 1)) {
2458 master_th->th.th_ident = loc;
2459 // only one notification scheme (either "submit" or "forking/joined", not
2460 // both)
2461 if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2462 __kmp_forkjoin_frames_mode == 3)
2463 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2464 master_th->th.th_frame_time, 0, loc,
2465 master_th->th.th_team_nproc, 1);
2466 else if ((__itt_frame_end_v3_ptr__kmp_itt_frame_end_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2467 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2468 __kmp_itt_region_joined(gtid);
2469 } // active_level == 1
2470#endif /* USE_ITT_BUILD */
2471
2472#if KMP_AFFINITY_SUPPORTED1
2473 if (!exit_teams) {
2474 // Restore master thread's partition.
2475 master_th->th.th_first_place = team->t.t_first_place;
2476 master_th->th.th_last_place = team->t.t_last_place;
2477 }
2478#endif // KMP_AFFINITY_SUPPORTED
2479
2480 if (master_th->th.th_teams_microtask && !exit_teams &&
2481 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2482 team->t.t_level == master_th->th.th_teams_level + 1) {
2483// AC: We need to leave the team structure intact at the end of parallel
2484// inside the teams construct, so that at the next parallel same (hot) team
2485// works, only adjust nesting levels
2486#if OMPT_SUPPORT1
2487 ompt_data_t ompt_parallel_data = ompt_data_none{0};
2488 if (ompt_enabled.enabled) {
2489 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2490 if (ompt_enabled.ompt_callback_implicit_task) {
2491 int ompt_team_size = team->t.t_nproc;
2492 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
2493 ompt_scope_end, NULL__null, &(task_info->task_data), ompt_team_size,
2494 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
2495 }
2496 task_info->frame.exit_frame = ompt_data_none{0};
2497 task_info->task_data = ompt_data_none{0};
2498 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
;
2499 __ompt_lw_taskteam_unlink(master_th);
2500 }
2501#endif
2502 /* Decrement our nested depth level */
2503 team->t.t_level--;
2504 team->t.t_active_level--;
2505 KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
)
;
2506
2507 // Restore number of threads in the team if needed. This code relies on
2508 // the proper adjustment of th_teams_size.nth after the fork in
2509 // __kmp_teams_master on each teams primary thread in the case that
2510 // __kmp_reserve_threads reduced it.
2511 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2512 int old_num = master_th->th.th_team_nproc;
2513 int new_num = master_th->th.th_teams_size.nth;
2514 kmp_info_t **other_threads = team->t.t_threads;
2515 team->t.t_nproc = new_num;
2516 for (int i = 0; i < old_num; ++i) {
2517 other_threads[i]->th.th_team_nproc = new_num;
2518 }
2519 // Adjust states of non-used threads of the team
2520 for (int i = old_num; i < new_num; ++i) {
2521 // Re-initialize thread's barrier data.
2522 KMP_DEBUG_ASSERT(other_threads[i])if (!(other_threads[i])) { __kmp_debug_assert("other_threads[i]"
, "openmp/runtime/src/kmp_runtime.cpp", 2522); }
;
2523 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2524 for (int b = 0; b < bs_last_barrier; ++b) {
2525 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2526 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 2526); }
;
2527#if USE_DEBUGGER0
2528 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2529#endif
2530 }
2531 if (__kmp_tasking_mode != tskm_immediate_exec) {
2532 // Synchronize thread's task state
2533 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2534 }
2535 }
2536 }
2537
2538#if OMPT_SUPPORT1
2539 if (ompt_enabled.enabled) {
2540 __kmp_join_ompt(gtid, master_th, parent_team, &ompt_parallel_data,
2541 OMPT_INVOKER(fork_context)((fork_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
| ompt_parallel_team, codeptr);
2542 }
2543#endif
2544
2545 return;
2546 }
2547
2548 /* do cleanup and restore the parent team */
2549 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2550 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2551
2552 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2553
2554 /* jc: The following lock has instructions with REL and ACQ semantics,
2555 separating the parallel user code called in this parallel region
2556 from the serial user code called after this function returns. */
2557 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2558
2559 if (!master_th->th.th_teams_microtask ||
2560 team->t.t_level > master_th->th.th_teams_level) {
2561 /* Decrement our nested depth level */
2562 KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
)
;
2563 }
2564 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0)if (!(root->r.r_in_parallel >= 0)) { __kmp_debug_assert
("root->r.r_in_parallel >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 2564); }
;
2565
2566#if OMPT_SUPPORT1
2567 if (ompt_enabled.enabled) {
2568 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2569 if (ompt_enabled.ompt_callback_implicit_task) {
2570 int flags = (team_microtask == (void *)__kmp_teams_master)
2571 ? ompt_task_initial
2572 : ompt_task_implicit;
2573 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc;
2574 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
2575 ompt_scope_end, NULL__null, &(task_info->task_data), ompt_team_size,
2576 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, flags);
2577 }
2578 task_info->frame.exit_frame = ompt_data_none{0};
2579 task_info->task_data = ompt_data_none{0};
2580 }
2581#endif
2582
2583 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); }
2584 master_th, team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); }
;
2585 __kmp_pop_current_task_from_thread(master_th);
2586
2587 master_th->th.th_def_allocator = team->t.t_def_allocator;
2588
2589#if OMPD_SUPPORT1
2590 if (ompd_state & OMPD_ENABLE_BP0x1)
2591 ompd_bp_parallel_end();
2592#endif
2593 updateHWFPControl(team);
2594
2595 if (root->r.r_active != master_active)
2596 root->r.r_active = master_active;
2597
2598 __kmp_free_team(root, team USE_NESTED_HOT_ARG(, master_th
2599 master_th), master_th); // this will free worker threads
2600
2601 /* this race was fun to find. make sure the following is in the critical
2602 region otherwise assertions may fail occasionally since the old team may be
2603 reallocated and the hierarchy appears inconsistent. it is actually safe to
2604 run and won't cause any bugs, but will cause those assertion failures. it's
2605 only one deref&assign so might as well put this in the critical region */
2606 master_th->th.th_team = parent_team;
2607 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2608 master_th->th.th_team_master = parent_team->t.t_threads[0];
2609 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2610
2611 /* restore serialized team, if need be */
2612 if (parent_team->t.t_serialized &&
2613 parent_team != master_th->th.th_serial_team &&
2614 parent_team != root->r.r_root_team) {
2615 __kmp_free_team(root,
2616 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL), __null);
2617 master_th->th.th_serial_team = parent_team;
2618 }
2619
2620 if (__kmp_tasking_mode != tskm_immediate_exec) {
2621 if (master_th->th.th_task_state_top >
2622 0) { // Restore task state from memo stack
2623 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)if (!(master_th->th.th_task_state_memo_stack)) { __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "openmp/runtime/src/kmp_runtime.cpp"
, 2623); }
;
2624 // Remember primary thread's state if we re-use this nested hot team
2625 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2626 master_th->th.th_task_state;
2627 --master_th->th.th_task_state_top; // pop
2628 // Now restore state at this level
2629 master_th->th.th_task_state =
2630 master_th->th
2631 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2632 }
2633 // Copy the task team from the parent team to the primary thread
2634 master_th->th.th_task_team =
2635 parent_team->t.t_task_team[master_th->th.th_task_state];
2636 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
2637 ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
2638 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
2639 parent_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
;
2640 }
2641
2642 // TODO: GEH - cannot do this assertion because root thread not set up as
2643 // executing
2644 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
2645 master_th->th.th_current_task->td_flags.executing = 1;
2646
2647 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2648
2649#if KMP_AFFINITY_SUPPORTED1
2650 if (master_th->th.th_team->t.t_level == 0 && __kmp_affinity.flags.reset) {
2651 __kmp_reset_root_init_mask(gtid);
2652 }
2653#endif
2654#if OMPT_SUPPORT1
2655 int flags =
2656 OMPT_INVOKER(fork_context)((fork_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
|
2657 ((team_microtask == (void *)__kmp_teams_master) ? ompt_parallel_league
2658 : ompt_parallel_team);
2659 if (ompt_enabled.enabled) {
2660 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags,
2661 codeptr);
2662 }
2663#endif
2664
2665 KMP_MB();
2666 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: exit T#%d\n"
, gtid); }
;
2667}
2668
2669/* Check whether we should push an internal control record onto the
2670 serial team stack. If so, do it. */
2671void __kmp_save_internal_controls(kmp_info_t *thread) {
2672
2673 if (thread->th.th_team != thread->th.th_serial_team) {
2674 return;
2675 }
2676 if (thread->th.th_team->t.t_serialized > 1) {
2677 int push = 0;
2678
2679 if (thread->th.th_team->t.t_control_stack_top == NULL__null) {
2680 push = 1;
2681 } else {
2682 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2683 thread->th.th_team->t.t_serialized) {
2684 push = 1;
2685 }
2686 }
2687 if (push) { /* push a record on the serial team's stack */
2688 kmp_internal_control_t *control =
2689 (kmp_internal_control_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_internal_control_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 2690)
2690 sizeof(kmp_internal_control_t))___kmp_allocate((sizeof(kmp_internal_control_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 2690)
;
2691
2692 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2693
2694 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2695
2696 control->next = thread->th.th_team->t.t_control_stack_top;
2697 thread->th.th_team->t.t_control_stack_top = control;
2698 }
2699 }
2700}
2701
2702/* Changes set_nproc */
2703void __kmp_set_num_threads(int new_nth, int gtid) {
2704 kmp_info_t *thread;
2705 kmp_root_t *root;
2706
2707 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_num_threads: new __kmp_nth = %d\n"
, new_nth); }
;
2708 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2708); }
;
2709
2710 if (new_nth < 1)
2711 new_nth = 1;
2712 else if (new_nth > __kmp_max_nth)
2713 new_nth = __kmp_max_nth;
2714
2715 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth)((void)0);
2716 thread = __kmp_threads[gtid];
2717 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2718 return; // nothing to do
2719
2720 __kmp_save_internal_controls(thread);
2721
2722 set__nproc(thread, new_nth)(((thread)->th.th_current_task->td_icvs.nproc) = (new_nth
))
;
2723
2724 // If this omp_set_num_threads() call will cause the hot team size to be
2725 // reduced (in the absence of a num_threads clause), then reduce it now,
2726 // rather than waiting for the next parallel region.
2727 root = thread->th.th_root;
2728 if (__kmp_init_parallel && (!root->r.r_active) &&
2729 (root->r.r_hot_team->t.t_nproc > new_nth)
2730#if KMP_NESTED_HOT_TEAMS1
2731 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2732#endif
2733 ) {
2734 kmp_team_t *hot_team = root->r.r_hot_team;
2735 int f;
2736
2737 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2738
2739 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
2740 __kmp_resize_dist_barrier(hot_team, hot_team->t.t_nproc, new_nth);
2741 }
2742 // Release the extra threads we don't need any more.
2743 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2744 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)if (!(hot_team->t.t_threads[f] != __null)) { __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2744); }
;
2745 if (__kmp_tasking_mode != tskm_immediate_exec) {
2746 // When decreasing team size, threads no longer in the team should unref
2747 // task team.
2748 hot_team->t.t_threads[f]->th.th_task_team = NULL__null;
2749 }
2750 __kmp_free_thread(hot_team->t.t_threads[f]);
2751 hot_team->t.t_threads[f] = NULL__null;
2752 }
2753 hot_team->t.t_nproc = new_nth;
2754#if KMP_NESTED_HOT_TEAMS1
2755 if (thread->th.th_hot_teams) {
2756 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team)if (!(hot_team == thread->th.th_hot_teams[0].hot_team)) { __kmp_debug_assert
("hot_team == thread->th.th_hot_teams[0].hot_team", "openmp/runtime/src/kmp_runtime.cpp"
, 2756); }
;
2757 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2758 }
2759#endif
2760
2761 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
2762 hot_team->t.b->update_num_threads(new_nth);
2763 __kmp_add_threads_to_team(hot_team, new_nth);
2764 }
2765
2766 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2767
2768 // Update the t_nproc field in the threads that are still active.
2769 for (f = 0; f < new_nth; f++) {
2770 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)if (!(hot_team->t.t_threads[f] != __null)) { __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2770); }
;
2771 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2772 }
2773 // Special flag in case omp_set_num_threads() call
2774 hot_team->t.t_size_changed = -1;
2775 }
2776}
2777
2778/* Changes max_active_levels */
2779void __kmp_set_max_active_levels(int gtid, int max_active_levels) {
2780 kmp_info_t *thread;
2781
2782 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
2783 "%d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
2784 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
;
2785 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2785); }
;
2786
2787 // validate max_active_levels
2788 if (max_active_levels < 0) {
2789 KMP_WARNING(ActiveLevelsNegative, max_active_levels)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsNegative
, max_active_levels), __kmp_msg_null)
;
2790 // We ignore this call if the user has specified a negative value.
2791 // The current setting won't be changed. The last valid setting will be
2792 // used. A warning will be issued (if warnings are allowed as controlled by
2793 // the KMP_WARNINGS env var).
2794 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2795 "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2796 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
;
2797 return;
2798 }
2799 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647) {
2800 // it's OK, the max_active_levels is within the valid range: [ 0;
2801 // KMP_MAX_ACTIVE_LEVELS_LIMIT ]
2802 // We allow a zero value. (implementation defined behavior)
2803 } else {
2804 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null)
2805 KMP_MAX_ACTIVE_LEVELS_LIMIT)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null)
;
2806 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647;
2807 // Current upper limit is MAX_INT. (implementation defined behavior)
2808 // If the input exceeds the upper limit, we correct the input to be the
2809 // upper limit. (implementation defined behavior)
2810 // Actually, the flow should never get here until we use MAX_INT limit.
2811 }
2812 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2813 "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2814 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
;
2815
2816 thread = __kmp_threads[gtid];
2817
2818 __kmp_save_internal_controls(thread);
2819
2820 set__max_active_levels(thread, max_active_levels)(((thread)->th.th_current_task->td_icvs.max_active_levels
) = (max_active_levels))
;
2821}
2822
2823/* Gets max_active_levels */
2824int __kmp_get_max_active_levels(int gtid) {
2825 kmp_info_t *thread;
2826
2827 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d\n"
, gtid); }
;
2828 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2828); }
;
2829
2830 thread = __kmp_threads[gtid];
2831 KMP_DEBUG_ASSERT(thread->th.th_current_task)if (!(thread->th.th_current_task)) { __kmp_debug_assert("thread->th.th_current_task"
, "openmp/runtime/src/kmp_runtime.cpp", 2831); }
;
2832 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2833 "curtask_maxaclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2834 gtid, thread->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2835 thread->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
;
2836 return thread->th.th_current_task->td_icvs.max_active_levels;
2837}
2838
2839// nteams-var per-device ICV
2840void __kmp_set_num_teams(int num_teams) {
2841 if (num_teams > 0)
2842 __kmp_nteams = num_teams;
2843}
2844int __kmp_get_max_teams(void) { return __kmp_nteams; }
2845// teams-thread-limit-var per-device ICV
2846void __kmp_set_teams_thread_limit(int limit) {
2847 if (limit > 0)
2848 __kmp_teams_thread_limit = limit;
2849}
2850int __kmp_get_teams_thread_limit(void) { return __kmp_teams_thread_limit; }
2851
2852KMP_BUILD_ASSERT(sizeof(kmp_sched_t) == sizeof(int))static_assert(sizeof(kmp_sched_t) == sizeof(int), "Build condition error"
)
;
2853KMP_BUILD_ASSERT(sizeof(enum sched_type) == sizeof(int))static_assert(sizeof(enum sched_type) == sizeof(int), "Build condition error"
)
;
2854
2855/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
2856void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
2857 kmp_info_t *thread;
2858 kmp_sched_t orig_kind;
2859 // kmp_team_t *team;
2860
2861 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); }
2862 gtid, (int)kind, chunk))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); }
;
2863 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2863); }
;
2864
2865 // Check if the kind parameter is valid, correct if needed.
2866 // Valid parameters should fit in one of two intervals - standard or extended:
2867 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
2868 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103
2869 orig_kind = kind;
2870 kind = __kmp_sched_without_mods(kind);
2871
2872 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2873 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2874 // TODO: Hint needs attention in case we change the default schedule.
2875 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind)__kmp_msg_format(kmp_i18n_msg_ScheduleKindOutOfRange, kind),
2876 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk")__kmp_msg_format(kmp_i18n_hnt_DefaultScheduleKindUsed, "static, no chunk"
)
,
2877 __kmp_msg_null);
2878 kind = kmp_sched_default;
2879 chunk = 0; // ignore chunk value in case of bad kind
2880 }
2881
2882 thread = __kmp_threads[gtid];
2883
2884 __kmp_save_internal_controls(thread);
2885
2886 if (kind < kmp_sched_upper_std) {
2887 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK1) {
2888 // differ static chunked vs. unchunked: chunk should be invalid to
2889 // indicate unchunked schedule (which is the default)
2890 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
2891 } else {
2892 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2893 __kmp_sch_map[kind - kmp_sched_lower - 1];
2894 }
2895 } else {
2896 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2897 // kmp_sched_lower - 2 ];
2898 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2899 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2900 kmp_sched_lower - 2];
2901 }
2902 __kmp_sched_apply_mods_intkind(
2903 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type));
2904 if (kind == kmp_sched_auto || chunk < 1) {
2905 // ignore parameter chunk for schedule auto
2906 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK1;
2907 } else {
2908 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2909 }
2910}
2911
2912/* Gets def_sched_var ICV values */
2913void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
2914 kmp_info_t *thread;
2915 enum sched_type th_type;
2916
2917 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_schedule: thread %d\n"
, gtid); }
;
2918 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2918); }
;
2919
2920 thread = __kmp_threads[gtid];
2921
2922 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2923 switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)(enum sched_type)( (th_type) & ~(kmp_sch_modifier_nonmonotonic
| kmp_sch_modifier_monotonic))
) {
2924 case kmp_sch_static:
2925 case kmp_sch_static_greedy:
2926 case kmp_sch_static_balanced:
2927 *kind = kmp_sched_static;
2928 __kmp_sched_apply_mods_stdkind(kind, th_type);
2929 *chunk = 0; // chunk was not set, try to show this fact via zero value
2930 return;
2931 case kmp_sch_static_chunked:
2932 *kind = kmp_sched_static;
2933 break;
2934 case kmp_sch_dynamic_chunked:
2935 *kind = kmp_sched_dynamic;
2936 break;
2937 case kmp_sch_guided_chunked:
2938 case kmp_sch_guided_iterative_chunked:
2939 case kmp_sch_guided_analytical_chunked:
2940 *kind = kmp_sched_guided;
2941 break;
2942 case kmp_sch_auto:
2943 *kind = kmp_sched_auto;
2944 break;
2945 case kmp_sch_trapezoidal:
2946 *kind = kmp_sched_trapezoidal;
2947 break;
2948#if KMP_STATIC_STEAL_ENABLED1
2949 case kmp_sch_static_steal:
2950 *kind = kmp_sched_static_steal;
2951 break;
2952#endif
2953 default:
2954 KMP_FATAL(UnknownSchedulingType, th_type)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownSchedulingType
, th_type), __kmp_msg_null)
;
2955 }
2956
2957 __kmp_sched_apply_mods_stdkind(kind, th_type);
2958 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2959}
2960
2961int __kmp_get_ancestor_thread_num(int gtid, int level) {
2962
2963 int ii, dd;
2964 kmp_team_t *team;
2965 kmp_info_t *thr;
2966
2967 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_ancestor_thread_num: thread %d %d\n"
, gtid, level); }
;
2968 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2968); }
;
2969
2970 // validate level
2971 if (level == 0)
2972 return 0;
2973 if (level < 0)
2974 return -1;
2975 thr = __kmp_threads[gtid];
2976 team = thr->th.th_team;
2977 ii = team->t.t_level;
2978 if (level > ii)
2979 return -1;
2980
2981 if (thr->th.th_teams_microtask) {
2982 // AC: we are in teams region where multiple nested teams have same level
2983 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2984 if (level <=
2985 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2986 KMP_DEBUG_ASSERT(ii >= tlevel)if (!(ii >= tlevel)) { __kmp_debug_assert("ii >= tlevel"
, "openmp/runtime/src/kmp_runtime.cpp", 2986); }
;
2987 // AC: As we need to pass by the teams league, we need to artificially
2988 // increase ii
2989 if (ii == tlevel) {
2990 ii += 2; // three teams have same level
2991 } else {
2992 ii++; // two teams have same level
2993 }
2994 }
2995 }
2996
2997 if (ii == level)
2998 return __kmp_tid_from_gtid(gtid);
2999
3000 dd = team->t.t_serialized;
3001 level++;
3002 while (ii > level) {
3003 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
3004 }
3005 if ((team->t.t_serialized) && (!dd)) {
3006 team = team->t.t_parent;
3007 continue;
3008 }
3009 if (ii > level) {
3010 team = team->t.t_parent;
3011 dd = team->t.t_serialized;
3012 ii--;
3013 }
3014 }
3015
3016 return (dd > 1) ? (0) : (team->t.t_master_tid);
3017}
3018
3019int __kmp_get_team_size(int gtid, int level) {
3020
3021 int ii, dd;
3022 kmp_team_t *team;
3023 kmp_info_t *thr;
3024
3025 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_team_size: thread %d %d\n"
, gtid, level); }
;
3026 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 3026); }
;
3027
3028 // validate level
3029 if (level == 0)
3030 return 1;
3031 if (level < 0)
3032 return -1;
3033 thr = __kmp_threads[gtid];
3034 team = thr->th.th_team;
3035 ii = team->t.t_level;
3036 if (level > ii)
3037 return -1;
3038
3039 if (thr->th.th_teams_microtask) {
3040 // AC: we are in teams region where multiple nested teams have same level
3041 int tlevel = thr->th.th_teams_level; // the level of the teams construct
3042 if (level <=
3043 tlevel) { // otherwise usual algorithm works (will not touch the teams)
3044 KMP_DEBUG_ASSERT(ii >= tlevel)if (!(ii >= tlevel)) { __kmp_debug_assert("ii >= tlevel"
, "openmp/runtime/src/kmp_runtime.cpp", 3044); }
;
3045 // AC: As we need to pass by the teams league, we need to artificially
3046 // increase ii
3047 if (ii == tlevel) {
3048 ii += 2; // three teams have same level
3049 } else {
3050 ii++; // two teams have same level
3051 }
3052 }
3053 }
3054
3055 while (ii > level) {
3056 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
3057 }
3058 if (team->t.t_serialized && (!dd)) {
3059 team = team->t.t_parent;
3060 continue;
3061 }
3062 if (ii > level) {
3063 team = team->t.t_parent;
3064 ii--;
3065 }
3066 }
3067
3068 return team->t.t_nproc;
3069}
3070
3071kmp_r_sched_t __kmp_get_schedule_global() {
3072 // This routine created because pairs (__kmp_sched, __kmp_chunk) and
3073 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults
3074 // independently. So one can get the updated schedule here.
3075
3076 kmp_r_sched_t r_sched;
3077
3078 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
3079 // __kmp_guided. __kmp_sched should keep original value, so that user can set
3080 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in
3081 // different roots (even in OMP 2.5)
3082 enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched)(enum sched_type)( (__kmp_sched) & ~(kmp_sch_modifier_nonmonotonic
| kmp_sch_modifier_monotonic))
;
3083 enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched)((enum sched_type)( (__kmp_sched) & (kmp_sch_modifier_nonmonotonic
| kmp_sch_modifier_monotonic)))
;
3084 if (s == kmp_sch_static) {
3085 // replace STATIC with more detailed schedule (balanced or greedy)
3086 r_sched.r_sched_type = __kmp_static;
3087 } else if (s == kmp_sch_guided_chunked) {
3088 // replace GUIDED with more detailed schedule (iterative or analytical)
3089 r_sched.r_sched_type = __kmp_guided;
3090 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
3091 r_sched.r_sched_type = __kmp_sched;
3092 }
3093 SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers)(r_sched.r_sched_type = (enum sched_type)((kmp_int32)r_sched.
r_sched_type | (kmp_int32)sched_modifiers))
;
3094
3095 if (__kmp_chunk < KMP_DEFAULT_CHUNK1) {
3096 // __kmp_chunk may be wrong here (if it was not ever set)
3097 r_sched.chunk = KMP_DEFAULT_CHUNK1;
3098 } else {
3099 r_sched.chunk = __kmp_chunk;
3100 }
3101
3102 return r_sched;
3103}
3104
3105/* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
3106 at least argc number of *t_argv entries for the requested team. */
3107static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) {
3108
3109 KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 3109); }
;
3110 if (!realloc || argc > team->t.t_max_argc) {
3111
3112 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
3113 "current entries=%d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
3114 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
;
3115 /* if previously allocated heap space for args, free them */
3116 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3117 __kmp_free((void *)team->t.t_argv)___kmp_free(((void *)team->t.t_argv), "openmp/runtime/src/kmp_runtime.cpp"
, 3117)
;
3118
3119 if (argc <= KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
) {
3120 /* use unused space in the cache line for arguments */
3121 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
;
3122 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3123 "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3124 team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
;
3125 team->t.t_argv = &team->t.t_inline_argv[0];
3126 if (__kmp_storage_map) {
3127 __kmp_print_storage_map_gtid(
3128 -1, &team->t.t_inline_argv[0],
3129 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
],
3130 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
), "team_%d.t_inline_argv",
3131 team->t.t_id);
3132 }
3133 } else {
3134 /* allocate space for arguments in the heap */
3135 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES100 >> 1))
3136 ? KMP_MIN_MALLOC_ARGV_ENTRIES100
3137 : 2 * argc;
3138 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3139 "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3140 team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
;
3141 team->t.t_argv =
3142 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc)___kmp_page_allocate((sizeof(void *) * team->t.t_max_argc)
, "openmp/runtime/src/kmp_runtime.cpp", 3142)
;
3143 if (__kmp_storage_map) {
3144 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3145 &team->t.t_argv[team->t.t_max_argc],
3146 sizeof(void *) * team->t.t_max_argc,
3147 "team_%d.t_argv", team->t.t_id);
3148 }
3149 }
3150 }
3151}
3152
3153static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) {
3154 int i;
3155 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3156 team->t.t_threads =
3157 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth)___kmp_allocate((sizeof(kmp_info_t *) * max_nth), "openmp/runtime/src/kmp_runtime.cpp"
, 3157)
;
3158 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "openmp/runtime/src/kmp_runtime.cpp", 3159)
3159 sizeof(dispatch_shared_info_t) * num_disp_buff)___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "openmp/runtime/src/kmp_runtime.cpp", 3159)
;
3160 team->t.t_dispatch =
3161 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth)___kmp_allocate((sizeof(kmp_disp_t) * max_nth), "openmp/runtime/src/kmp_runtime.cpp"
, 3161)
;
3162 team->t.t_implicit_task_taskdata =
3163 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth)___kmp_allocate((sizeof(kmp_taskdata_t) * max_nth), "openmp/runtime/src/kmp_runtime.cpp"
, 3163)
;
3164 team->t.t_max_nproc = max_nth;
3165
3166 /* setup dispatch buffers */
3167 for (i = 0; i < num_disp_buff; ++i) {
3168 team->t.t_disp_buffer[i].buffer_index = i;
3169 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3170 }
3171}
3172
3173static void __kmp_free_team_arrays(kmp_team_t *team) {
3174 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */
3175 int i;
3176 for (i = 0; i < team->t.t_max_nproc; ++i) {
3177 if (team->t.t_dispatch[i].th_disp_buffer != NULL__null) {
3178 __kmp_free(team->t.t_dispatch[i].th_disp_buffer)___kmp_free((team->t.t_dispatch[i].th_disp_buffer), "openmp/runtime/src/kmp_runtime.cpp"
, 3178)
;
3179 team->t.t_dispatch[i].th_disp_buffer = NULL__null;
3180 }
3181 }
3182#if KMP_USE_HIER_SCHED0
3183 __kmp_dispatch_free_hierarchies(team);
3184#endif
3185 __kmp_free(team->t.t_threads)___kmp_free((team->t.t_threads), "openmp/runtime/src/kmp_runtime.cpp"
, 3185)
;
3186 __kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "openmp/runtime/src/kmp_runtime.cpp"
, 3186)
;
3187 __kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "openmp/runtime/src/kmp_runtime.cpp"
, 3187)
;
3188 __kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "openmp/runtime/src/kmp_runtime.cpp"
, 3188)
;
3189 team->t.t_threads = NULL__null;
3190 team->t.t_disp_buffer = NULL__null;
3191 team->t.t_dispatch = NULL__null;
3192 team->t.t_implicit_task_taskdata = 0;
3193}
3194
3195static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
3196 kmp_info_t **oldThreads = team->t.t_threads;
3197
3198 __kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "openmp/runtime/src/kmp_runtime.cpp"
, 3198)
;
3199 __kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "openmp/runtime/src/kmp_runtime.cpp"
, 3199)
;
3200 __kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "openmp/runtime/src/kmp_runtime.cpp"
, 3200)
;
3201 __kmp_allocate_team_arrays(team, max_nth);
3202
3203 KMP_MEMCPYmemcpy(team->t.t_threads, oldThreads,
3204 team->t.t_nproc * sizeof(kmp_info_t *));
3205
3206 __kmp_free(oldThreads)___kmp_free((oldThreads), "openmp/runtime/src/kmp_runtime.cpp"
, 3206)
;
3207}
3208
3209static kmp_internal_control_t __kmp_get_global_icvs(void) {
3210
3211 kmp_r_sched_t r_sched =
3212 __kmp_get_schedule_global(); // get current state of scheduling globals
3213
3214 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0)if (!(__kmp_nested_proc_bind.used > 0)) { __kmp_debug_assert
("__kmp_nested_proc_bind.used > 0", "openmp/runtime/src/kmp_runtime.cpp"
, 3214); }
;
3215
3216 kmp_internal_control_t g_icvs = {
3217 0, // int serial_nesting_level; //corresponds to value of th_team_serialized
3218 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
3219 // adjustment of threads (per thread)
3220 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
3221 // whether blocktime is explicitly set
3222 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
3223#if KMP_USE_MONITOR
3224 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
3225// intervals
3226#endif
3227 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for
3228 // next parallel region (per thread)
3229 // (use a max ub on value if __kmp_parallel_initialize not called yet)
3230 __kmp_cg_max_nth, // int thread_limit;
3231 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control
3232 // for max_active_levels
3233 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule
3234 // {sched,chunk} pair
3235 __kmp_nested_proc_bind.bind_types[0],
3236 __kmp_default_device,
3237 NULL__null // struct kmp_internal_control *next;
3238 };
3239
3240 return g_icvs;
3241}
3242
3243static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) {
3244
3245 kmp_internal_control_t gx_icvs;
3246 gx_icvs.serial_nesting_level =
3247 0; // probably =team->t.t_serial like in save_inter_controls
3248 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3249 gx_icvs.next = NULL__null;
3250
3251 return gx_icvs;
3252}
3253
3254static void __kmp_initialize_root(kmp_root_t *root) {
3255 int f;
3256 kmp_team_t *root_team;
3257 kmp_team_t *hot_team;
3258 int hot_team_max_nth;
3259 kmp_r_sched_t r_sched =
3260 __kmp_get_schedule_global(); // get current state of scheduling globals
3261 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3262 KMP_DEBUG_ASSERT(root)if (!(root)) { __kmp_debug_assert("root", "openmp/runtime/src/kmp_runtime.cpp"
, 3262); }
;
3263 KMP_ASSERT(!root->r.r_begin)if (!(!root->r.r_begin)) { __kmp_debug_assert("!root->r.r_begin"
, "openmp/runtime/src/kmp_runtime.cpp", 3263); }
;
3264
3265 /* setup the root state structure */
3266 __kmp_init_lock(&root->r.r_begin_lock);
3267 root->r.r_begin = FALSE0;
3268 root->r.r_active = FALSE0;
3269 root->r.r_in_parallel = 0;
3270 root->r.r_blocktime = __kmp_dflt_blocktime;
3271#if KMP_AFFINITY_SUPPORTED1
3272 root->r.r_affinity_assigned = FALSE0;
3273#endif
3274
3275 /* setup the root team for this task */
3276 /* allocate the root team structure */
3277 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before root_team\n"
); }
;
3278
3279 root_team =
3280 __kmp_allocate_team(root,
3281 1, // new_nproc
3282 1, // max_nproc
3283#if OMPT_SUPPORT1
3284 ompt_data_none{0}, // root parallel id
3285#endif
3286 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3287 0 // argc
3288 USE_NESTED_HOT_ARG(NULL), __null // primary thread is unknown
3289 );
3290#if USE_DEBUGGER0
3291 // Non-NULL value should be assigned to make the debugger display the root
3292 // team.
3293 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0))((root_team->t.t_pkfn)) = (((microtask_t)(~0)));
3294#endif
3295
3296 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after root_team = %p\n"
, root_team); }
;
3297
3298 root->r.r_root_team = root_team;
3299 root_team->t.t_control_stack_top = NULL__null;
3300
3301 /* initialize root team */
3302 root_team->t.t_threads[0] = NULL__null;
3303 root_team->t.t_nproc = 1;
3304 root_team->t.t_serialized = 1;
3305 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3306 root_team->t.t_sched.sched = r_sched.sched;
3307 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3308 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3309 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3310 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
;
3311
3312 /* setup the hot team for this task */
3313 /* allocate the hot team structure */
3314 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before hot_team\n"
); }
;
3315
3316 hot_team =
3317 __kmp_allocate_team(root,
3318 1, // new_nproc
3319 __kmp_dflt_team_nth_ub * 2, // max_nproc
3320#if OMPT_SUPPORT1
3321 ompt_data_none{0}, // root parallel id
3322#endif
3323 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3324 0 // argc
3325 USE_NESTED_HOT_ARG(NULL), __null // primary thread is unknown
3326 );
3327 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after hot_team = %p\n"
, hot_team); }
;
3328
3329 root->r.r_hot_team = hot_team;
3330 root_team->t.t_control_stack_top = NULL__null;
3331
3332 /* first-time initialization */
3333 hot_team->t.t_parent = root_team;
3334
3335 /* initialize hot team */
3336 hot_team_max_nth = hot_team->t.t_max_nproc;
3337 for (f = 0; f < hot_team_max_nth; ++f) {
3338 hot_team->t.t_threads[f] = NULL__null;
3339 }
3340 hot_team->t.t_nproc = 1;
3341 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3342 hot_team->t.t_sched.sched = r_sched.sched;
3343 hot_team->t.t_size_changed = 0;
3344}
3345
3346#ifdef KMP_DEBUG1
3347
3348typedef struct kmp_team_list_item {
3349 kmp_team_p const *entry;
3350 struct kmp_team_list_item *next;
3351} kmp_team_list_item_t;
3352typedef kmp_team_list_item_t *kmp_team_list_t;
3353
3354static void __kmp_print_structure_team_accum( // Add team to list of teams.
3355 kmp_team_list_t list, // List of teams.
3356 kmp_team_p const *team // Team to add.
3357) {
3358
3359 // List must terminate with item where both entry and next are NULL.
3360 // Team is added to the list only once.
3361 // List is sorted in ascending order by team id.
3362 // Team id is *not* a key.
3363
3364 kmp_team_list_t l;
3365
3366 KMP_DEBUG_ASSERT(list != NULL)if (!(list != __null)) { __kmp_debug_assert("list != __null",
"openmp/runtime/src/kmp_runtime.cpp", 3366); }
;
3367 if (team == NULL__null) {
3368 return;
3369 }
3370
3371 __kmp_print_structure_team_accum(list, team->t.t_parent);
3372 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3373
3374 // Search list for the team.
3375 l = list;
3376 while (l->next != NULL__null && l->entry != team) {
3377 l = l->next;
3378 }
3379 if (l->next != NULL__null) {
3380 return; // Team has been added before, exit.
3381 }
3382
3383 // Team is not found. Search list again for insertion point.
3384 l = list;
3385 while (l->next != NULL__null && l->entry->t.t_id <= team->t.t_id) {
3386 l = l->next;
3387 }
3388
3389 // Insert team.
3390 {
3391 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(malloc(sizeof(kmp_team_list_item_t))
3392 sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
3393 *item = *l;
3394 l->entry = team;
3395 l->next = item;
3396 }
3397}
3398
3399static void __kmp_print_structure_team(char const *title, kmp_team_p const *team
3400
3401) {
3402 __kmp_printf("%s", title);
3403 if (team != NULL__null) {
3404 __kmp_printf("%2x %p\n", team->t.t_id, team);
3405 } else {
3406 __kmp_printf(" - (nil)\n");
3407 }
3408}
3409
3410static void __kmp_print_structure_thread(char const *title,
3411 kmp_info_p const *thread) {
3412 __kmp_printf("%s", title);
3413 if (thread != NULL__null) {
3414 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3415 } else {
3416 __kmp_printf(" - (nil)\n");
3417 }
3418}
3419
3420void __kmp_print_structure(void) {
3421
3422 kmp_team_list_t list;
3423
3424 // Initialize list of teams.
3425 list =
3426 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
3427 list->entry = NULL__null;
3428 list->next = NULL__null;
3429
3430 __kmp_printf("\n------------------------------\nGlobal Thread "
3431 "Table\n------------------------------\n");
3432 {
3433 int gtid;
3434 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3435 __kmp_printf("%2d", gtid);
3436 if (__kmp_threads != NULL__null) {
3437 __kmp_printf(" %p", __kmp_threads[gtid]);
3438 }
3439 if (__kmp_root != NULL__null) {
3440 __kmp_printf(" %p", __kmp_root[gtid]);
3441 }
3442 __kmp_printf("\n");
3443 }
3444 }
3445
3446 // Print out __kmp_threads array.
3447 __kmp_printf("\n------------------------------\nThreads\n--------------------"
3448 "----------\n");
3449 if (__kmp_threads != NULL__null) {
3450 int gtid;
3451 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3452 kmp_info_t const *thread = __kmp_threads[gtid];
3453 if (thread != NULL__null) {
3454 __kmp_printf("GTID %2d %p:\n", gtid, thread);
3455 __kmp_printf(" Our Root: %p\n", thread->th.th_root);
3456 __kmp_print_structure_team(" Our Team: ", thread->th.th_team);
3457 __kmp_print_structure_team(" Serial Team: ",
3458 thread->th.th_serial_team);
3459 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc);
3460 __kmp_print_structure_thread(" Primary: ",
3461 thread->th.th_team_master);
3462 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized);
3463 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc);
3464 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3465 __kmp_print_structure_thread(" Next in pool: ",
3466 thread->th.th_next_pool);
3467 __kmp_printf("\n");
3468 __kmp_print_structure_team_accum(list, thread->th.th_team);
3469 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3470 }
3471 }
3472 } else {
3473 __kmp_printf("Threads array is not allocated.\n");
3474 }
3475
3476 // Print out __kmp_root array.
3477 __kmp_printf("\n------------------------------\nUbers\n----------------------"
3478 "--------\n");
3479 if (__kmp_root != NULL__null) {
3480 int gtid;
3481 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3482 kmp_root_t const *root = __kmp_root[gtid];
3483 if (root != NULL__null) {
3484 __kmp_printf("GTID %2d %p:\n", gtid, root);
3485 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team);
3486 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team);
3487 __kmp_print_structure_thread(" Uber Thread: ",
3488 root->r.r_uber_thread);
3489 __kmp_printf(" Active?: %2d\n", root->r.r_active);
3490 __kmp_printf(" In Parallel: %2d\n",
3491 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)(&root->r.r_in_parallel)->load(std::memory_order_relaxed
)
);
3492 __kmp_printf("\n");
3493 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3494 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3495 }
3496 }
3497 } else {
3498 __kmp_printf("Ubers array is not allocated.\n");
3499 }
3500
3501 __kmp_printf("\n------------------------------\nTeams\n----------------------"
3502 "--------\n");
3503 while (list->next != NULL__null) {
3504 kmp_team_p const *team = list->entry;
3505 int i;
3506 __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
3507 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent);
3508 __kmp_printf(" Primary TID: %2d\n", team->t.t_master_tid);
3509 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc);
3510 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized);
3511 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc);
3512 for (i = 0; i < team->t.t_nproc; ++i) {
3513 __kmp_printf(" Thread %2d: ", i);
3514 __kmp_print_structure_thread("", team->t.t_threads[i]);
3515 }
3516 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool);
3517 __kmp_printf("\n");
3518 list = list->next;
3519 }
3520
3521 // Print out __kmp_thread_pool and __kmp_team_pool.
3522 __kmp_printf("\n------------------------------\nPools\n----------------------"
3523 "--------\n");
3524 __kmp_print_structure_thread("Thread pool: ",
3525 CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool));
3526 __kmp_print_structure_team("Team pool: ",
3527 CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool));
3528 __kmp_printf("\n");
3529
3530 // Free team list.
3531 while (list != NULL__null) {
3532 kmp_team_list_item_t *item = list;
3533 list = list->next;
3534 KMP_INTERNAL_FREE(item)free(item);
3535 }
3536}
3537
3538#endif
3539
3540//---------------------------------------------------------------------------
3541// Stuff for per-thread fast random number generator
3542// Table of primes
3543static const unsigned __kmp_primes[] = {
3544 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3545 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3546 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3547 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3548 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3549 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3550 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3551 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3552 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3553 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3554 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3555
3556//---------------------------------------------------------------------------
3557// __kmp_get_random: Get a random number using a linear congruential method.
3558unsigned short __kmp_get_random(kmp_info_t *thread) {
3559 unsigned x = thread->th.th_x;
3560 unsigned short r = (unsigned short)(x >> 16);
3561
3562 thread->th.th_x = x * thread->th.th_a + 1;
3563
3564 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); }
3565 thread->th.th_info.ds.ds_tid, r))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); }
;
3566
3567 return r;
3568}
3569//--------------------------------------------------------
3570// __kmp_init_random: Initialize a random number generator
3571void __kmp_init_random(kmp_info_t *thread) {
3572 unsigned seed = thread->th.th_info.ds.ds_tid;
3573
3574 thread->th.th_a =
3575 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))];
3576 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3577 KA_TRACE(30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); }
3578 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); }
;
3579}
3580
3581#if KMP_OS_WINDOWS0
3582/* reclaim array entries for root threads that are already dead, returns number
3583 * reclaimed */
3584static int __kmp_reclaim_dead_roots(void) {
3585 int i, r = 0;
3586
3587 for (i = 0; i < __kmp_threads_capacity; ++i) {
3588 if (KMP_UBER_GTID(i) &&
3589 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])((void *)(__kmp_threads[i]))) &&
3590 !__kmp_root[i]
3591 ->r.r_active) { // AC: reclaim only roots died in non-active state
3592 r += __kmp_unregister_root_other_thread(i);
3593 }
3594 }
3595 return r;
3596}
3597#endif
3598
3599/* This function attempts to create free entries in __kmp_threads and
3600 __kmp_root, and returns the number of free entries generated.
3601
3602 For Windows* OS static library, the first mechanism used is to reclaim array
3603 entries for root threads that are already dead.
3604
3605 On all platforms, expansion is attempted on the arrays __kmp_threads_ and
3606 __kmp_root, with appropriate update to __kmp_threads_capacity. Array
3607 capacity is increased by doubling with clipping to __kmp_tp_capacity, if
3608 threadprivate cache array has been created. Synchronization with
3609 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.
3610
3611 After any dead root reclamation, if the clipping value allows array expansion
3612 to result in the generation of a total of nNeed free slots, the function does
3613 that expansion. If not, nothing is done beyond the possible initial root
3614 thread reclamation.
3615
3616 If any argument is negative, the behavior is undefined. */
3617static int __kmp_expand_threads(int nNeed) {
3618 int added = 0;
3619 int minimumRequiredCapacity;
3620 int newCapacity;
3621 kmp_info_t **newThreads;
3622 kmp_root_t **newRoot;
3623
3624 // All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so
3625 // resizing __kmp_threads does not need additional protection if foreign
3626 // threads are present
3627
3628#if KMP_OS_WINDOWS0 && !KMP_DYNAMIC_LIB1
3629 /* only for Windows static library */
3630 /* reclaim array entries for root threads that are already dead */
3631 added = __kmp_reclaim_dead_roots();
3632
3633 if (nNeed) {
3634 nNeed -= added;
3635 if (nNeed < 0)
3636 nNeed = 0;
3637 }
3638#endif
3639 if (nNeed <= 0)
3640 return added;
3641
3642 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
3643 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
3644 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
3645 // > __kmp_max_nth in one of two ways:
3646 //
3647 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0]
3648 // may not be reused by another thread, so we may need to increase
3649 // __kmp_threads_capacity to __kmp_max_nth + 1.
3650 //
3651 // 2) New foreign root(s) are encountered. We always register new foreign
3652 // roots. This may cause a smaller # of threads to be allocated at
3653 // subsequent parallel regions, but the worker threads hang around (and
3654 // eventually go to sleep) and need slots in the __kmp_threads[] array.
3655 //
3656 // Anyway, that is the reason for moving the check to see if
3657 // __kmp_max_nth was exceeded into __kmp_reserve_threads()
3658 // instead of having it performed here. -BB
3659
3660 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity)if (!(__kmp_sys_max_nth >= __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_sys_max_nth >= __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 3660); }
;
3661
3662 /* compute expansion headroom to check if we can expand */
3663 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3664 /* possible expansion too small -- give up */
3665 return added;
3666 }
3667 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3668
3669 newCapacity = __kmp_threads_capacity;
3670 do {
3671 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3672 : __kmp_sys_max_nth;
3673 } while (newCapacity < minimumRequiredCapacity);
3674 newThreads = (kmp_info_t **)__kmp_allocate(___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "openmp/runtime/src/kmp_runtime.cpp", 3675
)
3675 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE)___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "openmp/runtime/src/kmp_runtime.cpp", 3675
)
;
3676 newRoot =
3677 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity);
3678 KMP_MEMCPYmemcpy(newThreads, __kmp_threads,
3679 __kmp_threads_capacity * sizeof(kmp_info_t *));
3680 KMP_MEMCPYmemcpy(newRoot, __kmp_root,
3681 __kmp_threads_capacity * sizeof(kmp_root_t *));
3682 // Put old __kmp_threads array on a list. Any ongoing references to the old
3683 // list will be valid. This list is cleaned up at library shutdown.
3684 kmp_old_threads_list_t *node =
3685 (kmp_old_threads_list_t *)__kmp_allocate(sizeof(kmp_old_threads_list_t))___kmp_allocate((sizeof(kmp_old_threads_list_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3685)
;
3686 node->threads = __kmp_threads;
3687 node->next = __kmp_old_threads_list;
3688 __kmp_old_threads_list = node;
3689
3690 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads;
3691 *(kmp_root_t * *volatile *)&__kmp_root = newRoot;
3692 added += newCapacity - __kmp_threads_capacity;
3693 *(volatile int *)&__kmp_threads_capacity = newCapacity;
3694
3695 if (newCapacity > __kmp_tp_capacity) {
3696 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3697 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3698 __kmp_threadprivate_resize_cache(newCapacity);
3699 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size
3700 *(volatile int *)&__kmp_tp_capacity = newCapacity;
3701 }
3702 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3703 }
3704
3705 return added;
3706}
3707
3708/* Register the current thread as a root thread and obtain our gtid. We must
3709 have the __kmp_initz_lock held at this point. Argument TRUE only if are the
3710 thread that calls from __kmp_do_serial_initialize() */
3711int __kmp_register_root(int initial_thread) {
3712 kmp_info_t *root_thread;
3713 kmp_root_t *root;
3714 int gtid;
3715 int capacity;
3716 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3717 KA_TRACE(20, ("__kmp_register_root: entered\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: entered\n"
); }
;
2
Assuming 'kmp_a_debug' is < 20
3
Taking false branch
3718 KMP_MB();
3719
3720 /* 2007-03-02:
3721 If initial thread did not invoke OpenMP RTL yet, and this thread is not an
3722 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not
3723 work as expected -- it may return false (that means there is at least one
3724 empty slot in __kmp_threads array), but it is possible the only free slot
3725 is #0, which is reserved for initial thread and so cannot be used for this
3726 one. Following code workarounds this bug.
3727
3728 However, right solution seems to be not reserving slot #0 for initial
3729 thread because:
3730 (1) there is no magic in slot #0,
3731 (2) we cannot detect initial thread reliably (the first thread which does
3732 serial initialization may be not a real initial thread).
3733 */
3734 capacity = __kmp_threads_capacity;
3735 if (!initial_thread
3.1
'initial_thread' is 1
&& TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
3736 --capacity;
3737 }
3738
3739 // If it is not for initializing the hidden helper team, we need to take
3740 // __kmp_hidden_helper_threads_num out of the capacity because it is included
3741 // in __kmp_threads_capacity.
3742 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
4
Assuming '__kmp_enable_hidden_helper' is 0
3743 capacity -= __kmp_hidden_helper_threads_num;
3744 }
3745
3746 /* see if there are too many threads */
3747 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
5
Assuming '__kmp_all_nth' is < 'capacity'
3748 if (__kmp_tp_cached) {
3749 __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread),
3750 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
)
,
3751 KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
3752 } else {
3753 __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads),
3754 __kmp_msg_null);
3755 }
3756 }
3757
3758 // When hidden helper task is enabled, __kmp_threads is organized as follows:
3759 // 0: initial thread, also a regular OpenMP thread.
3760 // [1, __kmp_hidden_helper_threads_num]: slots for hidden helper threads.
3761 // [__kmp_hidden_helper_threads_num + 1, __kmp_threads_capacity): slots for
3762 // regular OpenMP threads.
3763 if (TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
6
Assuming '__kmp_init_hidden_helper_threads' is 0
3764 // Find an available thread slot for hidden helper thread. Slots for hidden
3765 // helper threads start from 1 to __kmp_hidden_helper_threads_num.
3766 for (gtid = 1; TCR_PTR(__kmp_threads[gtid])((void *)(__kmp_threads[gtid])) != NULL__null &&
3767 gtid <= __kmp_hidden_helper_threads_num;
3768 gtid++)
3769 ;
3770 KMP_ASSERT(gtid <= __kmp_hidden_helper_threads_num)if (!(gtid <= __kmp_hidden_helper_threads_num)) { __kmp_debug_assert
("gtid <= __kmp_hidden_helper_threads_num", "openmp/runtime/src/kmp_runtime.cpp"
, 3770); }
;
3771 KA_TRACE(1, ("__kmp_register_root: found slot in threads array for "if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array for "
"hidden helper thread: T#%d\n", gtid); }
3772 "hidden helper thread: T#%d\n",if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array for "
"hidden helper thread: T#%d\n", gtid); }
3773 gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array for "
"hidden helper thread: T#%d\n", gtid); }
;
3774 } else {
3775 /* find an available thread slot */
3776 // Don't reassign the zero slot since we need that to only be used by
3777 // initial thread. Slots for hidden helper threads should also be skipped.
3778 if (initial_thread
6.1
'initial_thread' is 1
&& TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
7
Assuming the condition is false
8
Taking false branch
3779 gtid = 0;
3780 } else {
3781 for (gtid = __kmp_hidden_helper_threads_num + 1;
9
Loop condition is false. Execution continues on line 3785
3782 TCR_PTR(__kmp_threads[gtid])((void *)(__kmp_threads[gtid])) != NULL__null; gtid++)
3783 ;
3784 }
3785 KA_TRACE(if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); }
10
Assuming 'kmp_a_debug' is < 1
11
Taking false branch
3786 1, ("__kmp_register_root: found slot in threads array: T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); }
;
3787 KMP_ASSERT(gtid < __kmp_threads_capacity)if (!(gtid < __kmp_threads_capacity)) { __kmp_debug_assert
("gtid < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 3787); }
;
12
Assuming 'gtid' is >= '__kmp_threads_capacity'
13
Taking true branch
3788 }
3789
3790 /* update global accounting */
3791 __kmp_all_nth++;
3792 TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);
3793
3794 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
3795 // numbers of procs, and method #2 (keyed API call) for higher numbers.
3796 if (__kmp_adjust_gtid_mode) {
14
Assuming '__kmp_adjust_gtid_mode' is 0
3797 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3798 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
3799 TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
3800 }
3801 } else {
3802 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
3803 TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
3804 }
3805 }
3806 }
3807
3808#ifdef KMP_ADJUST_BLOCKTIME1
3809 /* Adjust blocktime to zero if necessary */
3810 /* Middle initialization might not have occurred yet */
3811 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
15
Assuming '__kmp_env_blocktime' is not equal to 0
3812 if (__kmp_nth > __kmp_avail_proc) {
3813 __kmp_zero_bt = TRUE(!0);
3814 }
3815 }
3816#endif /* KMP_ADJUST_BLOCKTIME */
3817
3818 /* setup this new hierarchy */
3819 if (!(root = __kmp_root[gtid])) {
16
Assuming 'root' is non-null
17
Taking false branch
3820 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t))___kmp_allocate((sizeof(kmp_root_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3820)
;
3821 KMP_DEBUG_ASSERT(!root->r.r_root_team)if (!(!root->r.r_root_team)) { __kmp_debug_assert("!root->r.r_root_team"
, "openmp/runtime/src/kmp_runtime.cpp", 3821); }
;
3822 }
3823
3824#if KMP_STATS_ENABLED0
3825 // Initialize stats as soon as possible (right after gtid assignment).
3826 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3827 __kmp_stats_thread_ptr->startLife();
3828 KMP_SET_THREAD_STATE(SERIAL_REGION)((void)0);
3829 KMP_INIT_PARTITIONED_TIMERS(OMP_serial)((void)0);
3830#endif
3831 __kmp_initialize_root(root);
3832
3833 /* setup new root thread structure */
3834 if (root->r.r_uber_thread) {
18
Assuming field 'r_uber_thread' is null
19
Taking false branch
3835 root_thread = root->r.r_uber_thread;
3836 } else {
3837 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3837)
;
3838 if (__kmp_storage_map) {
20
Assuming '__kmp_storage_map' is 0
21
Taking false branch
3839 __kmp_print_thread_storage_map(root_thread, gtid);
3840 }
3841 root_thread->th.th_info.ds.ds_gtid = gtid;
3842#if OMPT_SUPPORT1
3843 root_thread->th.ompt_thread_info.thread_data = ompt_data_none{0};
3844#endif
3845 root_thread->th.th_root = root;
3846 if (__kmp_env_consistency_check) {
22
Assuming '__kmp_env_consistency_check' is 0
23
Taking false branch
3847 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3848 }
3849#if USE_FAST_MEMORY3
3850 __kmp_initialize_fast_memory(root_thread);
3851#endif /* USE_FAST_MEMORY */
3852
3853#if KMP_USE_BGET1
3854 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL)if (!(root_thread->th.th_local.bget_data == __null)) { __kmp_debug_assert
("root_thread->th.th_local.bget_data == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 3854); }
;
24
Assuming the condition is true
25
Taking false branch
3855 __kmp_initialize_bget(root_thread);
3856#endif
3857 __kmp_init_random(root_thread); // Initialize random number generator
3858 }
3859
3860 /* setup the serial team held in reserve by the root thread */
3861 if (!root_thread->th.th_serial_team) {
26
Assuming field 'th_serial_team' is non-null
27
Taking false branch
3862 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3863 KF_TRACE(10, ("__kmp_register_root: before serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: before serial_team\n"
); }
;
3864 root_thread->th.th_serial_team = __kmp_allocate_team(
3865 root, 1, 1,
3866#if OMPT_SUPPORT1
3867 ompt_data_none{0}, // root parallel id
3868#endif
3869 proc_bind_default, &r_icvs, 0 USE_NESTED_HOT_ARG(NULL), __null);
3870 }
3871 KMP_ASSERT(root_thread->th.th_serial_team)if (!(root_thread->th.th_serial_team)) { __kmp_debug_assert
("root_thread->th.th_serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 3871); }
;
28
Assuming field 'th_serial_team' is non-null
29
Taking false branch
3872 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); }
30
Assuming 'kmp_f_debug' is < 10
31
Taking false branch
3873 root_thread->th.th_serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); }
;
3874
3875 /* drop root_thread into place */
3876 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread)((__kmp_threads[gtid])) = ((root_thread));
3877
3878 root->r.r_root_team->t.t_threads[0] = root_thread;
3879 root->r.r_hot_team->t.t_threads[0] = root_thread;
3880 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3881 // AC: the team created in reserve, not for execution (it is unused for now).
3882 root_thread->th.th_serial_team->t.t_serialized = 0;
3883 root->r.r_uber_thread = root_thread;
3884
3885 /* initialize the thread, get it ready to go */
3886 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
32
Calling '__kmp_initialize_info'
3887 TCW_4(__kmp_init_gtid, TRUE)(__kmp_init_gtid) = ((!0));
3888
3889 /* prepare the primary thread for get_gtid() */
3890 __kmp_gtid_set_specific(gtid);
3891
3892#if USE_ITT_BUILD1
3893 __kmp_itt_thread_name(gtid);
3894#endif /* USE_ITT_BUILD */
3895
3896#ifdef KMP_TDATA_GTID1
3897 __kmp_gtid = gtid;
3898#endif
3899 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3900 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid)if (!(__kmp_gtid_get_specific() == gtid)) { __kmp_debug_assert
("__kmp_gtid_get_specific() == gtid", "openmp/runtime/src/kmp_runtime.cpp"
, 3900); }
;
3901
3902 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3903 "plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3904 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3905 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3906 KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
;
3907 { // Initialize barrier data.
3908 int b;
3909 for (b = 0; b < bs_last_barrier; ++b) {
3910 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE0;
3911#if USE_DEBUGGER0
3912 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3913#endif
3914 }
3915 }
3916 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==if (!(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].
b_arrived == 0)) { __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 3917); }
3917 KMP_INIT_BARRIER_STATE)if (!(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].
b_arrived == 0)) { __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 3917); }
;
3918
3919#if KMP_AFFINITY_SUPPORTED1
3920 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
3921 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
3922 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
3923 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
3924#endif /* KMP_AFFINITY_SUPPORTED */
3925 root_thread->th.th_def_allocator = __kmp_def_allocator;
3926 root_thread->th.th_prev_level = 0;
3927 root_thread->th.th_prev_num_threads = 1;
3928
3929 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t))___kmp_allocate((sizeof(kmp_cg_root_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3929)
;
3930 tmp->cg_root = root_thread;
3931 tmp->cg_thread_limit = __kmp_cg_max_nth;
3932 tmp->cg_nthreads = 1;
3933 KA_TRACE(100, ("__kmp_register_root: Thread %p created node %p with"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_register_root: Thread %p created node %p with"
" cg_nthreads init to 1\n", root_thread, tmp); }
3934 " cg_nthreads init to 1\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_register_root: Thread %p created node %p with"
" cg_nthreads init to 1\n", root_thread, tmp); }
3935 root_thread, tmp))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_register_root: Thread %p created node %p with"
" cg_nthreads init to 1\n", root_thread, tmp); }
;
3936 tmp->up = NULL__null;
3937 root_thread->th.th_cg_roots = tmp;
3938
3939 __kmp_root_counter++;
3940
3941#if OMPT_SUPPORT1
3942 if (!initial_thread && ompt_enabled.enabled) {
3943
3944 kmp_info_t *root_thread = ompt_get_thread();
3945
3946 ompt_set_thread_state(root_thread, ompt_state_overhead);
3947
3948 if (ompt_enabled.ompt_callback_thread_begin) {
3949 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)ompt_callback_thread_begin_callback(
3950 ompt_thread_initial, __ompt_get_thread_data_internal());
3951 }
3952 ompt_data_t *task_data;
3953 ompt_data_t *parallel_data;
3954 __ompt_get_task_info_internal(0, NULL__null, &task_data, NULL__null, &parallel_data,
3955 NULL__null);
3956 if (ompt_enabled.ompt_callback_implicit_task) {
3957 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
3958 ompt_scope_begin, parallel_data, task_data, 1, 1, ompt_task_initial);
3959 }
3960
3961 ompt_set_thread_state(root_thread, ompt_state_work_serial);
3962 }
3963#endif
3964#if OMPD_SUPPORT1
3965 if (ompd_state & OMPD_ENABLE_BP0x1)
3966 ompd_bp_thread_begin();
3967#endif
3968
3969 KMP_MB();
3970 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3971
3972 return gtid;
3973}
3974
3975#if KMP_NESTED_HOT_TEAMS1
3976static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
3977 const int max_level) {
3978 int i, n, nth;
3979 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3980 if (!hot_teams || !hot_teams[level].hot_team) {
3981 return 0;
3982 }
3983 KMP_DEBUG_ASSERT(level < max_level)if (!(level < max_level)) { __kmp_debug_assert("level < max_level"
, "openmp/runtime/src/kmp_runtime.cpp", 3983); }
;
3984 kmp_team_t *team = hot_teams[level].hot_team;
3985 nth = hot_teams[level].hot_team_nth;
3986 n = nth - 1; // primary thread is not freed
3987 if (level < max_level - 1) {
3988 for (i = 0; i < nth; ++i) {
3989 kmp_info_t *th = team->t.t_threads[i];
3990 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3991 if (i > 0 && th->th.th_hot_teams) {
3992 __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "openmp/runtime/src/kmp_runtime.cpp"
, 3992)
;
3993 th->th.th_hot_teams = NULL__null;
3994 }
3995 }
3996 }
3997 __kmp_free_team(root, team, NULL__null);
3998 return n;
3999}
4000#endif
4001
4002// Resets a root thread and clear its root and hot teams.
4003// Returns the number of __kmp_threads entries directly and indirectly freed.
4004static int __kmp_reset_root(int gtid, kmp_root_t *root) {
4005 kmp_team_t *root_team = root->r.r_root_team;
4006 kmp_team_t *hot_team = root->r.r_hot_team;
4007 int n = hot_team->t.t_nproc;
4008 int i;
4009
4010 KMP_DEBUG_ASSERT(!root->r.r_active)if (!(!root->r.r_active)) { __kmp_debug_assert("!root->r.r_active"
, "openmp/runtime/src/kmp_runtime.cpp", 4010); }
;
4011
4012 root->r.r_root_team = NULL__null;
4013 root->r.r_hot_team = NULL__null;
4014 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team
4015 // before call to __kmp_free_team().
4016 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL), __null);
4017#if KMP_NESTED_HOT_TEAMS1
4018 if (__kmp_hot_teams_max_level >
4019 0) { // need to free nested hot teams and their threads if any
4020 for (i = 0; i < hot_team->t.t_nproc; ++i) {
4021 kmp_info_t *th = hot_team->t.t_threads[i];
4022 if (__kmp_hot_teams_max_level > 1) {
4023 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
4024 }
4025 if (th->th.th_hot_teams) {
4026 __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "openmp/runtime/src/kmp_runtime.cpp"
, 4026)
;
4027 th->th.th_hot_teams = NULL__null;
4028 }
4029 }
4030 }
4031#endif
4032 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL), __null);
4033
4034 // Before we can reap the thread, we need to make certain that all other
4035 // threads in the teams that had this root as ancestor have stopped trying to
4036 // steal tasks.
4037 if (__kmp_tasking_mode != tskm_immediate_exec) {
4038 __kmp_wait_to_unref_task_teams();
4039 }
4040
4041#if KMP_OS_WINDOWS0
4042 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
4043 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
4044 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPECif (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
4045 "\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
4046 (LPVOID) & (root->r.r_uber_thread->th),if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
4047 root->r.r_uber_thread->th.th_info.ds.ds_thread))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
;
4048 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
4049#endif /* KMP_OS_WINDOWS */
4050
4051#if OMPD_SUPPORT1
4052 if (ompd_state & OMPD_ENABLE_BP0x1)
4053 ompd_bp_thread_end();
4054#endif
4055
4056#if OMPT_SUPPORT1
4057 ompt_data_t *task_data;
4058 ompt_data_t *parallel_data;
4059 __ompt_get_task_info_internal(0, NULL__null, &task_data, NULL__null, &parallel_data,
4060 NULL__null);
4061 if (ompt_enabled.ompt_callback_implicit_task) {
4062 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
4063 ompt_scope_end, parallel_data, task_data, 0, 1, ompt_task_initial);
4064 }
4065 if (ompt_enabled.ompt_callback_thread_end) {
4066 ompt_callbacks.ompt_callback(ompt_callback_thread_end)ompt_callback_thread_end_callback(
4067 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
4068 }
4069#endif
4070
4071 TCW_4(__kmp_nth,(__kmp_nth) = (__kmp_nth - 1)
4072 __kmp_nth - 1)(__kmp_nth) = (__kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
4073 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;
4074 KA_TRACE(100, ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
" to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); }
4075 " to %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
" to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); }
4076 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
" to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); }
4077 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
" to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); }
;
4078 if (i == 1) {
4079 // need to free contention group structure
4080 KMP_DEBUG_ASSERT(root->r.r_uber_thread ==if (!(root->r.r_uber_thread == root->r.r_uber_thread->
th.th_cg_roots->cg_root)) { __kmp_debug_assert("root->r.r_uber_thread == root->r.r_uber_thread->th.th_cg_roots->cg_root"
, "openmp/runtime/src/kmp_runtime.cpp", 4081); }
4081 root->r.r_uber_thread->th.th_cg_roots->cg_root)if (!(root->r.r_uber_thread == root->r.r_uber_thread->
th.th_cg_roots->cg_root)) { __kmp_debug_assert("root->r.r_uber_thread == root->r.r_uber_thread->th.th_cg_roots->cg_root"
, "openmp/runtime/src/kmp_runtime.cpp", 4081); }
;
4082 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL)if (!(root->r.r_uber_thread->th.th_cg_roots->up == __null
)) { __kmp_debug_assert("root->r.r_uber_thread->th.th_cg_roots->up == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 4082); }
;
4083 __kmp_free(root->r.r_uber_thread->th.th_cg_roots)___kmp_free((root->r.r_uber_thread->th.th_cg_roots), "openmp/runtime/src/kmp_runtime.cpp"
, 4083)
;
4084 root->r.r_uber_thread->th.th_cg_roots = NULL__null;
4085 }
4086 __kmp_reap_thread(root->r.r_uber_thread, 1);
4087
4088 // We canot put root thread to __kmp_thread_pool, so we have to reap it
4089 // instead of freeing.
4090 root->r.r_uber_thread = NULL__null;
4091 /* mark root as no longer in use */
4092 root->r.r_begin = FALSE0;
4093
4094 return n;
4095}
4096
4097void __kmp_unregister_root_current_thread(int gtid) {
4098 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: enter T#%d\n"
, gtid); }
;
4099 /* this lock should be ok, since unregister_root_current_thread is never
4100 called during an abort, only during a normal close. furthermore, if you
4101 have the forkjoin lock, you should never try to get the initz lock */
4102 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
4103 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
4104 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
4105 "exiting T#%d\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
4106 gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
;
4107 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4108 return;
4109 }
4110 kmp_root_t *root = __kmp_root[gtid];
4111
4112 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])if (!(__kmp_threads && __kmp_threads[gtid])) { __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "openmp/runtime/src/kmp_runtime.cpp"
, 4112); }
;
4113 KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 4113); }
;
4114 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)if (!(root == __kmp_threads[gtid]->th.th_root)) { __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "openmp/runtime/src/kmp_runtime.cpp"
, 4114); }
;
4115 KMP_ASSERT(root->r.r_active == FALSE)if (!(root->r.r_active == 0)) { __kmp_debug_assert("root->r.r_active == FALSE"
, "openmp/runtime/src/kmp_runtime.cpp", 4115); }
;
4116
4117 KMP_MB();
4118
4119 kmp_info_t *thread = __kmp_threads[gtid];
4120 kmp_team_t *team = thread->th.th_team;
4121 kmp_task_team_t *task_team = thread->th.th_task_team;
4122
4123 // we need to wait for the proxy tasks before finishing the thread
4124 if (task_team != NULL__null && (task_team->tt.tt_found_proxy_tasks ||
4125 task_team->tt.tt_hidden_helper_task_encountered)) {
4126#if OMPT_SUPPORT1
4127 // the runtime is shutting down so we won't report any events
4128 thread->th.ompt_thread_info.state = ompt_state_undefined;
4129#endif
4130 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL), __null);
4131 }
4132
4133 __kmp_reset_root(gtid, root);
4134
4135 KMP_MB();
4136 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); }
4137 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); }
;
4138
4139 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4140}
4141
4142#if KMP_OS_WINDOWS0
4143/* __kmp_forkjoin_lock must be already held
4144 Unregisters a root thread that is not the current thread. Returns the number
4145 of __kmp_threads entries freed as a result. */
4146static int __kmp_unregister_root_other_thread(int gtid) {
4147 kmp_root_t *root = __kmp_root[gtid];
4148 int r;
4149
4150 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: enter T#%d\n"
, gtid); }
;
4151 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])if (!(__kmp_threads && __kmp_threads[gtid])) { __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "openmp/runtime/src/kmp_runtime.cpp"
, 4151); }
;
4152 KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 4152); }
;
4153 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)if (!(root == __kmp_threads[gtid]->th.th_root)) { __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "openmp/runtime/src/kmp_runtime.cpp"
, 4153); }
;
4154 KMP_ASSERT(root->r.r_active == FALSE)if (!(root->r.r_active == 0)) { __kmp_debug_assert("root->r.r_active == FALSE"
, "openmp/runtime/src/kmp_runtime.cpp", 4154); }
;
4155
4156 r = __kmp_reset_root(gtid, root);
4157 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); }
4158 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); }
;
4159 return r;
4160}
4161#endif
4162
4163#if KMP_DEBUG1
4164void __kmp_task_info() {
4165
4166 kmp_int32 gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
4167 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4168 kmp_info_t *this_thr = __kmp_threads[gtid];
4169 kmp_team_t *steam = this_thr->th.th_serial_team;
4170 kmp_team_t *team = this_thr->th.th_team;
4171
4172 __kmp_printf(
4173 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p "
4174 "ptask=%p\n",
4175 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4176 team->t.t_implicit_task_taskdata[tid].td_parent);
4177}
4178#endif // KMP_DEBUG
4179
4180/* TODO optimize with one big memclr, take out what isn't needed, split
4181 responsibility to workers as much as possible, and delay initialization of
4182 features as much as possible */
4183static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4184 int tid, int gtid) {
4185 /* this_thr->th.th_info.ds.ds_gtid is setup in
4186 kmp_allocate_thread/create_worker.
4187 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
4188 KMP_DEBUG_ASSERT(this_thr != NULL)if (!(this_thr != __null)) { __kmp_debug_assert("this_thr != __null"
, "openmp/runtime/src/kmp_runtime.cpp", 4188); }
;
33
Taking false branch
4189 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team)if (!(this_thr->th.th_serial_team)) { __kmp_debug_assert("this_thr->th.th_serial_team"
, "openmp/runtime/src/kmp_runtime.cpp", 4189); }
;
34
Assuming field 'th_serial_team' is non-null
35
Taking false branch
4190 KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 4190); }
;
36
Assuming 'team' is null
37
Taking true branch
4191 KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 4191); }
;
38
Dereference of null pointer
4192 KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 4192); }
;
4193 kmp_info_t *master = team->t.t_threads[0];
4194 KMP_DEBUG_ASSERT(master)if (!(master)) { __kmp_debug_assert("master", "openmp/runtime/src/kmp_runtime.cpp"
, 4194); }
;
4195 KMP_DEBUG_ASSERT(master->th.th_root)if (!(master->th.th_root)) { __kmp_debug_assert("master->th.th_root"
, "openmp/runtime/src/kmp_runtime.cpp", 4195); }
;
4196
4197 KMP_MB();
4198
4199 TCW_SYNC_PTR(this_thr->th.th_team, team)((this_thr->th.th_team)) = ((team));
4200
4201 this_thr->th.th_info.ds.ds_tid = tid;
4202 this_thr->th.th_set_nproc = 0;
4203 if (__kmp_tasking_mode != tskm_immediate_exec)
4204 // When tasking is possible, threads are not safe to reap until they are
4205 // done tasking; this will be set when tasking code is exited in wait
4206 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP0;
4207 else // no tasking --> always safe to reap
4208 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP1;
4209 this_thr->th.th_set_proc_bind = proc_bind_default;
4210#if KMP_AFFINITY_SUPPORTED1
4211 this_thr->th.th_new_place = this_thr->th.th_current_place;
4212#endif
4213 this_thr->th.th_root = master->th.th_root;
4214
4215 /* setup the thread's cache of the team structure */
4216 this_thr->th.th_team_nproc = team->t.t_nproc;
4217 this_thr->th.th_team_master = master;
4218 this_thr->th.th_team_serialized = team->t.t_serialized;
4219
4220 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata)if (!(team->t.t_implicit_task_taskdata)) { __kmp_debug_assert
("team->t.t_implicit_task_taskdata", "openmp/runtime/src/kmp_runtime.cpp"
, 4220); }
;
4221
4222 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
4223 tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
;
4224
4225 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4226 team, tid, TRUE(!0));
4227
4228 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
4229 tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
;
4230 // TODO: Initialize ICVs from parent; GEH - isn't that already done in
4231 // __kmp_initialize_team()?
4232
4233 /* TODO no worksharing in speculative threads */
4234 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4235
4236 this_thr->th.th_local.this_construct = 0;
4237
4238 if (!this_thr->th.th_pri_common) {
4239 this_thr->th.th_pri_common =
4240 (struct common_table *)__kmp_allocate(sizeof(struct common_table))___kmp_allocate((sizeof(struct common_table)), "openmp/runtime/src/kmp_runtime.cpp"
, 4240)
;
4241 if (__kmp_storage_map) {
4242 __kmp_print_storage_map_gtid(
4243 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4244 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
4245 }
4246 this_thr->th.th_pri_head = NULL__null;
4247 }
4248
4249 if (this_thr != master && // Primary thread's CG root is initialized elsewhere
4250 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set
4251 // Make new thread's CG root same as primary thread's
4252 KMP_DEBUG_ASSERT(master->th.th_cg_roots)if (!(master->th.th_cg_roots)) { __kmp_debug_assert("master->th.th_cg_roots"
, "openmp/runtime/src/kmp_runtime.cpp", 4252); }
;
4253 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
4254 if (tmp) {
4255 // worker changes CG, need to check if old CG should be freed
4256 int i = tmp->cg_nthreads--;
4257 KA_TRACE(100, ("__kmp_initialize_info: Thread %p decrement cg_nthreads"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p decrement cg_nthreads"
" on node %p of thread %p to %d\n", this_thr, tmp, tmp->cg_root
, tmp->cg_nthreads); }
4258 " on node %p of thread %p to %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p decrement cg_nthreads"
" on node %p of thread %p to %d\n", this_thr, tmp, tmp->cg_root
, tmp->cg_nthreads); }
4259 this_thr, tmp, tmp->cg_root, tmp->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p decrement cg_nthreads"
" on node %p of thread %p to %d\n", this_thr, tmp, tmp->cg_root
, tmp->cg_nthreads); }
;
4260 if (i == 1) {
4261 __kmp_free(tmp)___kmp_free((tmp), "openmp/runtime/src/kmp_runtime.cpp", 4261
)
; // last thread left CG --> free it
4262 }
4263 }
4264 this_thr->th.th_cg_roots = master->th.th_cg_roots;
4265 // Increment new thread's CG root's counter to add the new thread
4266 this_thr->th.th_cg_roots->cg_nthreads++;
4267 KA_TRACE(100, ("__kmp_initialize_info: Thread %p increment cg_nthreads on"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
" node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
4268 " node %p of thread %p to %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
" node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
4269 this_thr, this_thr->th.th_cg_roots,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
" node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
4270 this_thr->th.th_cg_roots->cg_root,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
" node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
4271 this_thr->th.th_cg_roots->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
" node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
;
4272 this_thr->th.th_current_task->td_icvs.thread_limit =
4273 this_thr->th.th_cg_roots->cg_thread_limit;
4274 }
4275
4276 /* Initialize dynamic dispatch */
4277 {
4278 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4279 // Use team max_nproc since this will never change for the team.
4280 size_t disp_size =
4281 sizeof(dispatch_private_info_t) *
4282 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4283 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); }
4284 team->t.t_max_nproc))if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); }
;
4285 KMP_ASSERT(dispatch)if (!(dispatch)) { __kmp_debug_assert("dispatch", "openmp/runtime/src/kmp_runtime.cpp"
, 4285); }
;
4286 KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 4286); }
;
4287 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid])if (!(dispatch == &team->t.t_dispatch[tid])) { __kmp_debug_assert
("dispatch == &team->t.t_dispatch[tid]", "openmp/runtime/src/kmp_runtime.cpp"
, 4287); }
;
4288
4289 dispatch->th_disp_index = 0;
4290 dispatch->th_doacross_buf_idx = 0;
4291 if (!dispatch->th_disp_buffer) {
4292 dispatch->th_disp_buffer =
4293 (dispatch_private_info_t *)__kmp_allocate(disp_size)___kmp_allocate((disp_size), "openmp/runtime/src/kmp_runtime.cpp"
, 4293)
;
4294
4295 if (__kmp_storage_map) {
4296 __kmp_print_storage_map_gtid(
4297 gtid, &dispatch->th_disp_buffer[0],
4298 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4299 ? 1
4300 : __kmp_dispatch_num_buffers],
4301 disp_size,
4302 "th_%d.th_dispatch.th_disp_buffer "
4303 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4304 gtid, team->t.t_id, gtid);
4305 }
4306 } else {
4307 memset(&dispatch->th_disp_buffer[0], '\0', disp_size);
4308 }
4309
4310 dispatch->th_dispatch_pr_current = 0;
4311 dispatch->th_dispatch_sh_current = 0;
4312
4313 dispatch->th_deo_fcn = 0; /* ORDERED */
4314 dispatch->th_dxo_fcn = 0; /* END ORDERED */
4315 }
4316
4317 this_thr->th.th_next_pool = NULL__null;
4318
4319 if (!this_thr->th.th_task_state_memo_stack) {
4320 size_t i;
4321 this_thr->th.th_task_state_memo_stack =
4322 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8))___kmp_allocate((4 * sizeof(kmp_uint8)), "openmp/runtime/src/kmp_runtime.cpp"
, 4322)
;
4323 this_thr->th.th_task_state_top = 0;
4324 this_thr->th.th_task_state_stack_sz = 4;
4325 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4326 ++i) // zero init the stack
4327 this_thr->th.th_task_state_memo_stack[i] = 0;
4328 }
4329
4330 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here)if (!(!this_thr->th.th_spin_here)) { __kmp_debug_assert("!this_thr->th.th_spin_here"
, "openmp/runtime/src/kmp_runtime.cpp", 4330); }
;
4331 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0)if (!(this_thr->th.th_next_waiting == 0)) { __kmp_debug_assert
("this_thr->th.th_next_waiting == 0", "openmp/runtime/src/kmp_runtime.cpp"
, 4331); }
;
4332
4333 KMP_MB();
4334}
4335
4336/* allocate a new thread for the requesting team. this is only called from
4337 within a forkjoin critical section. we will first try to get an available
4338 thread from the thread pool. if none is available, we will fork a new one
4339 assuming we are able to create a new one. this should be assured, as the
4340 caller should check on this first. */
4341kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4342 int new_tid) {
4343 kmp_team_t *serial_team;
4344 kmp_info_t *new_thr;
4345 int new_gtid;
4346
4347 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d\n"
, __kmp_get_global_thread_id()); }
;
4348 KMP_DEBUG_ASSERT(root && team)if (!(root && team)) { __kmp_debug_assert("root && team"
, "openmp/runtime/src/kmp_runtime.cpp", 4348); }
;
4349#if !KMP_NESTED_HOT_TEAMS1
4350 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()))if (!((0 == __kmp_tid_from_gtid((__kmp_get_global_thread_id()
))))) { __kmp_debug_assert("(0 == __kmp_tid_from_gtid((__kmp_get_global_thread_id())))"
, "openmp/runtime/src/kmp_runtime.cpp", 4350); }
;
4351#endif
4352 KMP_MB();
4353
4354 /* first, try to get one from the thread pool */
4355 if (__kmp_thread_pool) {
4356 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool);
4357 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;
4358 if (new_thr == __kmp_thread_pool_insert_pt) {
4359 __kmp_thread_pool_insert_pt = NULL__null;
4360 }
4361 TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
4362 __kmp_suspend_initialize_thread(new_thr);
4363 __kmp_lock_suspend_mx(new_thr);
4364 if (new_thr->th.th_active_in_pool == TRUE(!0)) {
4365 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE)if (!(new_thr->th.th_active == (!0))) { __kmp_debug_assert
("new_thr->th.th_active == (!0)", "openmp/runtime/src/kmp_runtime.cpp"
, 4365); }
;
4366 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth)(&__kmp_thread_pool_active_nth)->fetch_sub(1, std::memory_order_acq_rel
)
;
4367 new_thr->th.th_active_in_pool = FALSE0;
4368 }
4369 __kmp_unlock_suspend_mx(new_thr);
4370
4371 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); }
4372 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); }
;
4373 KMP_ASSERT(!new_thr->th.th_team)if (!(!new_thr->th.th_team)) { __kmp_debug_assert("!new_thr->th.th_team"
, "openmp/runtime/src/kmp_runtime.cpp", 4373); }
;
4374 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity)if (!(__kmp_nth < __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_nth < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 4374); }
;
4375
4376 /* setup the thread structure */
4377 __kmp_initialize_info(new_thr, team, new_tid,
4378 new_thr->th.th_info.ds.ds_gtid);
4379 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team)if (!(new_thr->th.th_serial_team)) { __kmp_debug_assert("new_thr->th.th_serial_team"
, "openmp/runtime/src/kmp_runtime.cpp", 4379); }
;
4380
4381 TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);
4382
4383 new_thr->th.th_task_state = 0;
4384 new_thr->th.th_task_state_top = 0;
4385 new_thr->th.th_task_state_stack_sz = 4;
4386
4387 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
4388 // Make sure pool thread has transitioned to waiting on own thread struct
4389 KMP_DEBUG_ASSERT(new_thr->th.th_used_in_team.load() == 0)if (!(new_thr->th.th_used_in_team.load() == 0)) { __kmp_debug_assert
("new_thr->th.th_used_in_team.load() == 0", "openmp/runtime/src/kmp_runtime.cpp"
, 4389); }
;
4390 // Thread activated in __kmp_allocate_team when increasing team size
4391 }
4392
4393#ifdef KMP_ADJUST_BLOCKTIME1
4394 /* Adjust blocktime back to zero if necessary */
4395 /* Middle initialization might not have occurred yet */
4396 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4397 if (__kmp_nth > __kmp_avail_proc) {
4398 __kmp_zero_bt = TRUE(!0);
4399 }
4400 }
4401#endif /* KMP_ADJUST_BLOCKTIME */
4402
4403#if KMP_DEBUG1
4404 // If thread entered pool via __kmp_free_thread, wait_flag should !=
4405 // KMP_BARRIER_PARENT_FLAG.
4406 int b;
4407 kmp_balign_t *balign = new_thr->th.th_bar;
4408 for (b = 0; b < bs_last_barrier; ++b)
4409 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 4409); }
;
4410#endif
4411
4412 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); }
4413 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); }
;
4414
4415 KMP_MB();
4416 return new_thr;
4417 }
4418
4419 /* no, well fork a new one */
4420 KMP_ASSERT(__kmp_nth == __kmp_all_nth)if (!(__kmp_nth == __kmp_all_nth)) { __kmp_debug_assert("__kmp_nth == __kmp_all_nth"
, "openmp/runtime/src/kmp_runtime.cpp", 4420); }
;
4421 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity)if (!(__kmp_all_nth < __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_all_nth < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 4421); }
;
4422
4423#if KMP_USE_MONITOR
4424 // If this is the first worker thread the RTL is creating, then also
4425 // launch the monitor thread. We try to do this as early as possible.
4426 if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
4427 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4428 if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
4429 KF_TRACE(10, ("before __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("before __kmp_create_monitor\n"
); }
;
4430 TCW_4(__kmp_init_monitor, 1)(__kmp_init_monitor) = (1);
4431 __kmp_create_monitor(&__kmp_monitor);
4432 KF_TRACE(10, ("after __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after __kmp_create_monitor\n"
); }
;
4433#if KMP_OS_WINDOWS0
4434 // AC: wait until monitor has started. This is a fix for CQ232808.
4435 // The reason is that if the library is loaded/unloaded in a loop with
4436 // small (parallel) work in between, then there is high probability that
4437 // monitor thread started after the library shutdown. At shutdown it is
4438 // too late to cope with the problem, because when the primary thread is
4439 // in DllMain (process detach) the monitor has no chances to start (it is
4440 // blocked), and primary thread has no means to inform the monitor that
4441 // the library has gone, because all the memory which the monitor can
4442 // access is going to be released/reset.
4443 while (TCR_4(__kmp_init_monitor)(__kmp_init_monitor) < 2) {
4444 KMP_YIELD(TRUE){ __kmp_x86_pause(); if (((!0)) && (((__kmp_use_yield
== 1) || (__kmp_use_yield == 2 && (((__kmp_nth) >
(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc))))))) __kmp_yield
(); }
;
4445 }
4446 KF_TRACE(10, ("after monitor thread has started\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after monitor thread has started\n"
); }
;
4447#endif
4448 }
4449 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4450 }
4451#endif
4452
4453 KMP_MB();
4454
4455 {
4456 int new_start_gtid = TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)
4457 ? 1
4458 : __kmp_hidden_helper_threads_num + 1;
4459
4460 for (new_gtid = new_start_gtid; TCR_PTR(__kmp_threads[new_gtid])((void *)(__kmp_threads[new_gtid])) != NULL__null;
4461 ++new_gtid) {
4462 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity)if (!(new_gtid < __kmp_threads_capacity)) { __kmp_debug_assert
("new_gtid < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 4462); }
;
4463 }
4464
4465 if (TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
4466 KMP_DEBUG_ASSERT(new_gtid <= __kmp_hidden_helper_threads_num)if (!(new_gtid <= __kmp_hidden_helper_threads_num)) { __kmp_debug_assert
("new_gtid <= __kmp_hidden_helper_threads_num", "openmp/runtime/src/kmp_runtime.cpp"
, 4466); }
;
4467 }
4468 }
4469
4470 /* allocate space for it. */
4471 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 4471)
;
4472
4473 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr)((__kmp_threads[new_gtid])) = ((new_thr));
4474
4475#if USE_ITT_BUILD1 && USE_ITT_NOTIFY1 && KMP_DEBUG1
4476 // suppress race conditions detection on synchronization flags in debug mode
4477 // this helps to analyze library internals eliminating false positives
4478 __itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(
4479 __itt_suppress_range, __itt_suppress_threading_errors0x000000ff,
4480 &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc));
4481 __itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(
4482 __itt_suppress_range, __itt_suppress_threading_errors0x000000ff,
4483 &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state));
4484#if KMP_OS_WINDOWS0
4485 __itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(
4486 __itt_suppress_range, __itt_suppress_threading_errors0x000000ff,
4487 &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init));
4488#else
4489 __itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
4490 __itt_suppress_threading_errors0x000000ff,
4491 &new_thr->th.th_suspend_init_count,
4492 sizeof(new_thr->th.th_suspend_init_count));
4493#endif
4494 // TODO: check if we need to also suppress b_arrived flags
4495 __itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
4496 __itt_suppress_threading_errors0x000000ff,
4497 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go)const_cast<kmp_uint64 *>(&new_thr->th.th_bar[0].
bb.b_go)
,
4498 sizeof(new_thr->th.th_bar[0].bb.b_go));
4499 __itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
4500 __itt_suppress_threading_errors0x000000ff,
4501 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go)const_cast<kmp_uint64 *>(&new_thr->th.th_bar[1].
bb.b_go)
,
4502 sizeof(new_thr->th.th_bar[1].bb.b_go));
4503 __itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
4504 __itt_suppress_threading_errors0x000000ff,
4505 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go)const_cast<kmp_uint64 *>(&new_thr->th.th_bar[2].
bb.b_go)
,
4506 sizeof(new_thr->th.th_bar[2].bb.b_go));
4507#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */
4508 if (__kmp_storage_map) {
4509 __kmp_print_thread_storage_map(new_thr, new_gtid);
4510 }
4511
4512 // add the reserve serialized team, initialized from the team's primary thread
4513 {
4514 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4515 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before th_serial/serial_team\n"
); }
;
4516 new_thr->th.th_serial_team = serial_team =
4517 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4518#if OMPT_SUPPORT1
4519 ompt_data_none{0}, // root parallel id
4520#endif
4521 proc_bind_default, &r_icvs,
4522 0 USE_NESTED_HOT_ARG(NULL), __null);
4523 }
4524 KMP_ASSERT(serial_team)if (!(serial_team)) { __kmp_debug_assert("serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 4524); }
;
4525 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for
4526 // execution (it is unused for now).
4527 serial_team->t.t_threads[0] = new_thr;
4528 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
4529 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
4530 new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
;
4531
4532 /* setup the thread structures */
4533 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4534
4535#if USE_FAST_MEMORY3
4536 __kmp_initialize_fast_memory(new_thr);
4537#endif /* USE_FAST_MEMORY */
4538
4539#if KMP_USE_BGET1
4540 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL)if (!(new_thr->th.th_local.bget_data == __null)) { __kmp_debug_assert
("new_thr->th.th_local.bget_data == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4540); }
;
4541 __kmp_initialize_bget(new_thr);
4542#endif
4543
4544 __kmp_init_random(new_thr); // Initialize random number generator
4545
4546 /* Initialize these only once when thread is grabbed for a team allocation */
4547 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
4548 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
4549 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
;
4550
4551 int b;
4552 kmp_balign_t *balign = new_thr->th.th_bar;
4553 for (b = 0; b < bs_last_barrier; ++b) {
4554 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE0;
4555 balign[b].bb.team = NULL__null;
4556 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING0;
4557 balign[b].bb.use_oncore_barrier = 0;
4558 }
4559
4560 TCW_PTR(new_thr->th.th_sleep_loc, NULL)((new_thr->th.th_sleep_loc)) = ((__null));
4561 new_thr->th.th_sleep_loc_type = flag_unset;
4562
4563 new_thr->th.th_spin_here = FALSE0;
4564 new_thr->th.th_next_waiting = 0;
4565#if KMP_OS_UNIX1
4566 new_thr->th.th_blocking = false;
4567#endif
4568
4569#if KMP_AFFINITY_SUPPORTED1
4570 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
4571 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
4572 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
4573 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
4574#endif
4575 new_thr->th.th_def_allocator = __kmp_def_allocator;
4576 new_thr->th.th_prev_level = 0;
4577 new_thr->th.th_prev_num_threads = 1;
4578
4579 TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
4580 new_thr->th.th_active_in_pool = FALSE0;
4581 TCW_4(new_thr->th.th_active, TRUE)(new_thr->th.th_active) = ((!0));
4582
4583 /* adjust the global counters */
4584 __kmp_all_nth++;
4585 __kmp_nth++;
4586
4587 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
4588 // numbers of procs, and method #2 (keyed API call) for higher numbers.
4589 if (__kmp_adjust_gtid_mode) {
4590 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4591 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
4592 TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
4593 }
4594 } else {
4595 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
4596 TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
4597 }
4598 }
4599 }
4600
4601#ifdef KMP_ADJUST_BLOCKTIME1
4602 /* Adjust blocktime back to zero if necessary */
4603 /* Middle initialization might not have occurred yet */
4604 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4605 if (__kmp_nth > __kmp_avail_proc) {
4606 __kmp_zero_bt = TRUE(!0);
4607 }
4608 }
4609#endif /* KMP_ADJUST_BLOCKTIME */
4610
4611 /* actually fork it and create the new worker thread */
4612 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); }
4613 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); }
;
4614 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4615 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); }
4616 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); }
;
4617
4618 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); }
4619 new_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); }
;
4620 KMP_MB();
4621 return new_thr;
4622}
4623
4624/* Reinitialize team for reuse.
4625 The hot team code calls this case at every fork barrier, so EPCC barrier
4626 test are extremely sensitive to changes in it, esp. writes to the team
4627 struct, which cause a cache invalidation in all threads.
4628 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4629static void __kmp_reinitialize_team(kmp_team_t *team,
4630 kmp_internal_control_t *new_icvs,
4631 ident_t *loc) {
4632 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
4633 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
;
4634 KMP_DEBUG_ASSERT(team && new_icvs)if (!(team && new_icvs)) { __kmp_debug_assert("team && new_icvs"
, "openmp/runtime/src/kmp_runtime.cpp", 4634); }
;
4635 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc)if (!((!(__kmp_init_parallel)) || new_icvs->nproc)) { __kmp_debug_assert
("(!(__kmp_init_parallel)) || new_icvs->nproc", "openmp/runtime/src/kmp_runtime.cpp"
, 4635); }
;
4636 KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
)
;
4637
4638 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID())if ((team->t.t_id) != ((~0))) (team->t.t_id) = ((~0));
4639 // Copy ICVs to the primary thread's implicit taskdata
4640 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE0);
4641 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4642
4643 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
4644 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
;
4645}
4646
4647/* Initialize the team data structure.
4648 This assumes the t_threads and t_max_nproc are already set.
4649 Also, we don't touch the arguments */
4650static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
4651 kmp_internal_control_t *new_icvs,
4652 ident_t *loc) {
4653 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: enter: team=%p\n"
, team); }
;
4654
4655 /* verify */
4656 KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 4656); }
;
4657 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc)if (!(new_nproc <= team->t.t_max_nproc)) { __kmp_debug_assert
("new_nproc <= team->t.t_max_nproc", "openmp/runtime/src/kmp_runtime.cpp"
, 4657); }
;
4658 KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 4658); }
;
4659 KMP_MB();
4660
4661 team->t.t_master_tid = 0; /* not needed */
4662 /* team->t.t_master_bar; not needed */
4663 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4664 team->t.t_nproc = new_nproc;
4665
4666 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */
4667 team->t.t_next_pool = NULL__null;
4668 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess
4669 * up hot team */
4670
4671 TCW_SYNC_PTR(team->t.t_pkfn, NULL)((team->t.t_pkfn)) = ((__null)); /* not needed */
4672 team->t.t_invoke = NULL__null; /* not needed */
4673
4674 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4675 team->t.t_sched.sched = new_icvs->sched.sched;
4676
4677#if KMP_ARCH_X860 || KMP_ARCH_X86_641
4678 team->t.t_fp_control_saved = FALSE0; /* not needed */
4679 team->t.t_x87_fpu_control_word = 0; /* not needed */
4680 team->t.t_mxcsr = 0; /* not needed */
4681#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4682
4683 team->t.t_construct = 0;
4684
4685 team->t.t_ordered.dt.t_value = 0;
4686 team->t.t_master_active = FALSE0;
4687
4688#ifdef KMP_DEBUG1
4689 team->t.t_copypriv_data = NULL__null; /* not necessary, but nice for debugging */
4690#endif
4691#if KMP_OS_WINDOWS0
4692 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
4693#endif
4694
4695 team->t.t_control_stack_top = NULL__null;
4696
4697 __kmp_reinitialize_team(team, new_icvs, loc);
4698
4699 KMP_MB();
4700 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: exit: team=%p\n"
, team); }
;
4701}
4702
4703#if (KMP_OS_LINUX1 || KMP_OS_FREEBSD0) && KMP_AFFINITY_SUPPORTED1
4704/* Sets full mask for thread and returns old mask, no changes to structures. */
4705static void
4706__kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4707 if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
4708 int status;
4709 if (old_mask != NULL__null) {
4710 status = __kmp_get_system_affinity(old_mask, TRUE)(old_mask)->get_system_affinity((!0));
4711 int error = errno(*__errno_location ());
4712 if (status != 0) {
4713 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError)__kmp_msg_format(kmp_i18n_msg_ChangeThreadAffMaskError), KMP_ERR(error)__kmp_msg_error_code(error),
4714 __kmp_msg_null);
4715 }
4716 }
4717 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE)(__kmp_affin_fullMask)->set_system_affinity((!0));
4718 }
4719}
4720#endif
4721
4722#if KMP_AFFINITY_SUPPORTED1
4723
4724// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
4725// It calculates the worker + primary thread's partition based upon the parent
4726// thread's partition, and binds each worker to a thread in their partition.
4727// The primary thread's partition should already include its current binding.
4728static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
4729 // Do not partition places for the hidden helper team
4730 if (KMP_HIDDEN_HELPER_TEAM(team)(team->t.t_threads[0] == __kmp_hidden_helper_main_thread))
4731 return;
4732 // Copy the primary thread's place partition to the team struct
4733 kmp_info_t *master_th = team->t.t_threads[0];
4734 KMP_DEBUG_ASSERT(master_th != NULL)if (!(master_th != __null)) { __kmp_debug_assert("master_th != __null"
, "openmp/runtime/src/kmp_runtime.cpp", 4734); }
;
4735 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4736 int first_place = master_th->th.th_first_place;
4737 int last_place = master_th->th.th_last_place;
4738 int masters_place = master_th->th.th_current_place;
4739 int num_masks = __kmp_affinity.num_masks;
4740 team->t.t_first_place = first_place;
4741 team->t.t_last_place = last_place;
4742
4743 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
4744 "bound to place %d partition = [%d,%d]\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
4745 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
4746 team->t.t_id, masters_place, first_place, last_place))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
;
4747
4748 switch (proc_bind) {
4749
4750 case proc_bind_default:
4751 // Serial teams might have the proc_bind policy set to proc_bind_default.
4752 // Not an issue -- we don't rebind primary thread for any proc_bind policy.
4753 KMP_DEBUG_ASSERT(team->t.t_nproc == 1)if (!(team->t.t_nproc == 1)) { __kmp_debug_assert("team->t.t_nproc == 1"
, "openmp/runtime/src/kmp_runtime.cpp", 4753); }
;
4754 break;
4755
4756 case proc_bind_primary: {
4757 int f;
4758 int n_th = team->t.t_nproc;
4759 for (f = 1; f < n_th; f++) {
4760 kmp_info_t *th = team->t.t_threads[f];
4761 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4761); }
;
4762 th->th.th_first_place = first_place;
4763 th->th.th_last_place = last_place;
4764 th->th.th_new_place = masters_place;
4765 if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
4766 team->t.t_display_affinity != 1) {
4767 team->t.t_display_affinity = 1;
4768 }
4769
4770 KA_TRACE(100, ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
4771 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
4772 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
4773 f, masters_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
;
4774 }
4775 } break;
4776
4777 case proc_bind_close: {
4778 int f;
4779 int n_th = team->t.t_nproc;
4780 int n_places;
4781 if (first_place <= last_place) {
4782 n_places = last_place - first_place + 1;
4783 } else {
4784 n_places = num_masks - first_place + last_place + 1;
4785 }
4786 if (n_th <= n_places) {
4787 int place = masters_place;
4788 for (f = 1; f < n_th; f++) {
4789 kmp_info_t *th = team->t.t_threads[f];
4790 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4790); }
;
4791
4792 if (place == last_place) {
4793 place = first_place;
4794 } else if (place == (num_masks - 1)) {
4795 place = 0;
4796 } else {
4797 place++;
4798 }
4799 th->th.th_first_place = first_place;
4800 th->th.th_last_place = last_place;
4801 th->th.th_new_place = place;
4802 if (__kmp_display_affinity && place != th->th.th_current_place &&
4803 team->t.t_display_affinity != 1) {
4804 team->t.t_display_affinity = 1;
4805 }
4806
4807 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
4808 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
4809 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
4810 team->t.t_id, f, place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
;
4811 }
4812 } else {
4813 int S, rem, gap, s_count;
4814 S = n_th / n_places;
4815 s_count = 0;
4816 rem = n_th - (S * n_places);
4817 gap = rem > 0 ? n_places / rem : n_places;
4818 int place = masters_place;
4819 int gap_ct = gap;
4820 for (f = 0; f < n_th; f++) {
4821 kmp_info_t *th = team->t.t_threads[f];
4822 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4822); }
;
4823
4824 th->th.th_first_place = first_place;
4825 th->th.th_last_place = last_place;
4826 th->th.th_new_place = place;
4827 if (__kmp_display_affinity && place != th->th.th_current_place &&
4828 team->t.t_display_affinity != 1) {
4829 team->t.t_display_affinity = 1;
4830 }
4831 s_count++;
4832
4833 if ((s_count == S) && rem && (gap_ct == gap)) {
4834 // do nothing, add an extra thread to place on next iteration
4835 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4836 // we added an extra thread to this place; move to next place
4837 if (place == last_place) {
4838 place = first_place;
4839 } else if (place == (num_masks - 1)) {
4840 place = 0;
4841 } else {
4842 place++;
4843 }
4844 s_count = 0;
4845 gap_ct = 1;
4846 rem--;
4847 } else if (s_count == S) { // place full; don't add extra
4848 if (place == last_place) {
4849 place = first_place;
4850 } else if (place == (num_masks - 1)) {
4851 place = 0;
4852 } else {
4853 place++;
4854 }
4855 gap_ct++;
4856 s_count = 0;
4857 }
4858
4859 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4860 ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4861 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4862 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4863 th->th.th_new_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
;
4864 }
4865 KMP_DEBUG_ASSERT(place == masters_place)if (!(place == masters_place)) { __kmp_debug_assert("place == masters_place"
, "openmp/runtime/src/kmp_runtime.cpp", 4865); }
;
4866 }
4867 } break;
4868
4869 case proc_bind_spread: {
4870 int f;
4871 int n_th = team->t.t_nproc;
4872 int n_places;
4873 int thidx;
4874 if (first_place <= last_place) {
4875 n_places = last_place - first_place + 1;
4876 } else {
4877 n_places = num_masks - first_place + last_place + 1;
4878 }
4879 if (n_th <= n_places) {
4880 int place = -1;
4881
4882 if (n_places != num_masks) {
4883 int S = n_places / n_th;
4884 int s_count, rem, gap, gap_ct;
4885
4886 place = masters_place;
4887 rem = n_places - n_th * S;
4888 gap = rem ? n_th / rem : 1;
4889 gap_ct = gap;
4890 thidx = n_th;
4891 if (update_master_only == 1)
4892 thidx = 1;
4893 for (f = 0; f < thidx; f++) {
4894 kmp_info_t *th = team->t.t_threads[f];
4895 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4895); }
;
4896
4897 th->th.th_first_place = place;
4898 th->th.th_new_place = place;
4899 if (__kmp_display_affinity && place != th->th.th_current_place &&
4900 team->t.t_display_affinity != 1) {
4901 team->t.t_display_affinity = 1;
4902 }
4903 s_count = 1;
4904 while (s_count < S) {
4905 if (place == last_place) {
4906 place = first_place;
4907 } else if (place == (num_masks - 1)) {
4908 place = 0;
4909 } else {
4910 place++;
4911 }
4912 s_count++;
4913 }
4914 if (rem && (gap_ct == gap)) {
4915 if (place == last_place) {
4916 place = first_place;
4917 } else if (place == (num_masks - 1)) {
4918 place = 0;
4919 } else {
4920 place++;
4921 }
4922 rem--;
4923 gap_ct = 0;
4924 }
4925 th->th.th_last_place = place;
4926 gap_ct++;
4927
4928 if (place == last_place) {
4929 place = first_place;
4930 } else if (place == (num_masks - 1)) {
4931 place = 0;
4932 } else {
4933 place++;
4934 }
4935
4936 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
4937 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
4938 "partition = [%d,%d], num_masks: %u\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
4939 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
4940 f, th->th.th_new_place, th->th.th_first_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
4941 th->th.th_last_place, num_masks))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
;
4942 }
4943 } else {
4944 /* Having uniform space of available computation places I can create
4945 T partitions of round(P/T) size and put threads into the first
4946 place of each partition. */
4947 double current = static_cast<double>(masters_place);
4948 double spacing =
4949 (static_cast<double>(n_places + 1) / static_cast<double>(n_th));
4950 int first, last;
4951 kmp_info_t *th;
4952
4953 thidx = n_th + 1;
4954 if (update_master_only == 1)
4955 thidx = 1;
4956 for (f = 0; f < thidx; f++) {
4957 first = static_cast<int>(current);
4958 last = static_cast<int>(current + spacing) - 1;
4959 KMP_DEBUG_ASSERT(last >= first)if (!(last >= first)) { __kmp_debug_assert("last >= first"
, "openmp/runtime/src/kmp_runtime.cpp", 4959); }
;
4960 if (first >= n_places) {
4961 if (masters_place) {
4962 first -= n_places;
4963 last -= n_places;
4964 if (first == (masters_place + 1)) {
4965 KMP_DEBUG_ASSERT(f == n_th)if (!(f == n_th)) { __kmp_debug_assert("f == n_th", "openmp/runtime/src/kmp_runtime.cpp"
, 4965); }
;
4966 first--;
4967 }
4968 if (last == masters_place) {
4969 KMP_DEBUG_ASSERT(f == (n_th - 1))if (!(f == (n_th - 1))) { __kmp_debug_assert("f == (n_th - 1)"
, "openmp/runtime/src/kmp_runtime.cpp", 4969); }
;
4970 last--;
4971 }
4972 } else {
4973 KMP_DEBUG_ASSERT(f == n_th)if (!(f == n_th)) { __kmp_debug_assert("f == n_th", "openmp/runtime/src/kmp_runtime.cpp"
, 4973); }
;
4974 first = 0;
4975 last = 0;
4976 }
4977 }
4978 if (last >= n_places) {
4979 last = (n_places - 1);
4980 }
4981 place = first;
4982 current += spacing;
4983 if (f < n_th) {
4984 KMP_DEBUG_ASSERT(0 <= first)if (!(0 <= first)) { __kmp_debug_assert("0 <= first", "openmp/runtime/src/kmp_runtime.cpp"
, 4984); }
;
4985 KMP_DEBUG_ASSERT(n_places > first)if (!(n_places > first)) { __kmp_debug_assert("n_places > first"
, "openmp/runtime/src/kmp_runtime.cpp", 4985); }
;
4986 KMP_DEBUG_ASSERT(0 <= last)if (!(0 <= last)) { __kmp_debug_assert("0 <= last", "openmp/runtime/src/kmp_runtime.cpp"
, 4986); }
;
4987 KMP_DEBUG_ASSERT(n_places > last)if (!(n_places > last)) { __kmp_debug_assert("n_places > last"
, "openmp/runtime/src/kmp_runtime.cpp", 4987); }
;
4988 KMP_DEBUG_ASSERT(last_place >= first_place)if (!(last_place >= first_place)) { __kmp_debug_assert("last_place >= first_place"
, "openmp/runtime/src/kmp_runtime.cpp", 4988); }
;
4989 th = team->t.t_threads[f];
4990 KMP_DEBUG_ASSERT(th)if (!(th)) { __kmp_debug_assert("th", "openmp/runtime/src/kmp_runtime.cpp"
, 4990); }
;
4991 th->th.th_first_place = first;
4992 th->th.th_new_place = place;
4993 th->th.th_last_place = last;
4994 if (__kmp_display_affinity && place != th->th.th_current_place &&
4995 team->t.t_display_affinity != 1) {
4996 team->t.t_display_affinity = 1;
4997 }
4998 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
4999 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
5000 "partition = [%d,%d], spacing = %.4f\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
5001 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
5002 team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
5003 th->th.th_first_place, th->th.th_last_place, spacing))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
;
5004 }
5005 }
5006 }
5007 KMP_DEBUG_ASSERT(update_master_only || place == masters_place)if (!(update_master_only || place == masters_place)) { __kmp_debug_assert
("update_master_only || place == masters_place", "openmp/runtime/src/kmp_runtime.cpp"
, 5007); }
;
5008 } else {
5009 int S, rem, gap, s_count;
5010 S = n_th / n_places;
5011 s_count = 0;
5012 rem = n_th - (S * n_places);
5013 gap = rem > 0 ? n_places / rem : n_places;
5014 int place = masters_place;
5015 int gap_ct = gap;
5016 thidx = n_th;
5017 if (update_master_only == 1)
5018 thidx = 1;
5019 for (f = 0; f < thidx; f++) {
5020 kmp_info_t *th = team->t.t_threads[f];
5021 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 5021); }
;
5022
5023 th->th.th_first_place = place;
5024 th->th.th_last_place = place;
5025 th->th.th_new_place = place;
5026 if (__kmp_display_affinity && place != th->th.th_current_place &&
5027 team->t.t_display_affinity != 1) {
5028 team->t.t_display_affinity = 1;
5029 }
5030 s_count++;
5031
5032 if ((s_count == S) && rem && (gap_ct == gap)) {
5033 // do nothing, add an extra thread to place on next iteration
5034 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
5035 // we added an extra thread to this place; move on to next place
5036 if (place == last_place) {
5037 place = first_place;
5038 } else if (place == (num_masks - 1)) {
5039 place = 0;
5040 } else {
5041 place++;
5042 }
5043 s_count = 0;
5044 gap_ct = 1;
5045 rem--;
5046 } else if (s_count == S) { // place is full; don't add extra thread
5047 if (place == last_place) {
5048 place = first_place;
5049 } else if (place == (num_masks - 1)) {
5050 place = 0;
5051 } else {
5052 place++;
5053 }
5054 gap_ct++;
5055 s_count = 0;
5056 }
5057
5058 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
5059 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
5060 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
5061 team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
5062 th->th.th_first_place, th->th.th_last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
;
5063 }
5064 KMP_DEBUG_ASSERT(update_master_only || place == masters_place)if (!(update_master_only || place == masters_place)) { __kmp_debug_assert
("update_master_only || place == masters_place", "openmp/runtime/src/kmp_runtime.cpp"
, 5064); }
;
5065 }
5066 } break;
5067
5068 default:
5069 break;
5070 }
5071
5072 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: exit T#%d\n"
, team->t.t_id); }
;
5073}
5074
5075#endif // KMP_AFFINITY_SUPPORTED
5076
5077/* allocate a new team data structure to use. take one off of the free pool if
5078 available */
5079kmp_team_t *
5080__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
5081#if OMPT_SUPPORT1
5082 ompt_data_t ompt_parallel_data,
5083#endif
5084 kmp_proc_bind_t new_proc_bind,
5085 kmp_internal_control_t *new_icvs,
5086 int argc USE_NESTED_HOT_ARG(kmp_info_t *master), kmp_info_t *master) {
5087 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team)((void)0);
5088 int f;
5089 kmp_team_t *team;
5090 int use_hot_team = !root->r.r_active;
5091 int level = 0;
5092 int do_place_partition = 1;
5093
5094 KA_TRACE(20, ("__kmp_allocate_team: called\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: called\n"
); }
;
5095 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0)if (!(new_nproc >= 1 && argc >= 0)) { __kmp_debug_assert
("new_nproc >= 1 && argc >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 5095); }
;
5096 KMP_DEBUG_ASSERT(max_nproc >= new_nproc)if (!(max_nproc >= new_nproc)) { __kmp_debug_assert("max_nproc >= new_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5096); }
;
5097 KMP_MB();
5098
5099#if KMP_NESTED_HOT_TEAMS1
5100 kmp_hot_team_ptr_t *hot_teams;
5101 if (master) {
5102 team = master->th.th_team;
5103 level = team->t.t_active_level;
5104 if (master->th.th_teams_microtask) { // in teams construct?
5105 if (master->th.th_teams_size.nteams > 1 &&
5106 ( // #teams > 1
5107 team->t.t_pkfn ==
5108 (microtask_t)__kmp_teams_master || // inner fork of the teams
5109 master->th.th_teams_level <
5110 team->t.t_level)) { // or nested parallel inside the teams
5111 ++level; // not increment if #teams==1, or for outer fork of the teams;
5112 // increment otherwise
5113 }
5114 // Do not perform the place partition if inner fork of the teams
5115 // Wait until nested parallel region encountered inside teams construct
5116 if ((master->th.th_teams_size.nteams == 1 &&
5117 master->th.th_teams_level >= team->t.t_level) ||
5118 (team->t.t_pkfn == (microtask_t)__kmp_teams_master))
5119 do_place_partition = 0;
5120 }
5121 hot_teams = master->th.th_hot_teams;
5122 if (level < __kmp_hot_teams_max_level && hot_teams &&
5123 hot_teams[level].hot_team) {
5124 // hot team has already been allocated for given level
5125 use_hot_team = 1;
5126 } else {
5127 use_hot_team = 0;
5128 }
5129 } else {
5130 // check we won't access uninitialized hot_teams, just in case
5131 KMP_DEBUG_ASSERT(new_nproc == 1)if (!(new_nproc == 1)) { __kmp_debug_assert("new_nproc == 1",
"openmp/runtime/src/kmp_runtime.cpp", 5131); }
;
5132 }
5133#endif
5134 // Optimization to use a "hot" team
5135 if (use_hot_team && new_nproc > 1) {
5136 KMP_DEBUG_ASSERT(new_nproc <= max_nproc)if (!(new_nproc <= max_nproc)) { __kmp_debug_assert("new_nproc <= max_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5136); }
;
5137#if KMP_NESTED_HOT_TEAMS1
5138 team = hot_teams[level].hot_team;
5139#else
5140 team = root->r.r_hot_team;
5141#endif
5142#if KMP_DEBUG1
5143 if (__kmp_tasking_mode != tskm_immediate_exec) {
5144 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
5145 "task_team[1] = %p before reinit\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
5146 team->t.t_task_team[0], team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
;
5147 }
5148#endif
5149
5150 if (team->t.t_nproc != new_nproc &&