Bug Summary

File:projects/openmp/runtime/src/kmp_runtime.cpp
Warning:line 2678, column 51
Dereference of null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name kmp_runtime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D omp_EXPORTS -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/projects/openmp/runtime/src -I /build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn345461/include -I /build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/i18n -I /build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/include/50 -I /build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/thirdparty/ittnotify -U NDEBUG -D _GNU_SOURCE -D _REENTRANT -D _FORTIFY_SOURCE=2 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -Wno-switch -Wno-missing-field-initializers -Wno-missing-braces -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/projects/openmp/runtime/src -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fno-rtti -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-10-27-211344-32123-1 -x c++ /build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp -faddrsig
1/*
2 * kmp_runtime.cpp -- KPTS runtime support library
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// The LLVM Compiler Infrastructure
8//
9// This file is dual licensed under the MIT and the University of Illinois Open
10// Source Licenses. See LICENSE.txt for details.
11//
12//===----------------------------------------------------------------------===//
13
14#include "kmp.h"
15#include "kmp_affinity.h"
16#include "kmp_atomic.h"
17#include "kmp_environment.h"
18#include "kmp_error.h"
19#include "kmp_i18n.h"
20#include "kmp_io.h"
21#include "kmp_itt.h"
22#include "kmp_settings.h"
23#include "kmp_stats.h"
24#include "kmp_str.h"
25#include "kmp_wait_release.h"
26#include "kmp_wrapper_getpid.h"
27#include "kmp_dispatch.h"
28#if KMP_USE_HIER_SCHED0
29#include "kmp_dispatch_hier.h"
30#endif
31
32#if OMPT_SUPPORT1
33#include "ompt-specific.h"
34#endif
35
36/* these are temporary issues to be dealt with */
37#define KMP_USE_PRCTL0 0
38
39#if KMP_OS_WINDOWS0
40#include <process.h>
41#endif
42
43#include "tsan_annotations.h"
44
45#if defined(KMP_GOMP_COMPAT)
46char const __kmp_version_alt_comp[] =
47 KMP_VERSION_PREFIX"\x00@(#) " "Intel(R) OMP " "alternative compiler support: yes";
48#endif /* defined(KMP_GOMP_COMPAT) */
49
50char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX"\x00@(#) " "Intel(R) OMP " "API version: "
51#if OMP_50_ENABLED(50 >= 50)
52 "5.0 (201611)";
53#elif OMP_45_ENABLED(50 >= 45)
54 "4.5 (201511)";
55#elif OMP_40_ENABLED(50 >= 40)
56 "4.0 (201307)";
57#else
58 "3.1 (201107)";
59#endif
60
61#ifdef KMP_DEBUG1
62char const __kmp_version_lock[] =
63 KMP_VERSION_PREFIX"\x00@(#) " "Intel(R) OMP " "lock type: run time selectable";
64#endif /* KMP_DEBUG */
65
66#define KMP_MIN(x, y)((x) < (y) ? (x) : (y)) ((x) < (y) ? (x) : (y))
67
68/* ------------------------------------------------------------------------ */
69
70#if KMP_USE_MONITOR
71kmp_info_t __kmp_monitor;
72#endif
73
74/* Forward declarations */
75
76void __kmp_cleanup(void);
77
78static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
79 int gtid);
80static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
81 kmp_internal_control_t *new_icvs,
82 ident_t *loc);
83#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
84static void __kmp_partition_places(kmp_team_t *team,
85 int update_master_only = 0);
86#endif
87static void __kmp_do_serial_initialize(void);
88void __kmp_fork_barrier(int gtid, int tid);
89void __kmp_join_barrier(int gtid);
90void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
91 kmp_internal_control_t *new_icvs, ident_t *loc);
92
93#ifdef USE_LOAD_BALANCE1
94static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
95#endif
96
97static int __kmp_expand_threads(int nNeed);
98#if KMP_OS_WINDOWS0
99static int __kmp_unregister_root_other_thread(int gtid);
100#endif
101static void __kmp_unregister_library(void); // called by __kmp_internal_end()
102static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
103kmp_info_t *__kmp_thread_pool_insert_pt = NULL__null;
104
105/* Calculate the identifier of the current thread */
106/* fast (and somewhat portable) way to get unique identifier of executing
107 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
108int __kmp_get_global_thread_id() {
109 int i;
110 kmp_info_t **other_threads;
111 size_t stack_data;
112 char *stack_addr;
113 size_t stack_size;
114 char *stack_base;
115
116 KA_TRACE(if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
117 1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
118 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
119 __kmp_nth, __kmp_all_nth))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
;
120
121 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
122 a parallel region, made it return KMP_GTID_DNE to force serial_initialize
123 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
124 __kmp_init_gtid for this to work. */
125
126 if (!TCR_4(__kmp_init_gtid)(__kmp_init_gtid))
127 return KMP_GTID_DNE(-2);
128
129#ifdef KMP_TDATA_GTID1
130 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
131 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using TDATA\n"
); }
;
132 return __kmp_gtid;
133 }
134#endif
135 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
136 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using keyed TLS\n"
); }
;
137 return __kmp_gtid_get_specific();
138 }
139 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using internal alg.\n"
); }
;
140
141 stack_addr = (char *)&stack_data;
142 other_threads = __kmp_threads;
143
144 /* ATT: The code below is a source of potential bugs due to unsynchronized
145 access to __kmp_threads array. For example:
146 1. Current thread loads other_threads[i] to thr and checks it, it is
147 non-NULL.
148 2. Current thread is suspended by OS.
149 3. Another thread unregisters and finishes (debug versions of free()
150 may fill memory with something like 0xEF).
151 4. Current thread is resumed.
152 5. Current thread reads junk from *thr.
153 TODO: Fix it. --ln */
154
155 for (i = 0; i < __kmp_threads_capacity; i++) {
156
157 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i])((void *)(other_threads[i]));
158 if (!thr)
159 continue;
160
161 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize)((void *)(thr->th.th_info.ds.ds_stacksize));
162 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase)((void *)(thr->th.th_info.ds.ds_stackbase));
163
164 /* stack grows down -- search through all of the active threads */
165
166 if (stack_addr <= stack_base) {
167 size_t stack_diff = stack_base - stack_addr;
168
169 if (stack_diff <= stack_size) {
170 /* The only way we can be closer than the allocated */
171 /* stack size is if we are running on this thread. */
172 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i)if (!(__kmp_gtid_get_specific() == i)) { __kmp_debug_assert("__kmp_gtid_get_specific() == i"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 172); }
;
173 return i;
174 }
175 }
176 }
177
178 /* get specific to try and determine our gtid */
179 KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
180 ("*** __kmp_get_global_thread_id: internal alg. failed to find "if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
181 "thread, using TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
;
182 i = __kmp_gtid_get_specific();
183
184 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */
185
186 /* if we havn't been assigned a gtid, then return code */
187 if (i < 0)
188 return i;
189
190 /* dynamically updated stack window for uber threads to avoid get_specific
191 call */
192 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
193 KMP_FATAL(StackOverflow, i)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_StackOverflow, i), __kmp_msg_null
)
;
194 }
195
196 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
197 if (stack_addr > stack_base) {
198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr)((other_threads[i]->th.th_info.ds.ds_stackbase)) = ((stack_addr
))
;
199 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
200 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
201 stack_base)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
;
202 } else {
203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
- stack_addr))
204 stack_base - stack_addr)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
- stack_addr))
;
205 }
206
207 /* Reprint stack bounds for ubermaster since they have been refined */
208 if (__kmp_storage_map) {
209 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
210 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
211 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
212 other_threads[i]->th.th_info.ds.ds_stacksize,
213 "th_%d stack (refinement)", i);
214 }
215 return i;
216}
217
218int __kmp_get_global_thread_id_reg() {
219 int gtid;
220
221 if (!__kmp_init_serial) {
222 gtid = KMP_GTID_DNE(-2);
223 } else
224#ifdef KMP_TDATA_GTID1
225 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
226 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using TDATA\n"
); }
;
227 gtid = __kmp_gtid;
228 } else
229#endif
230 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
231 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"
); }
;
232 gtid = __kmp_gtid_get_specific();
233 } else {
234 KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); }
235 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); }
;
236 gtid = __kmp_get_global_thread_id();
237 }
238
239 /* we must be a new uber master sibling thread */
240 if (gtid == KMP_GTID_DNE(-2)) {
241 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
242 ("__kmp_get_global_thread_id_reg: Encountered new root thread. "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
243 "Registering a new gtid.\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
;
244 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
245 if (!__kmp_init_serial) {
246 __kmp_do_serial_initialize();
247 gtid = __kmp_gtid_get_specific();
248 } else {
249 gtid = __kmp_register_root(FALSE0);
250 }
251 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
252 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
253 }
254
255 KMP_DEBUG_ASSERT(gtid >= 0)if (!(gtid >= 0)) { __kmp_debug_assert("gtid >= 0", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 255); }
;
256
257 return gtid;
258}
259
260/* caller must hold forkjoin_lock */
261void __kmp_check_stack_overlap(kmp_info_t *th) {
262 int f;
263 char *stack_beg = NULL__null;
264 char *stack_end = NULL__null;
265 int gtid;
266
267 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: called\n"
); }
;
268 if (__kmp_storage_map) {
269 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
270 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
271
272 gtid = __kmp_gtid_from_thread(th);
273
274 if (gtid == KMP_GTID_MONITOR(-4)) {
275 __kmp_print_storage_map_gtid(
276 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
277 "th_%s stack (%s)", "mon",
278 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
279 } else {
280 __kmp_print_storage_map_gtid(
281 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
282 "th_%d stack (%s)", gtid,
283 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
284 }
285 }
286
287 /* No point in checking ubermaster threads since they use refinement and
288 * cannot overlap */
289 gtid = __kmp_gtid_from_thread(th);
290 if (__kmp_env_checks == TRUE(!0) && !KMP_UBER_GTID(gtid)) {
291 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); }
292 ("__kmp_check_stack_overlap: performing extensive checking\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); }
;
293 if (stack_beg == NULL__null) {
294 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
295 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
296 }
297
298 for (f = 0; f < __kmp_threads_capacity; f++) {
299 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f])((void *)(__kmp_threads[f]));
300
301 if (f_th && f_th != th) {
302 char *other_stack_end =
303 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase)((void *)(f_th->th.th_info.ds.ds_stackbase));
304 char *other_stack_beg =
305 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize));
306 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
307 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
308
309 /* Print the other stack values before the abort */
310 if (__kmp_storage_map)
311 __kmp_print_storage_map_gtid(
312 -1, other_stack_beg, other_stack_end,
313 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize)),
314 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
315
316 __kmp_fatal(KMP_MSG(StackOverlap)__kmp_msg_format(kmp_i18n_msg_StackOverlap), KMP_HNT(ChangeStackLimit)__kmp_msg_format(kmp_i18n_hnt_ChangeStackLimit),
317 __kmp_msg_null);
318 }
319 }
320 }
321 }
322 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: returning\n"
); }
;
323}
324
325/* ------------------------------------------------------------------------ */
326
327void __kmp_infinite_loop(void) {
328 static int done = FALSE0;
329
330 while (!done) {
331 KMP_YIELD(1){ __kmp_x86_pause(); __kmp_yield((1)); };
332 }
333}
334
335#define MAX_MESSAGE512 512
336
337void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
338 char const *format, ...) {
339 char buffer[MAX_MESSAGE512];
340 va_list ap;
341
342 va_start(ap, format)__builtin_va_start(ap, format);
343 KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
344 p2, (unsigned long)size, format);
345 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
346 __kmp_vprintf(kmp_err, buffer, ap);
347#if KMP_PRINT_DATA_PLACEMENT
348 int node;
349 if (gtid >= 0) {
350 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
351 if (__kmp_storage_map_verbose) {
352 node = __kmp_get_host_node(p1);
353 if (node < 0) /* doesn't work, so don't try this next time */
354 __kmp_storage_map_verbose = FALSE0;
355 else {
356 char *last;
357 int lastNode;
358 int localProc = __kmp_get_cpu_from_gtid(gtid);
359
360 const int page_size = KMP_GET_PAGE_SIZE()getpagesize();
361
362 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
363 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
364 if (localProc >= 0)
365 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid,
366 localProc >> 1);
367 else
368 __kmp_printf_no_lock(" GTID %d\n", gtid);
369#if KMP_USE_PRCTL0
370 /* The more elaborate format is disabled for now because of the prctl
371 * hanging bug. */
372 do {
373 last = p1;
374 lastNode = node;
375 /* This loop collates adjacent pages with the same host node. */
376 do {
377 (char *)p1 += page_size;
378 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
379 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1,
380 lastNode);
381 } while (p1 <= p2);
382#else
383 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1,
384 (char *)p1 + (page_size - 1),
385 __kmp_get_host_node(p1));
386 if (p1 < p2) {
387 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2,
388 (char *)p2 + (page_size - 1),
389 __kmp_get_host_node(p2));
390 }
391#endif
392 }
393 }
394 } else
395 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)__kmp_i18n_catgets(kmp_i18n_str_StorageMapWarning));
396 }
397#endif /* KMP_PRINT_DATA_PLACEMENT */
398 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
399}
400
401void __kmp_warn(char const *format, ...) {
402 char buffer[MAX_MESSAGE512];
403 va_list ap;
404
405 if (__kmp_generate_warnings == kmp_warnings_off) {
406 return;
407 }
408
409 va_start(ap, format)__builtin_va_start(ap, format);
410
411 KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP warning: %s\n", format);
412 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
413 __kmp_vprintf(kmp_err, buffer, ap);
414 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
415
416 va_end(ap)__builtin_va_end(ap);
417}
418
419void __kmp_abort_process() {
420 // Later threads may stall here, but that's ok because abort() will kill them.
421 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
422
423 if (__kmp_debug_buf) {
424 __kmp_dump_debug_buffer();
425 }
426
427 if (KMP_OS_WINDOWS0) {
428 // Let other threads know of abnormal termination and prevent deadlock
429 // if abort happened during library initialization or shutdown
430 __kmp_global.g.g_abort = SIGABRT6;
431
432 /* On Windows* OS by default abort() causes pop-up error box, which stalls
433 nightly testing. Unfortunately, we cannot reliably suppress pop-up error
434 boxes. _set_abort_behavior() works well, but this function is not
435 available in VS7 (this is not problem for DLL, but it is a problem for
436 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
437 help, at least in some versions of MS C RTL.
438
439 It seems following sequence is the only way to simulate abort() and
440 avoid pop-up error box. */
441 raise(SIGABRT6);
442 _exit(3); // Just in case, if signal ignored, exit anyway.
443 } else {
444 abort();
445 }
446
447 __kmp_infinite_loop();
448 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
449
450} // __kmp_abort_process
451
452void __kmp_abort_thread(void) {
453 // TODO: Eliminate g_abort global variable and this function.
454 // In case of abort just call abort(), it will kill all the threads.
455 __kmp_infinite_loop();
456} // __kmp_abort_thread
457
458/* Print out the storage map for the major kmp_info_t thread data structures
459 that are allocated together. */
460
461static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
462 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
463 gtid);
464
465 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
466 sizeof(kmp_desc_t), "th_%d.th_info", gtid);
467
468 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
469 sizeof(kmp_local_t), "th_%d.th_local", gtid);
470
471 __kmp_print_storage_map_gtid(
472 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
473 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);
474
475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
476 &thr->th.th_bar[bs_plain_barrier + 1],
477 sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
478 gtid);
479
480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
481 &thr->th.th_bar[bs_forkjoin_barrier + 1],
482 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
483 gtid);
484
485#if KMP_FAST_REDUCTION_BARRIER1
486 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
487 &thr->th.th_bar[bs_reduction_barrier + 1],
488 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
489 gtid);
490#endif // KMP_FAST_REDUCTION_BARRIER
491}
492
493/* Print out the storage map for the major kmp_team_t team data structures
494 that are allocated together. */
495
496static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
497 int team_id, int num_thr) {
498 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
499 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
500 header, team_id);
501
502 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
503 &team->t.t_bar[bs_last_barrier],
504 sizeof(kmp_balign_team_t) * bs_last_barrier,
505 "%s_%d.t_bar", header, team_id);
506
507 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
508 &team->t.t_bar[bs_plain_barrier + 1],
509 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
510 header, team_id);
511
512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
513 &team->t.t_bar[bs_forkjoin_barrier + 1],
514 sizeof(kmp_balign_team_t),
515 "%s_%d.t_bar[forkjoin]", header, team_id);
516
517#if KMP_FAST_REDUCTION_BARRIER1
518 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
519 &team->t.t_bar[bs_reduction_barrier + 1],
520 sizeof(kmp_balign_team_t),
521 "%s_%d.t_bar[reduction]", header, team_id);
522#endif // KMP_FAST_REDUCTION_BARRIER
523
524 __kmp_print_storage_map_gtid(
525 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
526 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);
527
528 __kmp_print_storage_map_gtid(
529 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
530 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);
531
532 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
533 &team->t.t_disp_buffer[num_disp_buff],
534 sizeof(dispatch_shared_info_t) * num_disp_buff,
535 "%s_%d.t_disp_buffer", header, team_id);
536
537 __kmp_print_storage_map_gtid(-1, &team->t.t_taskq, &team->t.t_copypriv_data,
538 sizeof(kmp_taskq_t), "%s_%d.t_taskq", header,
539 team_id);
540}
541
542static void __kmp_init_allocator() {
543#if OMP_50_ENABLED(50 >= 50)
544 __kmp_init_memkind();
545#endif
546}
547static void __kmp_fini_allocator() {
548#if OMP_50_ENABLED(50 >= 50)
549 __kmp_fini_memkind();
550#endif
551}
552
553/* ------------------------------------------------------------------------ */
554
555#if KMP_DYNAMIC_LIB1
556#if KMP_OS_WINDOWS0
557
558static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
559 // TODO: Change to __kmp_break_bootstrap_lock().
560 __kmp_init_bootstrap_lock(lck); // make the lock released
561}
562
563static void __kmp_reset_locks_on_process_detach(int gtid_req) {
564 int i;
565 int thread_count;
566
567 // PROCESS_DETACH is expected to be called by a thread that executes
568 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one
569 // calling ProcessExit or FreeLibrary). So, it might be safe to access the
570 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some
571 // threads can be still alive here, although being about to be terminated. The
572 // threads in the array with ds_thread==0 are most suspicious. Actually, it
573 // can be not safe to access the __kmp_threads[].
574
575 // TODO: does it make sense to check __kmp_roots[] ?
576
577 // Let's check that there are no other alive threads registered with the OMP
578 // lib.
579 while (1) {
580 thread_count = 0;
581 for (i = 0; i < __kmp_threads_capacity; ++i) {
582 if (!__kmp_threads)
583 continue;
584 kmp_info_t *th = __kmp_threads[i];
585 if (th == NULL__null)
586 continue;
587 int gtid = th->th.th_info.ds.ds_gtid;
588 if (gtid == gtid_req)
589 continue;
590 if (gtid < 0)
591 continue;
592 DWORD exit_val;
593 int alive = __kmp_is_thread_alive(th, &exit_val);
594 if (alive) {
595 ++thread_count;
596 }
597 }
598 if (thread_count == 0)
599 break; // success
600 }
601
602 // Assume that I'm alone. Now it might be safe to check and reset locks.
603 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset.
604 __kmp_reset_lock(&__kmp_forkjoin_lock);
605#ifdef KMP_DEBUG1
606 __kmp_reset_lock(&__kmp_stdio_lock);
607#endif // KMP_DEBUG
608}
609
610BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
611 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
612
613 switch (fdwReason) {
614
615 case DLL_PROCESS_ATTACH:
616 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_ATTACH\n"
); }
;
617
618 return TRUE(!0);
619
620 case DLL_PROCESS_DETACH:
621 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_DETACH T#%d\n"
, __kmp_gtid_get_specific()); }
;
622
623 if (lpReserved != NULL__null) {
624 // lpReserved is used for telling the difference:
625 // lpReserved == NULL when FreeLibrary() was called,
626 // lpReserved != NULL when the process terminates.
627 // When FreeLibrary() is called, worker threads remain alive. So they will
628 // release the forkjoin lock by themselves. When the process terminates,
629 // worker threads disappear triggering the problem of unreleased forkjoin
630 // lock as described below.
631
632 // A worker thread can take the forkjoin lock. The problem comes up if
633 // that worker thread becomes dead before it releases the forkjoin lock.
634 // The forkjoin lock remains taken, while the thread executing
635 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try
636 // to take the forkjoin lock and will always fail, so that the application
637 // will never finish [normally]. This scenario is possible if
638 // __kmpc_end() has not been executed. It looks like it's not a corner
639 // case, but common cases:
640 // - the main function was compiled by an alternative compiler;
641 // - the main function was compiled by icl but without /Qopenmp
642 // (application with plugins);
643 // - application terminates by calling C exit(), Fortran CALL EXIT() or
644 // Fortran STOP.
645 // - alive foreign thread prevented __kmpc_end from doing cleanup.
646 //
647 // This is a hack to work around the problem.
648 // TODO: !!! figure out something better.
649 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
650 }
651
652 __kmp_internal_end_library(__kmp_gtid_get_specific());
653
654 return TRUE(!0);
655
656 case DLL_THREAD_ATTACH:
657 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_ATTACH\n"
); }
;
658
659 /* if we want to register new siblings all the time here call
660 * __kmp_get_gtid(); */
661 return TRUE(!0);
662
663 case DLL_THREAD_DETACH:
664 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_DETACH T#%d\n"
, __kmp_gtid_get_specific()); }
;
665
666 __kmp_internal_end_thread(__kmp_gtid_get_specific());
667 return TRUE(!0);
668 }
669
670 return TRUE(!0);
671}
672
673#endif /* KMP_OS_WINDOWS */
674#endif /* KMP_DYNAMIC_LIB */
675
676/* Change the library type to "status" and return the old type */
677/* called from within initialization routines where __kmp_initz_lock is held */
678int __kmp_change_library(int status) {
679 int old_status;
680
681 old_status = __kmp_yield_init &
682 1; // check whether KMP_LIBRARY=throughput (even init count)
683
684 if (status) {
685 __kmp_yield_init |= 1; // throughput => turnaround (odd init count)
686 } else {
687 __kmp_yield_init &= ~1; // turnaround => throughput (even init count)
688 }
689
690 return old_status; // return previous setting of whether
691 // KMP_LIBRARY=throughput
692}
693
694/* __kmp_parallel_deo -- Wait until it's our turn. */
695void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
696 int gtid = *gtid_ref;
697#ifdef BUILD_PARALLEL_ORDERED1
698 kmp_team_t *team = __kmp_team_from_gtid(gtid);
699#endif /* BUILD_PARALLEL_ORDERED */
700
701 if (__kmp_env_consistency_check) {
702 if (__kmp_threads[gtid]->th.th_root->r.r_active)
703#if KMP_USE_DYNAMIC_LOCK1
704 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null, 0);
705#else
706 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null);
707#endif
708 }
709#ifdef BUILD_PARALLEL_ORDERED1
710 if (!team->t.t_serialized) {
711 KMP_MB();
712 KMP_WAIT_YIELD__kmp_wait_yield_4(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid),
713 KMP_EQ__kmp_eq_4, NULL__null);
714 KMP_MB();
715 }
716#endif /* BUILD_PARALLEL_ORDERED */
717}
718
719/* __kmp_parallel_dxo -- Signal the next task. */
720void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
721 int gtid = *gtid_ref;
722#ifdef BUILD_PARALLEL_ORDERED1
723 int tid = __kmp_tid_from_gtid(gtid);
724 kmp_team_t *team = __kmp_team_from_gtid(gtid);
725#endif /* BUILD_PARALLEL_ORDERED */
726
727 if (__kmp_env_consistency_check) {
728 if (__kmp_threads[gtid]->th.th_root->r.r_active)
729 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
730 }
731#ifdef BUILD_PARALLEL_ORDERED1
732 if (!team->t.t_serialized) {
733 KMP_MB(); /* Flush all pending memory write invalidates. */
734
735 /* use the tid of the next thread in this team */
736 /* TODO replace with general release procedure */
737 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
738
739 KMP_MB(); /* Flush all pending memory write invalidates. */
740 }
741#endif /* BUILD_PARALLEL_ORDERED */
742}
743
744/* ------------------------------------------------------------------------ */
745/* The BARRIER for a SINGLE process section is always explicit */
746
747int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
748 int status;
749 kmp_info_t *th;
750 kmp_team_t *team;
751
752 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
753 __kmp_parallel_initialize();
754
755 th = __kmp_threads[gtid];
756 team = th->th.th_team;
757 status = 0;
758
759 th->th.th_ident = id_ref;
760
761 if (team->t.t_serialized) {
762 status = 1;
763 } else {
764 kmp_int32 old_this = th->th.th_local.this_construct;
765
766 ++th->th.th_local.this_construct;
767 /* try to set team count to thread count--success means thread got the
768 single block */
769 /* TODO: Should this be acquire or release? */
770 if (team->t.t_construct == old_this) {
771 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
772 th->th.th_local.this_construct);
773 }
774#if USE_ITT_BUILD1
775 if (__itt_metadata_add_ptr__kmp_itt_metadata_add_ptr__3_0 && __kmp_forkjoin_frames_mode == 3 &&
776 KMP_MASTER_GTID(gtid)(__kmp_tid_from_gtid((gtid)) == 0) &&
777#if OMP_40_ENABLED(50 >= 40)
778 th->th.th_teams_microtask == NULL__null &&
779#endif
780 team->t.t_active_level ==
781 1) { // Only report metadata by master of active team at level 1
782 __kmp_itt_metadata_single(id_ref);
783 }
784#endif /* USE_ITT_BUILD */
785 }
786
787 if (__kmp_env_consistency_check) {
788 if (status && push_ws) {
789 __kmp_push_workshare(gtid, ct_psingle, id_ref);
790 } else {
791 __kmp_check_workshare(gtid, ct_psingle, id_ref);
792 }
793 }
794#if USE_ITT_BUILD1
795 if (status) {
796 __kmp_itt_single_start(gtid);
797 }
798#endif /* USE_ITT_BUILD */
799 return status;
800}
801
802void __kmp_exit_single(int gtid) {
803#if USE_ITT_BUILD1
804 __kmp_itt_single_end(gtid);
805#endif /* USE_ITT_BUILD */
806 if (__kmp_env_consistency_check)
807 __kmp_pop_workshare(gtid, ct_psingle, NULL__null);
808}
809
810/* determine if we can go parallel or must use a serialized parallel region and
811 * how many threads we can use
812 * set_nproc is the number of threads requested for the team
813 * returns 0 if we should serialize or only use one thread,
814 * otherwise the number of threads to use
815 * The forkjoin lock is held by the caller. */
816static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
817 int master_tid, int set_nthreads
818#if OMP_40_ENABLED(50 >= 40)
819 ,
820 int enter_teams
821#endif /* OMP_40_ENABLED */
822 ) {
823 int capacity;
824 int new_nthreads;
825 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 825); }
;
826 KMP_DEBUG_ASSERT(root && parent_team)if (!(root && parent_team)) { __kmp_debug_assert("root && parent_team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 826); }
;
827
828 // If dyn-var is set, dynamically adjust the number of desired threads,
829 // according to the method specified by dynamic_mode.
830 new_nthreads = set_nthreads;
831 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
) {
832 ;
833 }
834#ifdef USE_LOAD_BALANCE1
835 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
836 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
837 if (new_nthreads == 1) {
838 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
839 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
840 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
;
841 return 1;
842 }
843 if (new_nthreads < set_nthreads) {
844 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
845 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
846 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
847 }
848 }
849#endif /* USE_LOAD_BALANCE */
850 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
851 new_nthreads = __kmp_avail_proc - __kmp_nth +
852 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
853 if (new_nthreads <= 1) {
854 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
855 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
856 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
;
857 return 1;
858 }
859 if (new_nthreads < set_nthreads) {
860 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
861 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
862 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
863 } else {
864 new_nthreads = set_nthreads;
865 }
866 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
867 if (set_nthreads > 2) {
868 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
869 new_nthreads = (new_nthreads % set_nthreads) + 1;
870 if (new_nthreads == 1) {
871 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
872 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
873 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
;
874 return 1;
875 }
876 if (new_nthreads < set_nthreads) {
877 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
878 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
879 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
880 }
881 }
882 } else {
883 KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 883); }
;
884 }
885
886 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
887 if (__kmp_nth + new_nthreads -
888 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
889 __kmp_max_nth) {
890 int tl_nthreads = __kmp_max_nth - __kmp_nth +
891 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
892 if (tl_nthreads <= 0) {
893 tl_nthreads = 1;
894 }
895
896 // If dyn-var is false, emit a 1-time warning.
897 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
898 __kmp_reserve_warn = 1;
899 __kmp_msg(kmp_ms_warning,
900 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
)
,
901 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
902 }
903 if (tl_nthreads == 1) {
904 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
905 "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
906 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
;
907 return 1;
908 }
909 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
910 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
911 master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
;
912 new_nthreads = tl_nthreads;
913 }
914
915 // Respect OMP_THREAD_LIMIT
916 if (root->r.r_cg_nthreads + new_nthreads -
917 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
918 __kmp_cg_max_nth) {
919 int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads +
920 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
921 if (tl_nthreads <= 0) {
922 tl_nthreads = 1;
923 }
924
925 // If dyn-var is false, emit a 1-time warning.
926 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
927 __kmp_reserve_warn = 1;
928 __kmp_msg(kmp_ms_warning,
929 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
)
,
930 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
931 }
932 if (tl_nthreads == 1) {
933 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
934 "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
935 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
;
936 return 1;
937 }
938 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
939 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
940 master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
;
941 new_nthreads = tl_nthreads;
942 }
943
944 // Check if the threads array is large enough, or needs expanding.
945 // See comment in __kmp_register_root() about the adjustment if
946 // __kmp_threads[0] == NULL.
947 capacity = __kmp_threads_capacity;
948 if (TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
949 --capacity;
950 }
951 if (__kmp_nth + new_nthreads -
952 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
953 capacity) {
954 // Expand the threads array.
955 int slotsRequired = __kmp_nth + new_nthreads -
956 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
957 capacity;
958 int slotsAdded = __kmp_expand_threads(slotsRequired);
959 if (slotsAdded < slotsRequired) {
960 // The threads array was not expanded enough.
961 new_nthreads -= (slotsRequired - slotsAdded);
962 KMP_ASSERT(new_nthreads >= 1)if (!(new_nthreads >= 1)) { __kmp_debug_assert("new_nthreads >= 1"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 962); }
;
963
964 // If dyn-var is false, emit a 1-time warning.
965 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
966 __kmp_reserve_warn = 1;
967 if (__kmp_tp_cached) {
968 __kmp_msg(kmp_ms_warning,
969 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
)
,
970 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
)
,
971 KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
972 } else {
973 __kmp_msg(kmp_ms_warning,
974 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
)
,
975 KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads), __kmp_msg_null);
976 }
977 }
978 }
979 }
980
981#ifdef KMP_DEBUG1
982 if (new_nthreads == 1) {
983 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
984 ("__kmp_reserve_threads: T#%d serializing team after reclaiming "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
985 "dead roots and rechecking; requested %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
986 __kmp_get_gtid(), set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
;
987 } else {
988 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
989 " %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
990 __kmp_get_gtid(), new_nthreads, set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
;
991 }
992#endif // KMP_DEBUG
993 return new_nthreads;
994}
995
996/* Allocate threads from the thread pool and assign them to the new team. We are
997 assured that there are enough threads available, because we checked on that
998 earlier within critical section forkjoin */
999static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
1000 kmp_info_t *master_th, int master_gtid) {
1001 int i;
1002 int use_hot_team;
1003
1004 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_fork_team_threads: new_nprocs = %d\n"
, team->t.t_nproc); }
;
1005 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid())if (!(master_gtid == __kmp_get_global_thread_id())) { __kmp_debug_assert
("master_gtid == __kmp_get_global_thread_id()", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1005); }
;
1006 KMP_MB();
1007
1008 /* first, let's setup the master thread */
1009 master_th->th.th_info.ds.ds_tid = 0;
1010 master_th->th.th_team = team;
1011 master_th->th.th_team_nproc = team->t.t_nproc;
1012 master_th->th.th_team_master = master_th;
1013 master_th->th.th_team_serialized = FALSE0;
1014 master_th->th.th_dispatch = &team->t.t_dispatch[0];
1015
1016/* make sure we are not the optimized hot team */
1017#if KMP_NESTED_HOT_TEAMS1
1018 use_hot_team = 0;
1019 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
1020 if (hot_teams) { // hot teams array is not allocated if
1021 // KMP_HOT_TEAMS_MAX_LEVEL=0
1022 int level = team->t.t_active_level - 1; // index in array of hot teams
1023 if (master_th->th.th_teams_microtask) { // are we inside the teams?
1024 if (master_th->th.th_teams_size.nteams > 1) {
1025 ++level; // level was not increased in teams construct for
1026 // team_of_masters
1027 }
1028 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1029 master_th->th.th_teams_level == team->t.t_level) {
1030 ++level; // level was not increased in teams construct for
1031 // team_of_workers before the parallel
1032 } // team->t.t_level will be increased inside parallel
1033 }
1034 if (level < __kmp_hot_teams_max_level) {
1035 if (hot_teams[level].hot_team) {
1036 // hot team has already been allocated for given level
1037 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team)if (!(hot_teams[level].hot_team == team)) { __kmp_debug_assert
("hot_teams[level].hot_team == team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1037); }
;
1038 use_hot_team = 1; // the team is ready to use
1039 } else {
1040 use_hot_team = 0; // AC: threads are not allocated yet
1041 hot_teams[level].hot_team = team; // remember new hot team
1042 hot_teams[level].hot_team_nth = team->t.t_nproc;
1043 }
1044 } else {
1045 use_hot_team = 0;
1046 }
1047 }
1048#else
1049 use_hot_team = team == root->r.r_hot_team;
1050#endif
1051 if (!use_hot_team) {
1052
1053 /* install the master thread */
1054 team->t.t_threads[0] = master_th;
1055 __kmp_initialize_info(master_th, team, 0, master_gtid);
1056
1057 /* now, install the worker threads */
1058 for (i = 1; i < team->t.t_nproc; i++) {
1059
1060 /* fork or reallocate a new thread and install it in team */
1061 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1062 team->t.t_threads[i] = thr;
1063 KMP_DEBUG_ASSERT(thr)if (!(thr)) { __kmp_debug_assert("thr", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1063); }
;
1064 KMP_DEBUG_ASSERT(thr->th.th_team == team)if (!(thr->th.th_team == team)) { __kmp_debug_assert("thr->th.th_team == team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1064); }
;
1065 /* align team and thread arrived states */
1066 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
1067 "T#%d(%d:%d) join =%llu, plain=%llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
1068 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
1069 __kmp_gtid_from_tid(i, team), team->t.t_id, i,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
1070 team->t.t_bar[bs_forkjoin_barrier].b_arrived,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
1071 team->t.t_bar[bs_plain_barrier].b_arrived))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
;
1072#if OMP_40_ENABLED(50 >= 40)
1073 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1074 thr->th.th_teams_level = master_th->th.th_teams_level;
1075 thr->th.th_teams_size = master_th->th.th_teams_size;
1076#endif
1077 { // Initialize threads' barrier data.
1078 int b;
1079 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1080 for (b = 0; b < bs_last_barrier; ++b) {
1081 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1082 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1082); }
;
1083#if USE_DEBUGGER0
1084 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1085#endif
1086 }
1087 }
1088 }
1089
1090#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
1091 __kmp_partition_places(team);
1092#endif
1093 }
1094
1095 KMP_MB();
1096}
1097
1098#if KMP_ARCH_X860 || KMP_ARCH_X86_641
1099// Propagate any changes to the floating point control registers out to the team
1100// We try to avoid unnecessary writes to the relevant cache line in the team
1101// structure, so we don't make changes unless they are needed.
1102inline static void propagateFPControl(kmp_team_t *team) {
1103 if (__kmp_inherit_fp_control) {
1104 kmp_int16 x87_fpu_control_word;
1105 kmp_uint32 mxcsr;
1106
1107 // Get master values of FPU control flags (both X87 and vector)
1108 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1109 __kmp_store_mxcsr(&mxcsr);
1110 mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
1111
1112 // There is no point looking at t_fp_control_saved here.
1113 // If it is TRUE, we still have to update the values if they are different
1114 // from those we now have. If it is FALSE we didn't save anything yet, but
1115 // our objective is the same. We have to ensure that the values in the team
1116 // are the same as those we have.
1117 // So, this code achieves what we need whether or not t_fp_control_saved is
1118 // true. By checking whether the value needs updating we avoid unnecessary
1119 // writes that would put the cache-line into a written state, causing all
1120 // threads in the team to have to read it again.
1121 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word)if ((team->t.t_x87_fpu_control_word) != (x87_fpu_control_word
)) (team->t.t_x87_fpu_control_word) = (x87_fpu_control_word
)
;
1122 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr)if ((team->t.t_mxcsr) != (mxcsr)) (team->t.t_mxcsr) = (
mxcsr)
;
1123 // Although we don't use this value, other code in the runtime wants to know
1124 // whether it should restore them. So we must ensure it is correct.
1125 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE)if ((team->t.t_fp_control_saved) != ((!0))) (team->t.t_fp_control_saved
) = ((!0))
;
1126 } else {
1127 // Similarly here. Don't write to this cache-line in the team structure
1128 // unless we have to.
1129 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE)if ((team->t.t_fp_control_saved) != (0)) (team->t.t_fp_control_saved
) = (0)
;
1130 }
1131}
1132
1133// Do the opposite, setting the hardware registers to the updated values from
1134// the team.
1135inline static void updateHWFPControl(kmp_team_t *team) {
1136 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1137 // Only reset the fp control regs if they have been changed in the team.
1138 // the parallel region that we are exiting.
1139 kmp_int16 x87_fpu_control_word;
1140 kmp_uint32 mxcsr;
1141 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1142 __kmp_store_mxcsr(&mxcsr);
1143 mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
1144
1145 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1146 __kmp_clear_x87_fpu_status_word();
1147 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1148 }
1149
1150 if (team->t.t_mxcsr != mxcsr) {
1151 __kmp_load_mxcsr(&team->t.t_mxcsr)_mm_setcsr(*(&team->t.t_mxcsr));
1152 }
1153 }
1154}
1155#else
1156#define propagateFPControl(x) ((void)0)
1157#define updateHWFPControl(x) ((void)0)
1158#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1159
1160static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1161 int realloc); // forward declaration
1162
1163/* Run a parallel region that has been serialized, so runs only in a team of the
1164 single master thread. */
1165void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
1166 kmp_info_t *this_thr;
1167 kmp_team_t *serial_team;
1168
1169 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: called by T#%d\n"
, global_tid); }
;
1170
1171 /* Skip all this code for autopar serialized loops since it results in
1172 unacceptable overhead */
1173 if (loc != NULL__null && (loc->flags & KMP_IDENT_AUTOPAR))
1174 return;
1175
1176 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
1177 __kmp_parallel_initialize();
1178
1179 this_thr = __kmp_threads[global_tid];
1180 serial_team = this_thr->th.th_serial_team;
1181
1182 /* utilize the serialized team held by this thread */
1183 KMP_DEBUG_ASSERT(serial_team)if (!(serial_team)) { __kmp_debug_assert("serial_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1183); }
;
1184 KMP_MB();
1185
1186 if (__kmp_tasking_mode != tskm_immediate_exec) {
1187 KMP_DEBUG_ASSERT(if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1189); }
1188 this_thr->th.th_task_team ==if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1189); }
1189 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state])if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1189); }
;
1190 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==if (!(serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null)) { __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1191); }
1191 NULL)if (!(serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null)) { __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1191); }
;
1192 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
1193 "team %p, new task_team = NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
1194 global_tid, this_thr->th.th_task_team, this_thr->th.th_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
;
1195 this_thr->th.th_task_team = NULL__null;
1196 }
1197
1198#if OMP_40_ENABLED(50 >= 40)
1199 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1200 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1201 proc_bind = proc_bind_false;
1202 } else if (proc_bind == proc_bind_default) {
1203 // No proc_bind clause was specified, so use the current value
1204 // of proc-bind-var for this parallel region.
1205 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1206 }
1207 // Reset for next parallel region
1208 this_thr->th.th_set_proc_bind = proc_bind_default;
1209#endif /* OMP_40_ENABLED */
1210
1211#if OMPT_SUPPORT1
1212 ompt_data_t ompt_parallel_data = ompt_data_none;
1213 ompt_data_t *implicit_task_data;
1214 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid)__ompt_load_return_address(global_tid);
1215 if (ompt_enabled.enabled &&
1216 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
1217
1218 ompt_task_info_t *parent_task_info;
1219 parent_task_info = OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info));
1220
1221 parent_task_info->frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1222 if (ompt_enabled.ompt_callback_parallel_begin) {
1223 int team_size = 1;
1224
1225 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
1226 &(parent_task_info->task_data), &(parent_task_info->frame),
1227 &ompt_parallel_data, team_size, ompt_parallel_invoker_program,
1228 codeptr);
1229 }
1230 }
1231#endif // OMPT_SUPPORT
1232
1233 if (this_thr->th.th_team != serial_team) {
1234 // Nested level will be an index in the nested nthreads array
1235 int level = this_thr->th.th_team->t.t_level;
1236
1237 if (serial_team->t.t_serialized) {
1238 /* this serial team was already used
1239 TODO increase performance by making this locks more specific */
1240 kmp_team_t *new_team;
1241
1242 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1243
1244 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1245#if OMPT_SUPPORT1
1246 ompt_parallel_data,
1247#endif
1248#if OMP_40_ENABLED(50 >= 40)
1249 proc_bind,
1250#endif
1251 &this_thr->th.th_current_task->td_icvs,
1252 0 USE_NESTED_HOT_ARG(NULL), __null);
1253 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1254 KMP_ASSERT(new_team)if (!(new_team)) { __kmp_debug_assert("new_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1254); }
;
1255
1256 /* setup new serialized team and install it */
1257 new_team->t.t_threads[0] = this_thr;
1258 new_team->t.t_parent = this_thr->th.th_team;
1259 serial_team = new_team;
1260 this_thr->th.th_serial_team = serial_team;
1261
1262 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1263 10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1264 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1265 global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
;
1266
1267 /* TODO the above breaks the requirement that if we run out of resources,
1268 then we can still guarantee that serialized teams are ok, since we may
1269 need to allocate a new one */
1270 } else {
1271 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1272 10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1273 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1274 global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
;
1275 }
1276
1277 /* we have to initialize this serial team */
1278 KMP_DEBUG_ASSERT(serial_team->t.t_threads)if (!(serial_team->t.t_threads)) { __kmp_debug_assert("serial_team->t.t_threads"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1278); }
;
1279 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)if (!(serial_team->t.t_threads[0] == this_thr)) { __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1279); }
;
1280 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team)if (!(this_thr->th.th_team != serial_team)) { __kmp_debug_assert
("this_thr->th.th_team != serial_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1280); }
;
1281 serial_team->t.t_ident = loc;
1282 serial_team->t.t_serialized = 1;
1283 serial_team->t.t_nproc = 1;
1284 serial_team->t.t_parent = this_thr->th.th_team;
1285 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1286 this_thr->th.th_team = serial_team;
1287 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1288
1289 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); }
1290 this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); }
;
1291 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1)if (!(this_thr->th.th_current_task->td_flags.executing ==
1)) { __kmp_debug_assert("this_thr->th.th_current_task->td_flags.executing == 1"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1291); }
;
1292 this_thr->th.th_current_task->td_flags.executing = 0;
1293
1294 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1295
1296 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
1297 implicit task for each serialized task represented by
1298 team->t.t_serialized? */
1299 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1300 &this_thr->th.th_current_task->td_parent->td_icvs);
1301
1302 // Thread value exists in the nested nthreads array for the next nested
1303 // level
1304 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1305 this_thr->th.th_current_task->td_icvs.nproc =
1306 __kmp_nested_nth.nth[level + 1];
1307 }
1308
1309#if OMP_40_ENABLED(50 >= 40)
1310 if (__kmp_nested_proc_bind.used &&
1311 (level + 1 < __kmp_nested_proc_bind.used)) {
1312 this_thr->th.th_current_task->td_icvs.proc_bind =
1313 __kmp_nested_proc_bind.bind_types[level + 1];
1314 }
1315#endif /* OMP_40_ENABLED */
1316
1317#if USE_DEBUGGER0
1318 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
1319#endif
1320 this_thr->th.th_info.ds.ds_tid = 0;
1321
1322 /* set thread cache values */
1323 this_thr->th.th_team_nproc = 1;
1324 this_thr->th.th_team_master = this_thr;
1325 this_thr->th.th_team_serialized = 1;
1326
1327 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1328 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1329#if OMP_50_ENABLED(50 >= 50)
1330 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save
1331#endif
1332
1333 propagateFPControl(serial_team);
1334
1335 /* check if we need to allocate dispatch buffers stack */
1336 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)if (!(serial_team->t.t_dispatch)) { __kmp_debug_assert("serial_team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1336); }
;
1337 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1338 serial_team->t.t_dispatch->th_disp_buffer =
1339 (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1340)
1340 sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1340)
;
1341 }
1342 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1343
1344 KMP_MB();
1345
1346 } else {
1347 /* this serialized team is already being used,
1348 * that's fine, just add another nested level */
1349 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team)if (!(this_thr->th.th_team == serial_team)) { __kmp_debug_assert
("this_thr->th.th_team == serial_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1349); }
;
1350 KMP_DEBUG_ASSERT(serial_team->t.t_threads)if (!(serial_team->t.t_threads)) { __kmp_debug_assert("serial_team->t.t_threads"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1350); }
;
1351 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)if (!(serial_team->t.t_threads[0] == this_thr)) { __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1351); }
;
1352 ++serial_team->t.t_serialized;
1353 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1354
1355 // Nested level will be an index in the nested nthreads array
1356 int level = this_thr->th.th_team->t.t_level;
1357 // Thread value exists in the nested nthreads array for the next nested
1358 // level
1359 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1360 this_thr->th.th_current_task->td_icvs.nproc =
1361 __kmp_nested_nth.nth[level + 1];
1362 }
1363 serial_team->t.t_level++;
1364 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
1365 "of serial team %p to %d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
1366 global_tid, serial_team, serial_team->t.t_level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
;
1367
1368 /* allocate/push dispatch buffers stack */
1369 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)if (!(serial_team->t.t_dispatch)) { __kmp_debug_assert("serial_team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1369); }
;
1370 {
1371 dispatch_private_info_t *disp_buffer =
1372 (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1373)
1373 sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1373)
;
1374 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1375 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1376 }
1377 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1378
1379 KMP_MB();
1380 }
1381#if OMP_40_ENABLED(50 >= 40)
1382 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq)if ((serial_team->t.t_cancel_request) != (cancel_noreq)) (
serial_team->t.t_cancel_request) = (cancel_noreq)
;
1383#endif
1384
1385 if (__kmp_env_consistency_check)
1386 __kmp_push_parallel(global_tid, NULL__null);
1387#if OMPT_SUPPORT1
1388 serial_team->t.ompt_team_info.master_return_address = codeptr;
1389 if (ompt_enabled.enabled &&
1390 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
1391 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1392
1393 ompt_lw_taskteam_t lw_taskteam;
1394 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1395 &ompt_parallel_data, codeptr);
1396
1397 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1398 // don't use lw_taskteam after linking. content was swaped
1399
1400 /* OMPT implicit task begin */
1401 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
))
;
1402 if (ompt_enabled.ompt_callback_implicit_task) {
1403 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1404 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr)(&(this_thr->th.th_team->t.ompt_team_info.parallel_data
))
,
1405 OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
))
, 1, __kmp_tid_from_gtid(global_tid));
1406 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))
1407 ->thread_num = __kmp_tid_from_gtid(global_tid);
1408 }
1409
1410 /* OMPT state */
1411 this_thr->th.ompt_thread_info.state = omp_state_work_parallel;
1412 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1413 }
1414#endif
1415}
1416
1417/* most of the work for a fork */
1418/* return true if we really went parallel, false if serialized */
1419int __kmp_fork_call(ident_t *loc, int gtid,
1420 enum fork_context_e call_context, // Intel, GNU, ...
1421 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1422/* TODO: revert workaround for Intel(R) 64 tracker #96 */
1423#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1424 va_list *ap
1425#else
1426 va_list ap
1427#endif
1428 ) {
1429 void **argv;
1430 int i;
1431 int master_tid;
1432 int master_this_cons;
1433 kmp_team_t *team;
1434 kmp_team_t *parent_team;
1435 kmp_info_t *master_th;
1436 kmp_root_t *root;
1437 int nthreads;
1438 int master_active;
1439 int master_set_numthreads;
1440 int level;
1441#if OMP_40_ENABLED(50 >= 40)
1442 int active_level;
1443 int teams_level;
1444#endif
1445#if KMP_NESTED_HOT_TEAMS1
1446 kmp_hot_team_ptr_t **p_hot_teams;
1447#endif
1448 { // KMP_TIME_BLOCK
1449 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call)((void)0);
1450 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc)((void)0);
1451
1452 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: enter T#%d\n"
, gtid); }
;
1453 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL__null) {
1454 /* Some systems prefer the stack for the root thread(s) to start with */
1455 /* some gap from the parent stack to prevent false sharing. */
1456 void *dummy = KMP_ALLOCA(__kmp_stkpadding)__builtin_alloca (__kmp_stkpadding);
1457 /* These 2 lines below are so this does not get optimized out */
1458 if (__kmp_stkpadding > KMP_MAX_STKPADDING(2 * 1024 * 1024))
1459 __kmp_stkpadding += (short)((kmp_int64)dummy);
1460 }
1461
1462 /* initialize if needed */
1463 KMP_DEBUG_ASSERT(if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1464); }
1464 __kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1464); }
; // AC: potentially unsafe, not in sync with shutdown
1465 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
1466 __kmp_parallel_initialize();
1467
1468 /* setup current data */
1469 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
1470 // shutdown
1471 parent_team = master_th->th.th_team;
1472 master_tid = master_th->th.th_info.ds.ds_tid;
1473 master_this_cons = master_th->th.th_local.this_construct;
1474 root = master_th->th.th_root;
1475 master_active = root->r.r_active;
1476 master_set_numthreads = master_th->th.th_set_nproc;
1477
1478#if OMPT_SUPPORT1
1479 ompt_data_t ompt_parallel_data = ompt_data_none;
1480 ompt_data_t *parent_task_data;
1481 omp_frame_t *ompt_frame;
1482 ompt_data_t *implicit_task_data;
1483 void *return_address = NULL__null;
1484
1485 if (ompt_enabled.enabled) {
1486 __ompt_get_task_info_internal(0, NULL__null, &parent_task_data, &ompt_frame,
1487 NULL__null, NULL__null);
1488 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid);
1489 }
1490#endif
1491
1492 // Nested level will be an index in the nested nthreads array
1493 level = parent_team->t.t_level;
1494 // used to launch non-serial teams even if nested is not allowed
1495 active_level = parent_team->t.t_active_level;
1496#if OMP_40_ENABLED(50 >= 40)
1497 // needed to check nesting inside the teams
1498 teams_level = master_th->th.th_teams_level;
1499#endif
1500#if KMP_NESTED_HOT_TEAMS1
1501 p_hot_teams = &master_th->th.th_hot_teams;
1502 if (*p_hot_teams == NULL__null && __kmp_hot_teams_max_level > 0) {
1503 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1504)
1504 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level)___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1504)
;
1505 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1506 // it is either actual or not needed (when active_level > 0)
1507 (*p_hot_teams)[0].hot_team_nth = 1;
1508 }
1509#endif
1510
1511#if OMPT_SUPPORT1
1512 if (ompt_enabled.enabled) {
1513 if (ompt_enabled.ompt_callback_parallel_begin) {
1514 int team_size = master_set_numthreads
1515 ? master_set_numthreads
1516 : get__nproc_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
;
1517 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
1518 parent_task_data, ompt_frame, &ompt_parallel_data, team_size,
1519 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
, return_address);
1520 }
1521 master_th->th.ompt_thread_info.state = omp_state_overhead;
1522 }
1523#endif
1524
1525 master_th->th.th_ident = loc;
1526
1527#if OMP_40_ENABLED(50 >= 40)
1528 if (master_th->th.th_teams_microtask && ap &&
1529 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1530 // AC: This is start of parallel that is nested inside teams construct.
1531 // The team is actual (hot), all workers are ready at the fork barrier.
1532 // No lock needed to initialize the team a bit, then free workers.
1533 parent_team->t.t_ident = loc;
1534 __kmp_alloc_argv_entries(argc, parent_team, TRUE(!0));
1535 parent_team->t.t_argc = argc;
1536 argv = (void **)parent_team->t.t_argv;
1537 for (i = argc - 1; i >= 0; --i)
1538/* TODO: revert workaround for Intel(R) 64 tracker #96 */
1539#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1540 *argv++ = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
1541#else
1542 *argv++ = va_arg(ap, void *)__builtin_va_arg(ap, void *);
1543#endif
1544 // Increment our nested depth levels, but not increase the serialization
1545 if (parent_team == master_th->th.th_serial_team) {
1546 // AC: we are in serialized parallel
1547 __kmpc_serialized_parallel(loc, gtid);
1548 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1)if (!(parent_team->t.t_serialized > 1)) { __kmp_debug_assert
("parent_team->t.t_serialized > 1", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1548); }
;
1549 // AC: need this in order enquiry functions work
1550 // correctly, will restore at join time
1551 parent_team->t.t_serialized--;
1552#if OMPT_SUPPORT1
1553 void *dummy;
1554 void **exit_runtime_p;
1555
1556 ompt_lw_taskteam_t lw_taskteam;
1557
1558 if (ompt_enabled.enabled) {
1559 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1560 &ompt_parallel_data, return_address);
1561 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame);
1562
1563 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1564 // don't use lw_taskteam after linking. content was swaped
1565
1566 /* OMPT implicit task begin */
1567 implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
;
1568 if (ompt_enabled.ompt_callback_implicit_task) {
1569 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1570 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1571 implicit_task_data, 1, __kmp_tid_from_gtid(gtid));
1572 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))
1573 ->thread_num = __kmp_tid_from_gtid(gtid);
1574 }
1575
1576 /* OMPT state */
1577 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
1578 } else {
1579 exit_runtime_p = &dummy;
1580 }
1581#endif
1582
1583 {
1584 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1585 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1586 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1587#if OMPT_SUPPORT1
1588 ,
1589 exit_runtime_p
1590#endif
1591 );
1592 }
1593
1594#if OMPT_SUPPORT1
1595 *exit_runtime_p = NULL__null;
1596 if (ompt_enabled.enabled) {
1597 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->frame.exit_frame = NULL__null;
1598 if (ompt_enabled.ompt_callback_implicit_task) {
1599 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1600 ompt_scope_end, NULL__null, implicit_task_data, 1,
1601 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
1602 }
1603 __ompt_lw_taskteam_unlink(master_th);
1604
1605 if (ompt_enabled.ompt_callback_parallel_end) {
1606 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1607 OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
, OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
,
1608 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
, return_address);
1609 }
1610 master_th->th.ompt_thread_info.state = omp_state_overhead;
1611 }
1612#endif
1613 return TRUE(!0);
1614 }
1615
1616 parent_team->t.t_pkfn = microtask;
1617 parent_team->t.t_invoke = invoker;
1618 KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
)
;
1619 parent_team->t.t_active_level++;
1620 parent_team->t.t_level++;
1621#if OMP_50_ENABLED(50 >= 50)
1622 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save
1623#endif
1624
1625 /* Change number of threads in the team if requested */
1626 if (master_set_numthreads) { // The parallel has num_threads clause
1627 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1628 // AC: only can reduce number of threads dynamically, can't increase
1629 kmp_info_t **other_threads = parent_team->t.t_threads;
1630 parent_team->t.t_nproc = master_set_numthreads;
1631 for (i = 0; i < master_set_numthreads; ++i) {
1632 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1633 }
1634 // Keep extra threads hot in the team for possible next parallels
1635 }
1636 master_th->th.th_set_nproc = 0;
1637 }
1638
1639#if USE_DEBUGGER0
1640 if (__kmp_debugging) { // Let debugger override number of threads.
1641 int nth = __kmp_omp_num_threads(loc);
1642 if (nth > 0) { // 0 means debugger doesn't want to change num threads
1643 master_set_numthreads = nth;
1644 }
1645 }
1646#endif
1647
1648 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1649 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1650 root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
;
1651 __kmp_internal_fork(loc, gtid, parent_team);
1652 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1653 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1654 root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
;
1655
1656 /* Invoke microtask for MASTER thread */
1657 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
1658 parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
;
1659
1660 if (!parent_team->t.t_invoke(gtid)) {
1661 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread")if (!(0)) { __kmp_debug_assert(("cannot invoke microtask for MASTER thread"
), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1661); }
;
1662 }
1663 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
1664 parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
;
1665 KMP_MB(); /* Flush all pending memory write invalidates. */
1666
1667 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
1668
1669 return TRUE(!0);
1670 } // Parallel closely nested in teams construct
1671#endif /* OMP_40_ENABLED */
1672
1673#if KMP_DEBUG1
1674 if (__kmp_tasking_mode != tskm_immediate_exec) {
1675 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1676); }
1676 parent_team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1676); }
;
1677 }
1678#endif
1679
1680 if (parent_team->t.t_active_level >=
1681 master_th->th.th_current_task->td_icvs.max_active_levels) {
1682 nthreads = 1;
1683 } else {
1684#if OMP_40_ENABLED(50 >= 40)
1685 int enter_teams = ((ap == NULL__null && active_level == 0) ||
1686 (ap && teams_level > 0 && teams_level == level));
1687#endif
1688 nthreads =
1689 master_set_numthreads
1690 ? master_set_numthreads
1691 : get__nproc_2(((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
1692 parent_team,((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
1693 master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
; // TODO: get nproc directly from current task
1694
1695 // Check if we need to take forkjoin lock? (no need for serialized
1696 // parallel out of teams construct). This code moved here from
1697 // __kmp_reserve_threads() to speedup nested serialized parallels.
1698 if (nthreads > 1) {
1699 if ((!get__nested(master_th)(((master_th)->th.th_current_task->td_icvs.nested) ? ((
!0)) : (0))
&& (root->r.r_in_parallel
1700#if OMP_40_ENABLED(50 >= 40)
1701 && !enter_teams
1702#endif /* OMP_40_ENABLED */
1703 )) ||
1704 (__kmp_library == library_serial)) {
1705 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
1706 " threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
1707 gtid, nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
;
1708 nthreads = 1;
1709 }
1710 }
1711 if (nthreads > 1) {
1712 /* determine how many new threads we can use */
1713 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1714 nthreads = __kmp_reserve_threads(
1715 root, parent_team, master_tid, nthreads
1716#if OMP_40_ENABLED(50 >= 40)
1717 /* AC: If we execute teams from parallel region (on host), then
1718 teams should be created but each can only have 1 thread if
1719 nesting is disabled. If teams called from serial region, then
1720 teams and their threads should be created regardless of the
1721 nesting setting. */
1722 ,
1723 enter_teams
1724#endif /* OMP_40_ENABLED */
1725 );
1726 if (nthreads == 1) {
1727 // Free lock for single thread execution here; for multi-thread
1728 // execution it will be freed later after team of threads created
1729 // and initialized
1730 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1731 }
1732 }
1733 }
1734 KMP_DEBUG_ASSERT(nthreads > 0)if (!(nthreads > 0)) { __kmp_debug_assert("nthreads > 0"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1734); }
;
1735
1736 // If we temporarily changed the set number of threads then restore it now
1737 master_th->th.th_set_nproc = 0;
1738
1739 /* create a serialized parallel region? */
1740 if (nthreads == 1) {
1741/* josh todo: hypothetical question: what do we do for OS X*? */
1742#if KMP_OS_LINUX1 && \
1743 (KMP_ARCH_X860 || KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640)
1744 void *args[argc];
1745#else
1746 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *))__builtin_alloca (argc * sizeof(void *));
1747#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \
1748 KMP_ARCH_AARCH64) */
1749
1750 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing parallel region\n"
, gtid); }
1751 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing parallel region\n"
, gtid); }
;
1752
1753 __kmpc_serialized_parallel(loc, gtid);
1754
1755 if (call_context == fork_context_intel) {
1756 /* TODO this sucks, use the compiler itself to pass args! :) */
1757 master_th->th.th_serial_team->t.t_ident = loc;
1758#if OMP_40_ENABLED(50 >= 40)
1759 if (!ap) {
1760 // revert change made in __kmpc_serialized_parallel()
1761 master_th->th.th_serial_team->t.t_level--;
1762// Get args from parent team for teams construct
1763
1764#if OMPT_SUPPORT1
1765 void *dummy;
1766 void **exit_runtime_p;
1767 ompt_task_info_t *task_info;
1768
1769 ompt_lw_taskteam_t lw_taskteam;
1770
1771 if (ompt_enabled.enabled) {
1772 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1773 &ompt_parallel_data, return_address);
1774
1775 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1776 // don't use lw_taskteam after linking. content was swaped
1777
1778 task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
1779 exit_runtime_p = &(task_info->frame.exit_frame);
1780 if (ompt_enabled.ompt_callback_implicit_task) {
1781 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1782 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1783 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid));
1784 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))
1785 ->thread_num = __kmp_tid_from_gtid(gtid);
1786 }
1787
1788 /* OMPT state */
1789 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
1790 } else {
1791 exit_runtime_p = &dummy;
1792 }
1793#endif
1794
1795 {
1796 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1797 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1798 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1799 parent_team->t.t_argv
1800#if OMPT_SUPPORT1
1801 ,
1802 exit_runtime_p
1803#endif
1804 );
1805 }
1806
1807#if OMPT_SUPPORT1
1808 if (ompt_enabled.enabled) {
1809 exit_runtime_p = NULL__null;
1810 if (ompt_enabled.ompt_callback_implicit_task) {
1811 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1812 ompt_scope_end, NULL__null, &(task_info->task_data), 1,
1813 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
1814 }
1815
1816 __ompt_lw_taskteam_unlink(master_th);
1817 if (ompt_enabled.ompt_callback_parallel_end) {
1818 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1819 OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
, parent_task_data,
1820 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
, return_address);
1821 }
1822 master_th->th.ompt_thread_info.state = omp_state_overhead;
1823 }
1824#endif
1825 } else if (microtask == (microtask_t)__kmp_teams_master) {
1826 KMP_DEBUG_ASSERT(master_th->th.th_team ==if (!(master_th->th.th_team == master_th->th.th_serial_team
)) { __kmp_debug_assert("master_th->th.th_team == master_th->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1827); }
1827 master_th->th.th_serial_team)if (!(master_th->th.th_team == master_th->th.th_serial_team
)) { __kmp_debug_assert("master_th->th.th_team == master_th->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1827); }
;
1828 team = master_th->th.th_team;
1829 // team->t.t_pkfn = microtask;
1830 team->t.t_invoke = invoker;
1831 __kmp_alloc_argv_entries(argc, team, TRUE(!0));
1832 team->t.t_argc = argc;
1833 argv = (void **)team->t.t_argv;
1834 if (ap) {
1835 for (i = argc - 1; i >= 0; --i)
1836// TODO: revert workaround for Intel(R) 64 tracker #96
1837#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1838 *argv++ = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
1839#else
1840 *argv++ = va_arg(ap, void *)__builtin_va_arg(ap, void *);
1841#endif
1842 } else {
1843 for (i = 0; i < argc; ++i)
1844 // Get args from parent team for teams construct
1845 argv[i] = parent_team->t.t_argv[i];
1846 }
1847 // AC: revert change made in __kmpc_serialized_parallel()
1848 // because initial code in teams should have level=0
1849 team->t.t_level--;
1850 // AC: call special invoker for outer "parallel" of teams construct
1851 invoker(gtid);
1852 } else {
1853#endif /* OMP_40_ENABLED */
1854 argv = args;
1855 for (i = argc - 1; i >= 0; --i)
1856// TODO: revert workaround for Intel(R) 64 tracker #96
1857#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1858 *argv++ = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
1859#else
1860 *argv++ = va_arg(ap, void *)__builtin_va_arg(ap, void *);
1861#endif
1862 KMP_MB();
1863
1864#if OMPT_SUPPORT1
1865 void *dummy;
1866 void **exit_runtime_p;
1867 ompt_task_info_t *task_info;
1868
1869 ompt_lw_taskteam_t lw_taskteam;
1870
1871 if (ompt_enabled.enabled) {
1872 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1873 &ompt_parallel_data, return_address);
1874 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1875 // don't use lw_taskteam after linking. content was swaped
1876 task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
1877 exit_runtime_p = &(task_info->frame.exit_frame);
1878
1879 /* OMPT implicit task begin */
1880 implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
;
1881 if (ompt_enabled.ompt_callback_implicit_task) {
1882 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1883 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1884 implicit_task_data, 1, __kmp_tid_from_gtid(gtid));
1885 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))
1886 ->thread_num = __kmp_tid_from_gtid(gtid);
1887 }
1888
1889 /* OMPT state */
1890 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
1891 } else {
1892 exit_runtime_p = &dummy;
1893 }
1894#endif
1895
1896 {
1897 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1898 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1899 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1900#if OMPT_SUPPORT1
1901 ,
1902 exit_runtime_p
1903#endif
1904 );
1905 }
1906
1907#if OMPT_SUPPORT1
1908 if (ompt_enabled.enabled) {
1909 *exit_runtime_p = NULL__null;
1910 if (ompt_enabled.ompt_callback_implicit_task) {
1911 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1912 ompt_scope_end, NULL__null, &(task_info->task_data), 1,
1913 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
1914 }
1915
1916 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
;
1917 __ompt_lw_taskteam_unlink(master_th);
1918 if (ompt_enabled.ompt_callback_parallel_end) {
1919 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1920 &ompt_parallel_data, parent_task_data,
1921 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
, return_address);
1922 }
1923 master_th->th.ompt_thread_info.state = omp_state_overhead;
1924 }
1925#endif
1926#if OMP_40_ENABLED(50 >= 40)
1927 }
1928#endif /* OMP_40_ENABLED */
1929 } else if (call_context == fork_context_gnu) {
1930#if OMPT_SUPPORT1
1931 ompt_lw_taskteam_t lwt;
1932 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1933 return_address);
1934
1935 lwt.ompt_task_info.frame.exit_frame = NULL__null;
1936 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1937// don't use lw_taskteam after linking. content was swaped
1938#endif
1939
1940 // we were called from GNU native code
1941 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serial exit\n"
, gtid); }
;
1942 return FALSE0;
1943 } else {
1944 KMP_ASSERT2(call_context < fork_context_last,if (!(call_context < fork_context_last)) { __kmp_debug_assert
(("__kmp_fork_call: unknown fork_context parameter"), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1945); }
1945 "__kmp_fork_call: unknown fork_context parameter")if (!(call_context < fork_context_last)) { __kmp_debug_assert
(("__kmp_fork_call: unknown fork_context parameter"), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1945); }
;
1946 }
1947
1948 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serial exit\n"
, gtid); }
;
1949 KMP_MB();
1950 return FALSE0;
1951 }
1952
1953 // GEH: only modify the executing flag in the case when not serialized
1954 // serialized case is handled in kmpc_serialized_parallel
1955 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1956 "curtask=%p, curtask_max_aclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1957 parent_team->t.t_active_level, master_th,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1958 master_th->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1959 master_th->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
;
1960 // TODO: GEH - cannot do this assertion because root thread not set up as
1961 // executing
1962 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
1963 master_th->th.th_current_task->td_flags.executing = 0;
1964
1965#if OMP_40_ENABLED(50 >= 40)
1966 if (!master_th->th.th_teams_microtask || level > teams_level)
1967#endif /* OMP_40_ENABLED */
1968 {
1969 /* Increment our nested depth level */
1970 KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
)
;
1971 }
1972
1973 // See if we need to make a copy of the ICVs.
1974 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
1975 if ((level + 1 < __kmp_nested_nth.used) &&
1976 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1977 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1978 } else {
1979 nthreads_icv = 0; // don't update
1980 }
1981
1982#if OMP_40_ENABLED(50 >= 40)
1983 // Figure out the proc_bind_policy for the new team.
1984 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
1985 kmp_proc_bind_t proc_bind_icv =
1986 proc_bind_default; // proc_bind_default means don't update
1987 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1988 proc_bind = proc_bind_false;
1989 } else {
1990 if (proc_bind == proc_bind_default) {
1991 // No proc_bind clause specified; use current proc-bind-var for this
1992 // parallel region
1993 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
1994 }
1995 /* else: The proc_bind policy was specified explicitly on parallel clause.
1996 This overrides proc-bind-var for this parallel region, but does not
1997 change proc-bind-var. */
1998 // Figure the value of proc-bind-var for the child threads.
1999 if ((level + 1 < __kmp_nested_proc_bind.used) &&
2000 (__kmp_nested_proc_bind.bind_types[level + 1] !=
2001 master_th->th.th_current_task->td_icvs.proc_bind)) {
2002 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
2003 }
2004 }
2005
2006 // Reset for next parallel region
2007 master_th->th.th_set_proc_bind = proc_bind_default;
2008#endif /* OMP_40_ENABLED */
2009
2010 if ((nthreads_icv > 0)
2011#if OMP_40_ENABLED(50 >= 40)
2012 || (proc_bind_icv != proc_bind_default)
2013#endif /* OMP_40_ENABLED */
2014 ) {
2015 kmp_internal_control_t new_icvs;
2016 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2017 new_icvs.next = NULL__null;
2018 if (nthreads_icv > 0) {
2019 new_icvs.nproc = nthreads_icv;
2020 }
2021
2022#if OMP_40_ENABLED(50 >= 40)
2023 if (proc_bind_icv != proc_bind_default) {
2024 new_icvs.proc_bind = proc_bind_icv;
2025 }
2026#endif /* OMP_40_ENABLED */
2027
2028 /* allocate a new parallel team */
2029 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); }
;
2030 team = __kmp_allocate_team(root, nthreads, nthreads,
2031#if OMPT_SUPPORT1
2032 ompt_parallel_data,
2033#endif
2034#if OMP_40_ENABLED(50 >= 40)
2035 proc_bind,
2036#endif
2037 &new_icvs, argc USE_NESTED_HOT_ARG(master_th), master_th);
2038 } else {
2039 /* allocate a new parallel team */
2040 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); }
;
2041 team = __kmp_allocate_team(root, nthreads, nthreads,
2042#if OMPT_SUPPORT1
2043 ompt_parallel_data,
2044#endif
2045#if OMP_40_ENABLED(50 >= 40)
2046 proc_bind,
2047#endif
2048 &master_th->th.th_current_task->td_icvs,
2049 argc USE_NESTED_HOT_ARG(master_th), master_th);
2050 }
2051 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); }
2052 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); }
;
2053
2054 /* setup the new team */
2055 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid)if ((team->t.t_master_tid) != (master_tid)) (team->t.t_master_tid
) = (master_tid)
;
2056 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons)if ((team->t.t_master_this_cons) != (master_this_cons)) (team
->t.t_master_this_cons) = (master_this_cons)
;
2057 KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
)
;
2058 KMP_CHECK_UPDATE(team->t.t_parent, parent_team)if ((team->t.t_parent) != (parent_team)) (team->t.t_parent
) = (parent_team)
;
2059 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask)if ((team->t.t_pkfn) != (microtask)) (((team->t.t_pkfn)
)) = (((microtask)))
;
2060#if OMPT_SUPPORT1
2061 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)))
2062 return_address)if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)))
;
2063#endif
2064 KMP_CHECK_UPDATE(team->t.t_invoke, invoker)if ((team->t.t_invoke) != (invoker)) (team->t.t_invoke)
= (invoker)
; // TODO move to root, maybe
2065// TODO: parent_team->t.t_level == INT_MAX ???
2066#if OMP_40_ENABLED(50 >= 40)
2067 if (!master_th->th.th_teams_microtask || level > teams_level) {
2068#endif /* OMP_40_ENABLED */
2069 int new_level = parent_team->t.t_level + 1;
2070 KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
= (new_level)
;
2071 new_level = parent_team->t.t_active_level + 1;
2072 KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level)
;
2073#if OMP_40_ENABLED(50 >= 40)
2074 } else {
2075 // AC: Do not increase parallel level at start of the teams construct
2076 int new_level = parent_team->t.t_level;
2077 KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
= (new_level)
;
2078 new_level = parent_team->t.t_active_level;
2079 KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level)
;
2080 }
2081#endif /* OMP_40_ENABLED */
2082 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.sched)
;
2083 // set master's schedule as new run-time schedule
2084 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched)if ((team->t.t_sched.sched) != (new_sched.sched)) (team->
t.t_sched.sched) = (new_sched.sched)
;
2085
2086#if OMP_40_ENABLED(50 >= 40)
2087 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq)if ((team->t.t_cancel_request) != (cancel_noreq)) (team->
t.t_cancel_request) = (cancel_noreq)
;
2088#endif
2089#if OMP_50_ENABLED(50 >= 50)
2090 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator)if ((team->t.t_def_allocator) != (master_th->th.th_def_allocator
)) (team->t.t_def_allocator) = (master_th->th.th_def_allocator
)
;
2091#endif
2092
2093 // Update the floating point rounding in the team if required.
2094 propagateFPControl(team);
2095
2096 if (__kmp_tasking_mode != tskm_immediate_exec) {
2097 // Set master's task team to team's task team. Unless this is hot team, it
2098 // should be NULL.
2099 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2100); }
2100 parent_team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2100); }
;
2101 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2102 "%p, new task_team %p / team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2103 __kmp_gtid_from_thread(master_th),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2104 master_th->th.th_task_team, parent_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
2105 team->t.t_task_team[master_th->th.th_task_state], team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
;
2106
2107 if (active_level || master_th->th.th_task_team) {
2108 // Take a memo of master's task_state
2109 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)if (!(master_th->th.th_task_state_memo_stack)) { __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2109); }
;
2110 if (master_th->th.th_task_state_top >=
2111 master_th->th.th_task_state_stack_sz) { // increase size
2112 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2113 kmp_uint8 *old_stack, *new_stack;
2114 kmp_uint32 i;
2115 new_stack = (kmp_uint8 *)__kmp_allocate(new_size)___kmp_allocate((new_size), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2115)
;
2116 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2117 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2118 }
2119 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2120 ++i) { // zero-init rest of stack
2121 new_stack[i] = 0;
2122 }
2123 old_stack = master_th->th.th_task_state_memo_stack;
2124 master_th->th.th_task_state_memo_stack = new_stack;
2125 master_th->th.th_task_state_stack_sz = new_size;
2126 __kmp_free(old_stack)___kmp_free((old_stack), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2126)
;
2127 }
2128 // Store master's task_state on stack
2129 master_th->th
2130 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2131 master_th->th.th_task_state;
2132 master_th->th.th_task_state_top++;
2133#if KMP_NESTED_HOT_TEAMS1
2134 if (master_th->th.th_hot_teams &&
2135 team == master_th->th.th_hot_teams[active_level].hot_team) {
2136 // Restore master's nested state if nested hot team
2137 master_th->th.th_task_state =
2138 master_th->th
2139 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2140 } else {
2141#endif
2142 master_th->th.th_task_state = 0;
2143#if KMP_NESTED_HOT_TEAMS1
2144 }
2145#endif
2146 }
2147#if !KMP_NESTED_HOT_TEAMS1
2148 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||if (!((master_th->th.th_task_team == __null) || (team == root
->r.r_hot_team))) { __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2149); }
2149 (team == root->r.r_hot_team))if (!((master_th->th.th_task_team == __null) || (team == root
->r.r_hot_team))) { __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2149); }
;
2150#endif
2151 }
2152
2153 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2154 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2155 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2156 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2157 team->t.t_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
;
2158 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2161); }
2159 (team->t.t_master_tid == 0 &&if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2161); }
2160 (team->t.t_parent == root->r.r_root_team ||if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2161); }
2161 team->t.t_parent->t.t_serialized)))if (!(team != root->r.r_hot_team || (team->t.t_master_tid
== 0 && (team->t.t_parent == root->r.r_root_team
|| team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2161); }
;
2162 KMP_MB();
2163
2164 /* now, setup the arguments */
2165 argv = (void **)team->t.t_argv;
2166#if OMP_40_ENABLED(50 >= 40)
2167 if (ap) {
2168#endif /* OMP_40_ENABLED */
2169 for (i = argc - 1; i >= 0; --i) {
2170// TODO: revert workaround for Intel(R) 64 tracker #96
2171#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
2172 void *new_argv = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
2173#else
2174 void *new_argv = va_arg(ap, void *)__builtin_va_arg(ap, void *);
2175#endif
2176 KMP_CHECK_UPDATE(*argv, new_argv)if ((*argv) != (new_argv)) (*argv) = (new_argv);
2177 argv++;
2178 }
2179#if OMP_40_ENABLED(50 >= 40)
2180 } else {
2181 for (i = 0; i < argc; ++i) {
2182 // Get args from parent team for teams construct
2183 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i])if ((argv[i]) != (team->t.t_parent->t.t_argv[i])) (argv
[i]) = (team->t.t_parent->t.t_argv[i])
;
2184 }
2185 }
2186#endif /* OMP_40_ENABLED */
2187
2188 /* now actually fork the threads */
2189 KMP_CHECK_UPDATE(team->t.t_master_active, master_active)if ((team->t.t_master_active) != (master_active)) (team->
t.t_master_active) = (master_active)
;
2190 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
2191 root->r.r_active = TRUE(!0);
2192
2193 __kmp_fork_team_threads(root, team, master_th, gtid);
2194 __kmp_setup_icv_copy(team, nthreads,
2195 &master_th->th.th_current_task->td_icvs, loc);
2196
2197#if OMPT_SUPPORT1
2198 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
2199#endif
2200
2201 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2202
2203#if USE_ITT_BUILD1
2204 if (team->t.t_active_level == 1 // only report frames at level 1
2205#if OMP_40_ENABLED(50 >= 40)
2206 && !master_th->th.th_teams_microtask // not in teams construct
2207#endif /* OMP_40_ENABLED */
2208 ) {
2209#if USE_ITT_NOTIFY1
2210 if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2211 (__kmp_forkjoin_frames_mode == 3 ||
2212 __kmp_forkjoin_frames_mode == 1)) {
2213 kmp_uint64 tmp_time = 0;
2214 if (__itt_get_timestamp_ptr__kmp_itt_get_timestamp_ptr__3_0)
2215 tmp_time = __itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0();
2216 // Internal fork - report frame begin
2217 master_th->th.th_frame_time = tmp_time;
2218 if (__kmp_forkjoin_frames_mode == 3)
2219 team->t.t_region_time = tmp_time;
2220 } else
2221// only one notification scheme (either "submit" or "forking/joined", not both)
2222#endif /* USE_ITT_NOTIFY */
2223 if ((__itt_frame_begin_v3_ptr__kmp_itt_frame_begin_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2224 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2225 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer.
2226 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2227 }
2228 }
2229#endif /* USE_ITT_BUILD */
2230
2231 /* now go on and do the work */
2232 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team)if (!(team == __kmp_threads[gtid]->th.th_team)) { __kmp_debug_assert
("team == __kmp_threads[gtid]->th.th_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2232); }
;
2233 KMP_MB();
2234 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
2235 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
2236 root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
;
2237
2238#if USE_ITT_BUILD1
2239 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
2240 team->t.t_stack_id =
2241 __kmp_itt_stack_caller_create(); // create new stack stitching id
2242 // before entering fork barrier
2243 }
2244#endif /* USE_ITT_BUILD */
2245
2246#if OMP_40_ENABLED(50 >= 40)
2247 // AC: skip __kmp_internal_fork at teams construct, let only master
2248 // threads execute
2249 if (ap)
2250#endif /* OMP_40_ENABLED */
2251 {
2252 __kmp_internal_fork(loc, gtid, team);
2253 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
2254 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
2255 root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
;
2256 }
2257
2258 if (call_context == fork_context_gnu) {
2259 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
2260 return TRUE(!0);
2261 }
2262
2263 /* Invoke microtask for MASTER thread */
2264 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
2265 team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
;
2266 } // END of timer KMP_fork_call block
2267
2268 if (!team->t.t_invoke(gtid)) {
2269 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread")if (!(0)) { __kmp_debug_assert(("cannot invoke microtask for MASTER thread"
), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2269); }
;
2270 }
2271 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
2272 team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
;
2273 KMP_MB(); /* Flush all pending memory write invalidates. */
2274
2275 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
2276
2277#if OMPT_SUPPORT1
2278 if (ompt_enabled.enabled) {
2279 master_th->th.ompt_thread_info.state = omp_state_overhead;
2280 }
2281#endif
2282
2283 return TRUE(!0);
2284}
2285
2286#if OMPT_SUPPORT1
2287static inline void __kmp_join_restore_state(kmp_info_t *thread,
2288 kmp_team_t *team) {
2289 // restore state outside the region
2290 thread->th.ompt_thread_info.state =
2291 ((team->t.t_serialized) ? omp_state_work_serial
2292 : omp_state_work_parallel);
2293}
2294
2295static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread,
2296 kmp_team_t *team, ompt_data_t *parallel_data,
2297 fork_context_e fork_context, void *codeptr) {
2298 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2299 if (ompt_enabled.ompt_callback_parallel_end) {
2300 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
2301 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context)((fork_context == fork_context_gnu) ? ompt_parallel_invoker_program
: ompt_parallel_invoker_runtime)
,
2302 codeptr);
2303 }
2304
2305 task_info->frame.enter_frame = NULL__null;
2306 __kmp_join_restore_state(thread, team);
2307}
2308#endif
2309
2310void __kmp_join_call(ident_t *loc, int gtid
2311#if OMPT_SUPPORT1
2312 ,
2313 enum fork_context_e fork_context
2314#endif
2315#if OMP_40_ENABLED(50 >= 40)
2316 ,
2317 int exit_teams
2318#endif /* OMP_40_ENABLED */
2319 ) {
2320 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call)((void)0);
2321 kmp_team_t *team;
2322 kmp_team_t *parent_team;
2323 kmp_info_t *master_th;
2324 kmp_root_t *root;
2325 int master_active;
2326 int i;
2327
2328 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: enter T#%d\n"
, gtid); }
;
2329
2330 /* setup current data */
2331 master_th = __kmp_threads[gtid];
2332 root = master_th->th.th_root;
2333 team = master_th->th.th_team;
2334 parent_team = team->t.t_parent;
2335
2336 master_th->th.th_ident = loc;
2337
2338#if OMPT_SUPPORT1
2339 if (ompt_enabled.enabled) {
2340 master_th->th.ompt_thread_info.state = omp_state_overhead;
2341 }
2342#endif
2343
2344#if KMP_DEBUG1
2345 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2346 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2347 "th_task_team = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2348 __kmp_gtid_from_thread(master_th), team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2349 team->t.t_task_team[master_th->th.th_task_state],if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
2350 master_th->th.th_task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
;
2351 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2352); }
2352 team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2352); }
;
2353 }
2354#endif
2355
2356 if (team->t.t_serialized) {
2357#if OMP_40_ENABLED(50 >= 40)
2358 if (master_th->th.th_teams_microtask) {
2359 // We are in teams construct
2360 int level = team->t.t_level;
2361 int tlevel = master_th->th.th_teams_level;
2362 if (level == tlevel) {
2363 // AC: we haven't incremented it earlier at start of teams construct,
2364 // so do it here - at the end of teams construct
2365 team->t.t_level++;
2366 } else if (level == tlevel + 1) {
2367 // AC: we are exiting parallel inside teams, need to increment
2368 // serialization in order to restore it in the next call to
2369 // __kmpc_end_serialized_parallel
2370 team->t.t_serialized++;
2371 }
2372 }
2373#endif /* OMP_40_ENABLED */
2374 __kmpc_end_serialized_parallel(loc, gtid);
2375
2376#if OMPT_SUPPORT1
2377 if (ompt_enabled.enabled) {
2378 __kmp_join_restore_state(master_th, parent_team);
2379 }
2380#endif
2381
2382 return;
2383 }
2384
2385 master_active = team->t.t_master_active;
2386
2387#if OMP_40_ENABLED(50 >= 40)
2388 if (!exit_teams)
2389#endif /* OMP_40_ENABLED */
2390 {
2391 // AC: No barrier for internal teams at exit from teams construct.
2392 // But there is barrier for external team (league).
2393 __kmp_internal_join(loc, gtid, team);
2394 }
2395#if OMP_40_ENABLED(50 >= 40)
2396 else {
2397 master_th->th.th_task_state =
2398 0; // AC: no tasking in teams (out of any parallel)
2399 }
2400#endif /* OMP_40_ENABLED */
2401
2402 KMP_MB();
2403
2404#if OMPT_SUPPORT1
2405 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2406 void *codeptr = team->t.ompt_team_info.master_return_address;
2407#endif
2408
2409#if USE_ITT_BUILD1
2410 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
2411 __kmp_itt_stack_caller_destroy(
2412 (__itt_caller)team->t
2413 .t_stack_id); // destroy the stack stitching id after join barrier
2414 }
2415
2416 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer.
2417 if (team->t.t_active_level == 1
2418#if OMP_40_ENABLED(50 >= 40)
2419 && !master_th->th.th_teams_microtask /* not in teams construct */
2420#endif /* OMP_40_ENABLED */
2421 ) {
2422 master_th->th.th_ident = loc;
2423 // only one notification scheme (either "submit" or "forking/joined", not
2424 // both)
2425 if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2426 __kmp_forkjoin_frames_mode == 3)
2427 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2428 master_th->th.th_frame_time, 0, loc,
2429 master_th->th.th_team_nproc, 1);
2430 else if ((__itt_frame_end_v3_ptr__kmp_itt_frame_end_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2431 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2432 __kmp_itt_region_joined(gtid);
2433 } // active_level == 1
2434#endif /* USE_ITT_BUILD */
2435
2436#if OMP_40_ENABLED(50 >= 40)
2437 if (master_th->th.th_teams_microtask && !exit_teams &&
2438 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2439 team->t.t_level == master_th->th.th_teams_level + 1) {
2440 // AC: We need to leave the team structure intact at the end of parallel
2441 // inside the teams construct, so that at the next parallel same (hot) team
2442 // works, only adjust nesting levels
2443
2444 /* Decrement our nested depth level */
2445 team->t.t_level--;
2446 team->t.t_active_level--;
2447 KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
)
;
2448
2449 /* Restore number of threads in the team if needed */
2450 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2451 int old_num = master_th->th.th_team_nproc;
2452 int new_num = master_th->th.th_teams_size.nth;
2453 kmp_info_t **other_threads = team->t.t_threads;
2454 team->t.t_nproc = new_num;
2455 for (i = 0; i < old_num; ++i) {
2456 other_threads[i]->th.th_team_nproc = new_num;
2457 }
2458 // Adjust states of non-used threads of the team
2459 for (i = old_num; i < new_num; ++i) {
2460 // Re-initialize thread's barrier data.
2461 int b;
2462 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2463 for (b = 0; b < bs_last_barrier; ++b) {
2464 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2465 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2465); }
;
2466#if USE_DEBUGGER0
2467 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2468#endif
2469 }
2470 if (__kmp_tasking_mode != tskm_immediate_exec) {
2471 // Synchronize thread's task state
2472 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2473 }
2474 }
2475 }
2476
2477#if OMPT_SUPPORT1
2478 if (ompt_enabled.enabled) {
2479 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2480 codeptr);
2481 }
2482#endif
2483
2484 return;
2485 }
2486#endif /* OMP_40_ENABLED */
2487
2488 /* do cleanup and restore the parent team */
2489 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2490 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2491
2492 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2493
2494 /* jc: The following lock has instructions with REL and ACQ semantics,
2495 separating the parallel user code called in this parallel region
2496 from the serial user code called after this function returns. */
2497 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2498
2499#if OMP_40_ENABLED(50 >= 40)
2500 if (!master_th->th.th_teams_microtask ||
2501 team->t.t_level > master_th->th.th_teams_level)
2502#endif /* OMP_40_ENABLED */
2503 {
2504 /* Decrement our nested depth level */
2505 KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
)
;
2506 }
2507 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0)if (!(root->r.r_in_parallel >= 0)) { __kmp_debug_assert
("root->r.r_in_parallel >= 0", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2507); }
;
2508
2509#if OMPT_SUPPORT1
2510 if (ompt_enabled.enabled) {
2511 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2512 if (ompt_enabled.ompt_callback_implicit_task) {
2513 int ompt_team_size = team->t.t_nproc;
2514 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
2515 ompt_scope_end, NULL__null, &(task_info->task_data), ompt_team_size,
2516 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
2517 }
2518
2519 task_info->frame.exit_frame = NULL__null;
2520 task_info->task_data = ompt_data_none;
2521 }
2522#endif
2523
2524 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); }
2525 master_th, team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); }
;
2526 __kmp_pop_current_task_from_thread(master_th);
2527
2528#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
2529 // Restore master thread's partition.
2530 master_th->th.th_first_place = team->t.t_first_place;
2531 master_th->th.th_last_place = team->t.t_last_place;
2532#endif /* OMP_40_ENABLED */
2533#if OMP_50_ENABLED(50 >= 50)
2534 master_th->th.th_def_allocator = team->t.t_def_allocator;
2535#endif
2536
2537 updateHWFPControl(team);
2538
2539 if (root->r.r_active != master_active)
2540 root->r.r_active = master_active;
2541
2542 __kmp_free_team(root, team USE_NESTED_HOT_ARG(, master_th
2543 master_th), master_th); // this will free worker threads
2544
2545 /* this race was fun to find. make sure the following is in the critical
2546 region otherwise assertions may fail occasionally since the old team may be
2547 reallocated and the hierarchy appears inconsistent. it is actually safe to
2548 run and won't cause any bugs, but will cause those assertion failures. it's
2549 only one deref&assign so might as well put this in the critical region */
2550 master_th->th.th_team = parent_team;
2551 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2552 master_th->th.th_team_master = parent_team->t.t_threads[0];
2553 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2554
2555 /* restore serialized team, if need be */
2556 if (parent_team->t.t_serialized &&
2557 parent_team != master_th->th.th_serial_team &&
2558 parent_team != root->r.r_root_team) {
2559 __kmp_free_team(root,
2560 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL), __null);
2561 master_th->th.th_serial_team = parent_team;
2562 }
2563
2564 if (__kmp_tasking_mode != tskm_immediate_exec) {
2565 if (master_th->th.th_task_state_top >
2566 0) { // Restore task state from memo stack
2567 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)if (!(master_th->th.th_task_state_memo_stack)) { __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2567); }
;
2568 // Remember master's state if we re-use this nested hot team
2569 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2570 master_th->th.th_task_state;
2571 --master_th->th.th_task_state_top; // pop
2572 // Now restore state at this level
2573 master_th->th.th_task_state =
2574 master_th->th
2575 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2576 }
2577 // Copy the task team from the parent team to the master thread
2578 master_th->th.th_task_team =
2579 parent_team->t.t_task_team[master_th->th.th_task_state];
2580 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
2581 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
2582 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
2583 parent_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
;
2584 }
2585
2586 // TODO: GEH - cannot do this assertion because root thread not set up as
2587 // executing
2588 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
2589 master_th->th.th_current_task->td_flags.executing = 1;
2590
2591 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2592
2593#if OMPT_SUPPORT1
2594 if (ompt_enabled.enabled) {
2595 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2596 codeptr);
2597 }
2598#endif
2599
2600 KMP_MB();
2601 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: exit T#%d\n"
, gtid); }
;
2602}
2603
2604/* Check whether we should push an internal control record onto the
2605 serial team stack. If so, do it. */
2606void __kmp_save_internal_controls(kmp_info_t *thread) {
2607
2608 if (thread->th.th_team != thread->th.th_serial_team) {
2609 return;
2610 }
2611 if (thread->th.th_team->t.t_serialized > 1) {
2612 int push = 0;
2613
2614 if (thread->th.th_team->t.t_control_stack_top == NULL__null) {
2615 push = 1;
2616 } else {
2617 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2618 thread->th.th_team->t.t_serialized) {
2619 push = 1;
2620 }
2621 }
2622 if (push) { /* push a record on the serial team's stack */
2623 kmp_internal_control_t *control =
2624 (kmp_internal_control_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_internal_control_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2625)
2625 sizeof(kmp_internal_control_t))___kmp_allocate((sizeof(kmp_internal_control_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2625)
;
2626
2627 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2628
2629 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2630
2631 control->next = thread->th.th_team->t.t_control_stack_top;
2632 thread->th.th_team->t.t_control_stack_top = control;
2633 }
2634 }
2635}
2636
2637/* Changes set_nproc */
2638void __kmp_set_num_threads(int new_nth, int gtid) {
2639 kmp_info_t *thread;
2640 kmp_root_t *root;
2641
2642 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_num_threads: new __kmp_nth = %d\n"
, new_nth); }
;
2643 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2643); }
;
2644
2645 if (new_nth < 1)
1
Assuming 'new_nth' is < 1
2
Taking true branch
2646 new_nth = 1;
2647 else if (new_nth > __kmp_max_nth)
2648 new_nth = __kmp_max_nth;
2649
2650 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth)((void)0);
2651 thread = __kmp_threads[gtid];
2652
2653 __kmp_save_internal_controls(thread);
2654
2655 set__nproc(thread, new_nth)(((thread)->th.th_current_task->td_icvs.nproc) = (new_nth
))
;
2656
2657 // If this omp_set_num_threads() call will cause the hot team size to be
2658 // reduced (in the absence of a num_threads clause), then reduce it now,
2659 // rather than waiting for the next parallel region.
2660 root = thread->th.th_root;
2661 if (__kmp_init_parallel && (!root->r.r_active) &&
3
Assuming '__kmp_init_parallel' is not equal to 0
4
Assuming the condition is true
8
Taking true branch
2662 (root->r.r_hot_team->t.t_nproc > new_nth)
5
Assuming the condition is true
2663#if KMP_NESTED_HOT_TEAMS1
2664 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
6
Assuming '__kmp_hot_teams_max_level' is not equal to 0
7
Assuming '__kmp_hot_teams_mode' is 0
2665#endif
2666 ) {
2667 kmp_team_t *hot_team = root->r.r_hot_team;
2668 int f;
2669
2670 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2671
2672 // Release the extra threads we don't need any more.
2673 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
9
Loop condition is true. Entering loop body
2674 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)if (!(hot_team->t.t_threads[f] != __null)) { __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2674); }
;
10
Within the expansion of the macro 'KMP_DEBUG_ASSERT':
a
Assuming pointer value is null
2675 if (__kmp_tasking_mode != tskm_immediate_exec) {
11
Assuming '__kmp_tasking_mode' is not equal to tskm_immediate_exec
12
Taking true branch
2676 // When decreasing team size, threads no longer in the team should unref
2677 // task team.
2678 hot_team->t.t_threads[f]->th.th_task_team = NULL__null;
13
Dereference of null pointer
2679 }
2680 __kmp_free_thread(hot_team->t.t_threads[f]);
2681 hot_team->t.t_threads[f] = NULL__null;
2682 }
2683 hot_team->t.t_nproc = new_nth;
2684#if KMP_NESTED_HOT_TEAMS1
2685 if (thread->th.th_hot_teams) {
2686 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team)if (!(hot_team == thread->th.th_hot_teams[0].hot_team)) { __kmp_debug_assert
("hot_team == thread->th.th_hot_teams[0].hot_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2686); }
;
2687 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2688 }
2689#endif
2690
2691 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2692
2693 // Update the t_nproc field in the threads that are still active.
2694 for (f = 0; f < new_nth; f++) {
2695 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)if (!(hot_team->t.t_threads[f] != __null)) { __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2695); }
;
2696 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2697 }
2698 // Special flag in case omp_set_num_threads() call
2699 hot_team->t.t_size_changed = -1;
2700 }
2701}
2702
2703/* Changes max_active_levels */
2704void __kmp_set_max_active_levels(int gtid, int max_active_levels) {
2705 kmp_info_t *thread;
2706
2707 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
2708 "%d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
2709 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
;
2710 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2710); }
;
2711
2712 // validate max_active_levels
2713 if (max_active_levels < 0) {
2714 KMP_WARNING(ActiveLevelsNegative, max_active_levels)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsNegative
, max_active_levels), __kmp_msg_null)
;
2715 // We ignore this call if the user has specified a negative value.
2716 // The current setting won't be changed. The last valid setting will be
2717 // used. A warning will be issued (if warnings are allowed as controlled by
2718 // the KMP_WARNINGS env var).
2719 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2720 "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2721 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
;
2722 return;
2723 }
2724 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647) {
2725 // it's OK, the max_active_levels is within the valid range: [ 0;
2726 // KMP_MAX_ACTIVE_LEVELS_LIMIT ]
2727 // We allow a zero value. (implementation defined behavior)
2728 } else {
2729 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null)
2730 KMP_MAX_ACTIVE_LEVELS_LIMIT)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null)
;
2731 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647;
2732 // Current upper limit is MAX_INT. (implementation defined behavior)
2733 // If the input exceeds the upper limit, we correct the input to be the
2734 // upper limit. (implementation defined behavior)
2735 // Actually, the flow should never get here until we use MAX_INT limit.
2736 }
2737 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2738 "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2739 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
;
2740
2741 thread = __kmp_threads[gtid];
2742
2743 __kmp_save_internal_controls(thread);
2744
2745 set__max_active_levels(thread, max_active_levels)(((thread)->th.th_current_task->td_icvs.max_active_levels
) = (max_active_levels))
;
2746}
2747
2748/* Gets max_active_levels */
2749int __kmp_get_max_active_levels(int gtid) {
2750 kmp_info_t *thread;
2751
2752 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d\n"
, gtid); }
;
2753 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2753); }
;
2754
2755 thread = __kmp_threads[gtid];
2756 KMP_DEBUG_ASSERT(thread->th.th_current_task)if (!(thread->th.th_current_task)) { __kmp_debug_assert("thread->th.th_current_task"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2756); }
;
2757 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2758 "curtask_maxaclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2759 gtid, thread->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2760 thread->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
;
2761 return thread->th.th_current_task->td_icvs.max_active_levels;
2762}
2763
2764/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
2765void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
2766 kmp_info_t *thread;
2767 // kmp_team_t *team;
2768
2769 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); }
2770 gtid, (int)kind, chunk))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); }
;
2771 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2771); }
;
2772
2773 // Check if the kind parameter is valid, correct if needed.
2774 // Valid parameters should fit in one of two intervals - standard or extended:
2775 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
2776 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103
2777 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2778 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2779 // TODO: Hint needs attention in case we change the default schedule.
2780 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind)__kmp_msg_format(kmp_i18n_msg_ScheduleKindOutOfRange, kind),
2781 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk")__kmp_msg_format(kmp_i18n_hnt_DefaultScheduleKindUsed, "static, no chunk"
)
,
2782 __kmp_msg_null);
2783 kind = kmp_sched_default;
2784 chunk = 0; // ignore chunk value in case of bad kind
2785 }
2786
2787 thread = __kmp_threads[gtid];
2788
2789 __kmp_save_internal_controls(thread);
2790
2791 if (kind < kmp_sched_upper_std) {
2792 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK1) {
2793 // differ static chunked vs. unchunked: chunk should be invalid to
2794 // indicate unchunked schedule (which is the default)
2795 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
2796 } else {
2797 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2798 __kmp_sch_map[kind - kmp_sched_lower - 1];
2799 }
2800 } else {
2801 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2802 // kmp_sched_lower - 2 ];
2803 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2804 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2805 kmp_sched_lower - 2];
2806 }
2807 if (kind == kmp_sched_auto || chunk < 1) {
2808 // ignore parameter chunk for schedule auto
2809 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK1;
2810 } else {
2811 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2812 }
2813}
2814
2815/* Gets def_sched_var ICV values */
2816void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
2817 kmp_info_t *thread;
2818 enum sched_type th_type;
2819
2820 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_schedule: thread %d\n"
, gtid); }
;
2821 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2821); }
;
2822
2823 thread = __kmp_threads[gtid];
2824
2825 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2826
2827 switch (th_type) {
2828 case kmp_sch_static:
2829 case kmp_sch_static_greedy:
2830 case kmp_sch_static_balanced:
2831 *kind = kmp_sched_static;
2832 *chunk = 0; // chunk was not set, try to show this fact via zero value
2833 return;
2834 case kmp_sch_static_chunked:
2835 *kind = kmp_sched_static;
2836 break;
2837 case kmp_sch_dynamic_chunked:
2838 *kind = kmp_sched_dynamic;
2839 break;
2840 case kmp_sch_guided_chunked:
2841 case kmp_sch_guided_iterative_chunked:
2842 case kmp_sch_guided_analytical_chunked:
2843 *kind = kmp_sched_guided;
2844 break;
2845 case kmp_sch_auto:
2846 *kind = kmp_sched_auto;
2847 break;
2848 case kmp_sch_trapezoidal:
2849 *kind = kmp_sched_trapezoidal;
2850 break;
2851#if KMP_STATIC_STEAL_ENABLED1
2852 case kmp_sch_static_steal:
2853 *kind = kmp_sched_static_steal;
2854 break;
2855#endif
2856 default:
2857 KMP_FATAL(UnknownSchedulingType, th_type)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownSchedulingType
, th_type), __kmp_msg_null)
;
2858 }
2859
2860 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2861}
2862
2863int __kmp_get_ancestor_thread_num(int gtid, int level) {
2864
2865 int ii, dd;
2866 kmp_team_t *team;
2867 kmp_info_t *thr;
2868
2869 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_ancestor_thread_num: thread %d %d\n"
, gtid, level); }
;
2870 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2870); }
;
2871
2872 // validate level
2873 if (level == 0)
2874 return 0;
2875 if (level < 0)
2876 return -1;
2877 thr = __kmp_threads[gtid];
2878 team = thr->th.th_team;
2879 ii = team->t.t_level;
2880 if (level > ii)
2881 return -1;
2882
2883#if OMP_40_ENABLED(50 >= 40)
2884 if (thr->th.th_teams_microtask) {
2885 // AC: we are in teams region where multiple nested teams have same level
2886 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2887 if (level <=
2888 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2889 KMP_DEBUG_ASSERT(ii >= tlevel)if (!(ii >= tlevel)) { __kmp_debug_assert("ii >= tlevel"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2889); }
;
2890 // AC: As we need to pass by the teams league, we need to artificially
2891 // increase ii
2892 if (ii == tlevel) {
2893 ii += 2; // three teams have same level
2894 } else {
2895 ii++; // two teams have same level
2896 }
2897 }
2898 }
2899#endif
2900
2901 if (ii == level)
2902 return __kmp_tid_from_gtid(gtid);
2903
2904 dd = team->t.t_serialized;
2905 level++;
2906 while (ii > level) {
2907 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2908 }
2909 if ((team->t.t_serialized) && (!dd)) {
2910 team = team->t.t_parent;
2911 continue;
2912 }
2913 if (ii > level) {
2914 team = team->t.t_parent;
2915 dd = team->t.t_serialized;
2916 ii--;
2917 }
2918 }
2919
2920 return (dd > 1) ? (0) : (team->t.t_master_tid);
2921}
2922
2923int __kmp_get_team_size(int gtid, int level) {
2924
2925 int ii, dd;
2926 kmp_team_t *team;
2927 kmp_info_t *thr;
2928
2929 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_team_size: thread %d %d\n"
, gtid, level); }
;
2930 KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2930); }
;
2931
2932 // validate level
2933 if (level == 0)
2934 return 1;
2935 if (level < 0)
2936 return -1;
2937 thr = __kmp_threads[gtid];
2938 team = thr->th.th_team;
2939 ii = team->t.t_level;
2940 if (level > ii)
2941 return -1;
2942
2943#if OMP_40_ENABLED(50 >= 40)
2944 if (thr->th.th_teams_microtask) {
2945 // AC: we are in teams region where multiple nested teams have same level
2946 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2947 if (level <=
2948 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2949 KMP_DEBUG_ASSERT(ii >= tlevel)if (!(ii >= tlevel)) { __kmp_debug_assert("ii >= tlevel"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2949); }
;
2950 // AC: As we need to pass by the teams league, we need to artificially
2951 // increase ii
2952 if (ii == tlevel) {
2953 ii += 2; // three teams have same level
2954 } else {
2955 ii++; // two teams have same level
2956 }
2957 }
2958 }
2959#endif
2960
2961 while (ii > level) {
2962 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2963 }
2964 if (team->t.t_serialized && (!dd)) {
2965 team = team->t.t_parent;
2966 continue;
2967 }
2968 if (ii > level) {
2969 team = team->t.t_parent;
2970 ii--;
2971 }
2972 }
2973
2974 return team->t.t_nproc;
2975}
2976
2977kmp_r_sched_t __kmp_get_schedule_global() {
2978 // This routine created because pairs (__kmp_sched, __kmp_chunk) and
2979 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults
2980 // independently. So one can get the updated schedule here.
2981
2982 kmp_r_sched_t r_sched;
2983
2984 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
2985 // __kmp_guided. __kmp_sched should keep original value, so that user can set
2986 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in
2987 // different roots (even in OMP 2.5)
2988 if (__kmp_sched == kmp_sch_static) {
2989 // replace STATIC with more detailed schedule (balanced or greedy)
2990 r_sched.r_sched_type = __kmp_static;
2991 } else if (__kmp_sched == kmp_sch_guided_chunked) {
2992 // replace GUIDED with more detailed schedule (iterative or analytical)
2993 r_sched.r_sched_type = __kmp_guided;
2994 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
2995 r_sched.r_sched_type = __kmp_sched;
2996 }
2997
2998 if (__kmp_chunk < KMP_DEFAULT_CHUNK1) {
2999 // __kmp_chunk may be wrong here (if it was not ever set)
3000 r_sched.chunk = KMP_DEFAULT_CHUNK1;
3001 } else {
3002 r_sched.chunk = __kmp_chunk;
3003 }
3004
3005 return r_sched;
3006}
3007
3008/* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
3009 at least argc number of *t_argv entries for the requested team. */
3010static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) {
3011
3012 KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3012); }
;
3013 if (!realloc || argc > team->t.t_max_argc) {
3014
3015 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
3016 "current entries=%d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
3017 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
;
3018 /* if previously allocated heap space for args, free them */
3019 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3020 __kmp_free((void *)team->t.t_argv)___kmp_free(((void *)team->t.t_argv), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3020)
;
3021
3022 if (argc <= KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
) {
3023 /* use unused space in the cache line for arguments */
3024 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
;
3025 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3026 "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3027 team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
;
3028 team->t.t_argv = &team->t.t_inline_argv[0];
3029 if (__kmp_storage_map) {
3030 __kmp_print_storage_map_gtid(
3031 -1, &team->t.t_inline_argv[0],
3032 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
],
3033 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
), "team_%d.t_inline_argv",
3034 team->t.t_id);
3035 }
3036 } else {
3037 /* allocate space for arguments in the heap */
3038 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES100 >> 1))
3039 ? KMP_MIN_MALLOC_ARGV_ENTRIES100
3040 : 2 * argc;
3041 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3042 "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3043 team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
;
3044 team->t.t_argv =
3045 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc)___kmp_page_allocate((sizeof(void *) * team->t.t_max_argc)
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3045)
;
3046 if (__kmp_storage_map) {
3047 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3048 &team->t.t_argv[team->t.t_max_argc],
3049 sizeof(void *) * team->t.t_max_argc,
3050 "team_%d.t_argv", team->t.t_id);
3051 }
3052 }
3053 }
3054}
3055
3056static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) {
3057 int i;
3058 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3059 team->t.t_threads =
3060 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth)___kmp_allocate((sizeof(kmp_info_t *) * max_nth), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3060)
;
3061 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3062)
3062 sizeof(dispatch_shared_info_t) * num_disp_buff)___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3062)
;
3063 team->t.t_dispatch =
3064 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth)___kmp_allocate((sizeof(kmp_disp_t) * max_nth), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3064)
;
3065 team->t.t_implicit_task_taskdata =
3066 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth)___kmp_allocate((sizeof(kmp_taskdata_t) * max_nth), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3066)
;
3067 team->t.t_max_nproc = max_nth;
3068
3069 /* setup dispatch buffers */
3070 for (i = 0; i < num_disp_buff; ++i) {
3071 team->t.t_disp_buffer[i].buffer_index = i;
3072#if OMP_45_ENABLED(50 >= 45)
3073 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3074#endif
3075 }
3076}
3077
3078static void __kmp_free_team_arrays(kmp_team_t *team) {
3079 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */
3080 int i;
3081 for (i = 0; i < team->t.t_max_nproc; ++i) {
3082 if (team->t.t_dispatch[i].th_disp_buffer != NULL__null) {
3083 __kmp_free(team->t.t_dispatch[i].th_disp_buffer)___kmp_free((team->t.t_dispatch[i].th_disp_buffer), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3083)
;
3084 team->t.t_dispatch[i].th_disp_buffer = NULL__null;
3085 }
3086 }
3087#if KMP_USE_HIER_SCHED0
3088 __kmp_dispatch_free_hierarchies(team);
3089#endif
3090 __kmp_free(team->t.t_threads)___kmp_free((team->t.t_threads), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3090)
;
3091 __kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3091)
;
3092 __kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3092)
;
3093 __kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3093)
;
3094 team->t.t_threads = NULL__null;
3095 team->t.t_disp_buffer = NULL__null;
3096 team->t.t_dispatch = NULL__null;
3097 team->t.t_implicit_task_taskdata = 0;
3098}
3099
3100static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
3101 kmp_info_t **oldThreads = team->t.t_threads;
3102
3103 __kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3103)
;
3104 __kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3104)
;
3105 __kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3105)
;
3106 __kmp_allocate_team_arrays(team, max_nth);
3107
3108 KMP_MEMCPYmemcpy(team->t.t_threads, oldThreads,
3109 team->t.t_nproc * sizeof(kmp_info_t *));
3110
3111 __kmp_free(oldThreads)___kmp_free((oldThreads), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3111)
;
3112}
3113
3114static kmp_internal_control_t __kmp_get_global_icvs(void) {
3115
3116 kmp_r_sched_t r_sched =
3117 __kmp_get_schedule_global(); // get current state of scheduling globals
3118
3119#if OMP_40_ENABLED(50 >= 40)
3120 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0)if (!(__kmp_nested_proc_bind.used > 0)) { __kmp_debug_assert
("__kmp_nested_proc_bind.used > 0", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3120); }
;
3121#endif /* OMP_40_ENABLED */
3122
3123 kmp_internal_control_t g_icvs = {
3124 0, // int serial_nesting_level; //corresponds to value of th_team_serialized
3125 (kmp_int8)__kmp_dflt_nested, // int nested; //internal control
3126 // for nested parallelism (per thread)
3127 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
3128 // adjustment of threads (per thread)
3129 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
3130 // whether blocktime is explicitly set
3131 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
3132#if KMP_USE_MONITOR
3133 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
3134// intervals
3135#endif
3136 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for
3137 // next parallel region (per thread)
3138 // (use a max ub on value if __kmp_parallel_initialize not called yet)
3139 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control
3140 // for max_active_levels
3141 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule
3142// {sched,chunk} pair
3143#if OMP_40_ENABLED(50 >= 40)
3144 __kmp_nested_proc_bind.bind_types[0],
3145 __kmp_default_device,
3146#endif /* OMP_40_ENABLED */
3147 NULL__null // struct kmp_internal_control *next;
3148 };
3149
3150 return g_icvs;
3151}
3152
3153static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) {
3154
3155 kmp_internal_control_t gx_icvs;
3156 gx_icvs.serial_nesting_level =
3157 0; // probably =team->t.t_serial like in save_inter_controls
3158 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3159 gx_icvs.next = NULL__null;
3160
3161 return gx_icvs;
3162}
3163
3164static void __kmp_initialize_root(kmp_root_t *root) {
3165 int f;
3166 kmp_team_t *root_team;
3167 kmp_team_t *hot_team;
3168 int hot_team_max_nth;
3169 kmp_r_sched_t r_sched =
3170 __kmp_get_schedule_global(); // get current state of scheduling globals
3171 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3172 KMP_DEBUG_ASSERT(root)if (!(root)) { __kmp_debug_assert("root", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3172); }
;
3173 KMP_ASSERT(!root->r.r_begin)if (!(!root->r.r_begin)) { __kmp_debug_assert("!root->r.r_begin"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3173); }
;
3174
3175 /* setup the root state structure */
3176 __kmp_init_lock(&root->r.r_begin_lock);
3177 root->r.r_begin = FALSE0;
3178 root->r.r_active = FALSE0;
3179 root->r.r_in_parallel = 0;
3180 root->r.r_blocktime = __kmp_dflt_blocktime;
3181 root->r.r_nested = __kmp_dflt_nested;
3182 root->r.r_cg_nthreads = 1;
3183
3184 /* setup the root team for this task */
3185 /* allocate the root team structure */
3186 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before root_team\n"
); }
;
3187
3188 root_team =
3189 __kmp_allocate_team(root,
3190 1, // new_nproc
3191 1, // max_nproc
3192#if OMPT_SUPPORT1
3193 ompt_data_none, // root parallel id
3194#endif
3195#if OMP_40_ENABLED(50 >= 40)
3196 __kmp_nested_proc_bind.bind_types[0],
3197#endif
3198 &r_icvs,
3199 0 // argc
3200 USE_NESTED_HOT_ARG(NULL), __null // master thread is unknown
3201 );
3202#if USE_DEBUGGER0
3203 // Non-NULL value should be assigned to make the debugger display the root
3204 // team.
3205 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0))((root_team->t.t_pkfn)) = (((microtask_t)(~0)));
3206#endif
3207
3208 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after root_team = %p\n"
, root_team); }
;
3209
3210 root->r.r_root_team = root_team;
3211 root_team->t.t_control_stack_top = NULL__null;
3212
3213 /* initialize root team */
3214 root_team->t.t_threads[0] = NULL__null;
3215 root_team->t.t_nproc = 1;
3216 root_team->t.t_serialized = 1;
3217 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3218 root_team->t.t_sched.sched = r_sched.sched;
3219 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3220 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3221 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3222 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
;
3223
3224 /* setup the hot team for this task */
3225 /* allocate the hot team structure */
3226 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before hot_team\n"
); }
;
3227
3228 hot_team =
3229 __kmp_allocate_team(root,
3230 1, // new_nproc
3231 __kmp_dflt_team_nth_ub * 2, // max_nproc
3232#if OMPT_SUPPORT1
3233 ompt_data_none, // root parallel id
3234#endif
3235#if OMP_40_ENABLED(50 >= 40)
3236 __kmp_nested_proc_bind.bind_types[0],
3237#endif
3238 &r_icvs,
3239 0 // argc
3240 USE_NESTED_HOT_ARG(NULL), __null // master thread is unknown
3241 );
3242 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after hot_team = %p\n"
, hot_team); }
;
3243
3244 root->r.r_hot_team = hot_team;
3245 root_team->t.t_control_stack_top = NULL__null;
3246
3247 /* first-time initialization */
3248 hot_team->t.t_parent = root_team;
3249
3250 /* initialize hot team */
3251 hot_team_max_nth = hot_team->t.t_max_nproc;
3252 for (f = 0; f < hot_team_max_nth; ++f) {
3253 hot_team->t.t_threads[f] = NULL__null;
3254 }
3255 hot_team->t.t_nproc = 1;
3256 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3257 hot_team->t.t_sched.sched = r_sched.sched;
3258 hot_team->t.t_size_changed = 0;
3259}
3260
3261#ifdef KMP_DEBUG1
3262
3263typedef struct kmp_team_list_item {
3264 kmp_team_p const *entry;
3265 struct kmp_team_list_item *next;
3266} kmp_team_list_item_t;
3267typedef kmp_team_list_item_t *kmp_team_list_t;
3268
3269static void __kmp_print_structure_team_accum( // Add team to list of teams.
3270 kmp_team_list_t list, // List of teams.
3271 kmp_team_p const *team // Team to add.
3272 ) {
3273
3274 // List must terminate with item where both entry and next are NULL.
3275 // Team is added to the list only once.
3276 // List is sorted in ascending order by team id.
3277 // Team id is *not* a key.
3278
3279 kmp_team_list_t l;
3280
3281 KMP_DEBUG_ASSERT(list != NULL)if (!(list != __null)) { __kmp_debug_assert("list != __null",
"/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3281); }
;
3282 if (team == NULL__null) {
3283 return;
3284 }
3285
3286 __kmp_print_structure_team_accum(list, team->t.t_parent);
3287 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3288
3289 // Search list for the team.
3290 l = list;
3291 while (l->next != NULL__null && l->entry != team) {
3292 l = l->next;
3293 }
3294 if (l->next != NULL__null) {
3295 return; // Team has been added before, exit.
3296 }
3297
3298 // Team is not found. Search list again for insertion point.
3299 l = list;
3300 while (l->next != NULL__null && l->entry->t.t_id <= team->t.t_id) {
3301 l = l->next;
3302 }
3303
3304 // Insert team.
3305 {
3306 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(malloc(sizeof(kmp_team_list_item_t))
3307 sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
3308 *item = *l;
3309 l->entry = team;
3310 l->next = item;
3311 }
3312}
3313
3314static void __kmp_print_structure_team(char const *title, kmp_team_p const *team
3315
3316 ) {
3317 __kmp_printf("%s", title);
3318 if (team != NULL__null) {
3319 __kmp_printf("%2x %p\n", team->t.t_id, team);
3320 } else {
3321 __kmp_printf(" - (nil)\n");
3322 }
3323}
3324
3325static void __kmp_print_structure_thread(char const *title,
3326 kmp_info_p const *thread) {
3327 __kmp_printf("%s", title);
3328 if (thread != NULL__null) {
3329 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3330 } else {
3331 __kmp_printf(" - (nil)\n");
3332 }
3333}
3334
3335void __kmp_print_structure(void) {
3336
3337 kmp_team_list_t list;
3338
3339 // Initialize list of teams.
3340 list =
3341 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
3342 list->entry = NULL__null;
3343 list->next = NULL__null;
3344
3345 __kmp_printf("\n------------------------------\nGlobal Thread "
3346 "Table\n------------------------------\n");
3347 {
3348 int gtid;
3349 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3350 __kmp_printf("%2d", gtid);
3351 if (__kmp_threads != NULL__null) {
3352 __kmp_printf(" %p", __kmp_threads[gtid]);
3353 }
3354 if (__kmp_root != NULL__null) {
3355 __kmp_printf(" %p", __kmp_root[gtid]);
3356 }
3357 __kmp_printf("\n");
3358 }
3359 }
3360
3361 // Print out __kmp_threads array.
3362 __kmp_printf("\n------------------------------\nThreads\n--------------------"
3363 "----------\n");
3364 if (__kmp_threads != NULL__null) {
3365 int gtid;
3366 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3367 kmp_info_t const *thread = __kmp_threads[gtid];
3368 if (thread != NULL__null) {
3369 __kmp_printf("GTID %2d %p:\n", gtid, thread);
3370 __kmp_printf(" Our Root: %p\n", thread->th.th_root);
3371 __kmp_print_structure_team(" Our Team: ", thread->th.th_team);
3372 __kmp_print_structure_team(" Serial Team: ",
3373 thread->th.th_serial_team);
3374 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc);
3375 __kmp_print_structure_thread(" Master: ",
3376 thread->th.th_team_master);
3377 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized);
3378 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc);
3379#if OMP_40_ENABLED(50 >= 40)
3380 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3381#endif
3382 __kmp_print_structure_thread(" Next in pool: ",
3383 thread->th.th_next_pool);
3384 __kmp_printf("\n");
3385 __kmp_print_structure_team_accum(list, thread->th.th_team);
3386 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3387 }
3388 }
3389 } else {
3390 __kmp_printf("Threads array is not allocated.\n");
3391 }
3392
3393 // Print out __kmp_root array.
3394 __kmp_printf("\n------------------------------\nUbers\n----------------------"
3395 "--------\n");
3396 if (__kmp_root != NULL__null) {
3397 int gtid;
3398 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3399 kmp_root_t const *root = __kmp_root[gtid];
3400 if (root != NULL__null) {
3401 __kmp_printf("GTID %2d %p:\n", gtid, root);
3402 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team);
3403 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team);
3404 __kmp_print_structure_thread(" Uber Thread: ",
3405 root->r.r_uber_thread);
3406 __kmp_printf(" Active?: %2d\n", root->r.r_active);
3407 __kmp_printf(" Nested?: %2d\n", root->r.r_nested);
3408 __kmp_printf(" In Parallel: %2d\n",
3409 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)(&root->r.r_in_parallel)->load(std::memory_order_relaxed
)
);
3410 __kmp_printf("\n");
3411 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3412 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3413 }
3414 }
3415 } else {
3416 __kmp_printf("Ubers array is not allocated.\n");
3417 }
3418
3419 __kmp_printf("\n------------------------------\nTeams\n----------------------"
3420 "--------\n");
3421 while (list->next != NULL__null) {
3422 kmp_team_p const *team = list->entry;
3423 int i;
3424 __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
3425 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent);
3426 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid);
3427 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc);
3428 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized);
3429 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc);
3430 for (i = 0; i < team->t.t_nproc; ++i) {
3431 __kmp_printf(" Thread %2d: ", i);
3432 __kmp_print_structure_thread("", team->t.t_threads[i]);
3433 }
3434 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool);
3435 __kmp_printf("\n");
3436 list = list->next;
3437 }
3438
3439 // Print out __kmp_thread_pool and __kmp_team_pool.
3440 __kmp_printf("\n------------------------------\nPools\n----------------------"
3441 "--------\n");
3442 __kmp_print_structure_thread("Thread pool: ",
3443 CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool));
3444 __kmp_print_structure_team("Team pool: ",
3445 CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool));
3446 __kmp_printf("\n");
3447
3448 // Free team list.
3449 while (list != NULL__null) {
3450 kmp_team_list_item_t *item = list;
3451 list = list->next;
3452 KMP_INTERNAL_FREE(item)free(item);
3453 }
3454}
3455
3456#endif
3457
3458//---------------------------------------------------------------------------
3459// Stuff for per-thread fast random number generator
3460// Table of primes
3461static const unsigned __kmp_primes[] = {
3462 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3463 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3464 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3465 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3466 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3467 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3468 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3469 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3470 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3471 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3472 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3473
3474//---------------------------------------------------------------------------
3475// __kmp_get_random: Get a random number using a linear congruential method.
3476unsigned short __kmp_get_random(kmp_info_t *thread) {
3477 unsigned x = thread->th.th_x;
3478 unsigned short r = x >> 16;
3479
3480 thread->th.th_x = x * thread->th.th_a + 1;
3481
3482 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); }
3483 thread->th.th_info.ds.ds_tid, r))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); }
;
3484
3485 return r;
3486}
3487//--------------------------------------------------------
3488// __kmp_init_random: Initialize a random number generator
3489void __kmp_init_random(kmp_info_t *thread) {
3490 unsigned seed = thread->th.th_info.ds.ds_tid;
3491
3492 thread->th.th_a =
3493 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))];
3494 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3495 KA_TRACE(30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); }
3496 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); }
;
3497}
3498
3499#if KMP_OS_WINDOWS0
3500/* reclaim array entries for root threads that are already dead, returns number
3501 * reclaimed */
3502static int __kmp_reclaim_dead_roots(void) {
3503 int i, r = 0;
3504
3505 for (i = 0; i < __kmp_threads_capacity; ++i) {
3506 if (KMP_UBER_GTID(i) &&
3507 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])((void *)(__kmp_threads[i]))) &&
3508 !__kmp_root[i]
3509 ->r.r_active) { // AC: reclaim only roots died in non-active state
3510 r += __kmp_unregister_root_other_thread(i);
3511 }
3512 }
3513 return r;
3514}
3515#endif
3516
3517/* This function attempts to create free entries in __kmp_threads and
3518 __kmp_root, and returns the number of free entries generated.
3519
3520 For Windows* OS static library, the first mechanism used is to reclaim array
3521 entries for root threads that are already dead.
3522
3523 On all platforms, expansion is attempted on the arrays __kmp_threads_ and
3524 __kmp_root, with appropriate update to __kmp_threads_capacity. Array
3525 capacity is increased by doubling with clipping to __kmp_tp_capacity, if
3526 threadprivate cache array has been created. Synchronization with
3527 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.
3528
3529 After any dead root reclamation, if the clipping value allows array expansion
3530 to result in the generation of a total of nNeed free slots, the function does
3531 that expansion. If not, nothing is done beyond the possible initial root
3532 thread reclamation.
3533
3534 If any argument is negative, the behavior is undefined. */
3535static int __kmp_expand_threads(int nNeed) {
3536 int added = 0;
3537 int minimumRequiredCapacity;
3538 int newCapacity;
3539 kmp_info_t **newThreads;
3540 kmp_root_t **newRoot;
3541
3542// All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so
3543// resizing __kmp_threads does not need additional protection if foreign
3544// threads are present
3545
3546#if KMP_OS_WINDOWS0 && !KMP_DYNAMIC_LIB1
3547 /* only for Windows static library */
3548 /* reclaim array entries for root threads that are already dead */
3549 added = __kmp_reclaim_dead_roots();
3550
3551 if (nNeed) {
3552 nNeed -= added;
3553 if (nNeed < 0)
3554 nNeed = 0;
3555 }
3556#endif
3557 if (nNeed <= 0)
3558 return added;
3559
3560 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
3561 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
3562 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
3563 // > __kmp_max_nth in one of two ways:
3564 //
3565 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0]
3566 // may not be resused by another thread, so we may need to increase
3567 // __kmp_threads_capacity to __kmp_max_nth + 1.
3568 //
3569 // 2) New foreign root(s) are encountered. We always register new foreign
3570 // roots. This may cause a smaller # of threads to be allocated at
3571 // subsequent parallel regions, but the worker threads hang around (and
3572 // eventually go to sleep) and need slots in the __kmp_threads[] array.
3573 //
3574 // Anyway, that is the reason for moving the check to see if
3575 // __kmp_max_nth was exceeded into __kmp_reserve_threads()
3576 // instead of having it performed here. -BB
3577
3578 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity)if (!(__kmp_sys_max_nth >= __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_sys_max_nth >= __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3578); }
;
3579
3580 /* compute expansion headroom to check if we can expand */
3581 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3582 /* possible expansion too small -- give up */
3583 return added;
3584 }
3585 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3586
3587 newCapacity = __kmp_threads_capacity;
3588 do {
3589 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3590 : __kmp_sys_max_nth;
3591 } while (newCapacity < minimumRequiredCapacity);
3592 newThreads = (kmp_info_t **)__kmp_allocate(___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3593)
3593 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE)___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3593)
;
3594 newRoot =
3595 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity);
3596 KMP_MEMCPYmemcpy(newThreads, __kmp_threads,
3597 __kmp_threads_capacity * sizeof(kmp_info_t *));
3598 KMP_MEMCPYmemcpy(newRoot, __kmp_root,
3599 __kmp_threads_capacity * sizeof(kmp_root_t *));
3600
3601 kmp_info_t **temp_threads = __kmp_threads;
3602 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads;
3603 *(kmp_root_t * *volatile *)&__kmp_root = newRoot;
3604 __kmp_free(temp_threads)___kmp_free((temp_threads), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3604)
;
3605 added += newCapacity - __kmp_threads_capacity;
3606 *(volatile int *)&__kmp_threads_capacity = newCapacity;
3607
3608 if (newCapacity > __kmp_tp_capacity) {
3609 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3610 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3611 __kmp_threadprivate_resize_cache(newCapacity);
3612 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size
3613 *(volatile int *)&__kmp_tp_capacity = newCapacity;
3614 }
3615 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3616 }
3617
3618 return added;
3619}
3620
3621/* Register the current thread as a root thread and obtain our gtid. We must
3622 have the __kmp_initz_lock held at this point. Argument TRUE only if are the
3623 thread that calls from __kmp_do_serial_initialize() */
3624int __kmp_register_root(int initial_thread) {
3625 kmp_info_t *root_thread;
3626 kmp_root_t *root;
3627 int gtid;
3628 int capacity;
3629 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3630 KA_TRACE(20, ("__kmp_register_root: entered\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: entered\n"
); }
;
3631 KMP_MB();
3632
3633 /* 2007-03-02:
3634 If initial thread did not invoke OpenMP RTL yet, and this thread is not an
3635 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not
3636 work as expected -- it may return false (that means there is at least one
3637 empty slot in __kmp_threads array), but it is possible the only free slot
3638 is #0, which is reserved for initial thread and so cannot be used for this
3639 one. Following code workarounds this bug.
3640
3641 However, right solution seems to be not reserving slot #0 for initial
3642 thread because:
3643 (1) there is no magic in slot #0,
3644 (2) we cannot detect initial thread reliably (the first thread which does
3645 serial initialization may be not a real initial thread).
3646 */
3647 capacity = __kmp_threads_capacity;
3648 if (!initial_thread && TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
3649 --capacity;
3650 }
3651
3652 /* see if there are too many threads */
3653 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
3654 if (__kmp_tp_cached) {
3655 __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread),
3656 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
)
,
3657 KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
3658 } else {
3659 __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads),
3660 __kmp_msg_null);
3661 }
3662 }
3663
3664 /* find an available thread slot */
3665 /* Don't reassign the zero slot since we need that to only be used by initial
3666 thread */
3667 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid])((void *)(__kmp_threads[gtid])) != NULL__null;
3668 gtid++)
3669 ;
3670 KA_TRACE(1,if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); }
3671 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); }
;
3672 KMP_ASSERT(gtid < __kmp_threads_capacity)if (!(gtid < __kmp_threads_capacity)) { __kmp_debug_assert
("gtid < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3672); }
;
3673
3674 /* update global accounting */
3675 __kmp_all_nth++;
3676 TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);
3677
3678 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
3679 // numbers of procs, and method #2 (keyed API call) for higher numbers.
3680 if (__kmp_adjust_gtid_mode) {
3681 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3682 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
3683 TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
3684 }
3685 } else {
3686 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
3687 TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
3688 }
3689 }
3690 }
3691
3692#ifdef KMP_ADJUST_BLOCKTIME1
3693 /* Adjust blocktime to zero if necessary */
3694 /* Middle initialization might not have occurred yet */
3695 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3696 if (__kmp_nth > __kmp_avail_proc) {
3697 __kmp_zero_bt = TRUE(!0);
3698 }
3699 }
3700#endif /* KMP_ADJUST_BLOCKTIME */
3701
3702 /* setup this new hierarchy */
3703 if (!(root = __kmp_root[gtid])) {
3704 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t))___kmp_allocate((sizeof(kmp_root_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3704)
;
3705 KMP_DEBUG_ASSERT(!root->r.r_root_team)if (!(!root->r.r_root_team)) { __kmp_debug_assert("!root->r.r_root_team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3705); }
;
3706 }
3707
3708#if KMP_STATS_ENABLED0
3709 // Initialize stats as soon as possible (right after gtid assignment).
3710 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3711 __kmp_stats_thread_ptr->startLife();
3712 KMP_SET_THREAD_STATE(SERIAL_REGION)((void)0);
3713 KMP_INIT_PARTITIONED_TIMERS(OMP_serial)((void)0);
3714#endif
3715 __kmp_initialize_root(root);
3716
3717 /* setup new root thread structure */
3718 if (root->r.r_uber_thread) {
3719 root_thread = root->r.r_uber_thread;
3720 } else {
3721 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3721)
;
3722 if (__kmp_storage_map) {
3723 __kmp_print_thread_storage_map(root_thread, gtid);
3724 }
3725 root_thread->th.th_info.ds.ds_gtid = gtid;
3726#if OMPT_SUPPORT1
3727 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
3728#endif
3729 root_thread->th.th_root = root;
3730 if (__kmp_env_consistency_check) {
3731 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3732 }
3733#if USE_FAST_MEMORY3
3734 __kmp_initialize_fast_memory(root_thread);
3735#endif /* USE_FAST_MEMORY */
3736
3737#if KMP_USE_BGET1
3738 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL)if (!(root_thread->th.th_local.bget_data == __null)) { __kmp_debug_assert
("root_thread->th.th_local.bget_data == __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3738); }
;
3739 __kmp_initialize_bget(root_thread);
3740#endif
3741 __kmp_init_random(root_thread); // Initialize random number generator
3742 }
3743
3744 /* setup the serial team held in reserve by the root thread */
3745 if (!root_thread->th.th_serial_team) {
3746 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3747 KF_TRACE(10, ("__kmp_register_root: before serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: before serial_team\n"
); }
;
3748 root_thread->th.th_serial_team =
3749 __kmp_allocate_team(root, 1, 1,
3750#if OMPT_SUPPORT1
3751 ompt_data_none, // root parallel id
3752#endif
3753#if OMP_40_ENABLED(50 >= 40)
3754 proc_bind_default,
3755#endif
3756 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL), __null);
3757 }
3758 KMP_ASSERT(root_thread->th.th_serial_team)if (!(root_thread->th.th_serial_team)) { __kmp_debug_assert
("root_thread->th.th_serial_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3758); }
;
3759 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); }
3760 root_thread->th.th_serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); }
;
3761
3762 /* drop root_thread into place */
3763 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread)((__kmp_threads[gtid])) = ((root_thread));
3764
3765 root->r.r_root_team->t.t_threads[0] = root_thread;
3766 root->r.r_hot_team->t.t_threads[0] = root_thread;
3767 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3768 // AC: the team created in reserve, not for execution (it is unused for now).
3769 root_thread->th.th_serial_team->t.t_serialized = 0;
3770 root->r.r_uber_thread = root_thread;
3771
3772 /* initialize the thread, get it ready to go */
3773 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3774 TCW_4(__kmp_init_gtid, TRUE)(__kmp_init_gtid) = ((!0));
3775
3776 /* prepare the master thread for get_gtid() */
3777 __kmp_gtid_set_specific(gtid);
3778
3779#if USE_ITT_BUILD1
3780 __kmp_itt_thread_name(gtid);
3781#endif /* USE_ITT_BUILD */
3782
3783#ifdef KMP_TDATA_GTID1
3784 __kmp_gtid = gtid;
3785#endif
3786 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3787 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid)if (!(__kmp_gtid_get_specific() == gtid)) { __kmp_debug_assert
("__kmp_gtid_get_specific() == gtid", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3787); }
;
3788
3789 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3790 "plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3791 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3792 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3793 KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
;
3794 { // Initialize barrier data.
3795 int b;
3796 for (b = 0; b < bs_last_barrier; ++b) {
3797 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE0;
3798#if USE_DEBUGGER0
3799 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3800#endif
3801 }
3802 }
3803 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==if (!(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].
b_arrived == 0)) { __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3804); }
3804 KMP_INIT_BARRIER_STATE)if (!(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].
b_arrived == 0)) { __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3804); }
;
3805
3806#if KMP_AFFINITY_SUPPORTED1
3807#if OMP_40_ENABLED(50 >= 40)
3808 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
3809 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
3810 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
3811 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
3812#endif
3813 if (TCR_4(__kmp_init_middle)(__kmp_init_middle)) {
3814 __kmp_affinity_set_init_mask(gtid, TRUE(!0));
3815 }
3816#endif /* KMP_AFFINITY_SUPPORTED */
3817#if OMP_50_ENABLED(50 >= 50)
3818 root_thread->th.th_def_allocator = __kmp_def_allocator;
3819#endif
3820
3821 __kmp_root_counter++;
3822
3823#if OMPT_SUPPORT1
3824 if (!initial_thread && ompt_enabled.enabled) {
3825
3826 kmp_info_t *root_thread = ompt_get_thread();
3827
3828 ompt_set_thread_state(root_thread, omp_state_overhead);
3829
3830 if (ompt_enabled.ompt_callback_thread_begin) {
3831 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)ompt_callback_thread_begin_callback(
3832 ompt_thread_initial, __ompt_get_thread_data_internal());
3833 }
3834 ompt_data_t *task_data;
3835 __ompt_get_task_info_internal(0, NULL__null, &task_data, NULL__null, NULL__null, NULL__null);
3836 if (ompt_enabled.ompt_callback_task_create) {
3837 ompt_callbacks.ompt_callback(ompt_callback_task_create)ompt_callback_task_create_callback(
3838 NULL__null, NULL__null, task_data, ompt_task_initial, 0, NULL__null);
3839 // initial task has nothing to return to
3840 }
3841
3842 ompt_set_thread_state(root_thread, omp_state_work_serial);
3843 }
3844#endif
3845
3846 KMP_MB();
3847 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3848
3849 return gtid;
3850}
3851
3852#if KMP_NESTED_HOT_TEAMS1
3853static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
3854 const int max_level) {
3855 int i, n, nth;
3856 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3857 if (!hot_teams || !hot_teams[level].hot_team) {
3858 return 0;
3859 }
3860 KMP_DEBUG_ASSERT(level < max_level)if (!(level < max_level)) { __kmp_debug_assert("level < max_level"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3860); }
;
3861 kmp_team_t *team = hot_teams[level].hot_team;
3862 nth = hot_teams[level].hot_team_nth;
3863 n = nth - 1; // master is not freed
3864 if (level < max_level - 1) {
3865 for (i = 0; i < nth; ++i) {
3866 kmp_info_t *th = team->t.t_threads[i];
3867 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3868 if (i > 0 && th->th.th_hot_teams) {
3869 __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3869)
;
3870 th->th.th_hot_teams = NULL__null;
3871 }
3872 }
3873 }
3874 __kmp_free_team(root, team, NULL__null);
3875 return n;
3876}
3877#endif
3878
3879// Resets a root thread and clear its root and hot teams.
3880// Returns the number of __kmp_threads entries directly and indirectly freed.
3881static int __kmp_reset_root(int gtid, kmp_root_t *root) {
3882 kmp_team_t *root_team = root->r.r_root_team;
3883 kmp_team_t *hot_team = root->r.r_hot_team;
3884 int n = hot_team->t.t_nproc;
3885 int i;
3886
3887 KMP_DEBUG_ASSERT(!root->r.r_active)if (!(!root->r.r_active)) { __kmp_debug_assert("!root->r.r_active"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3887); }
;
3888
3889 root->r.r_root_team = NULL__null;
3890 root->r.r_hot_team = NULL__null;
3891 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team
3892 // before call to __kmp_free_team().
3893 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL), __null);
3894#if KMP_NESTED_HOT_TEAMS1
3895 if (__kmp_hot_teams_max_level >
3896 0) { // need to free nested hot teams and their threads if any
3897 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3898 kmp_info_t *th = hot_team->t.t_threads[i];
3899 if (__kmp_hot_teams_max_level > 1) {
3900 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3901 }
3902 if (th->th.th_hot_teams) {
3903 __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3903)
;
3904 th->th.th_hot_teams = NULL__null;
3905 }
3906 }
3907 }
3908#endif
3909 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL), __null);
3910
3911 // Before we can reap the thread, we need to make certain that all other
3912 // threads in the teams that had this root as ancestor have stopped trying to
3913 // steal tasks.
3914 if (__kmp_tasking_mode != tskm_immediate_exec) {
3915 __kmp_wait_to_unref_task_teams();
3916 }
3917
3918#if KMP_OS_WINDOWS0
3919 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
3920 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3921 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPECif (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3922 "\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3923 (LPVOID) & (root->r.r_uber_thread->th),if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3924 root->r.r_uber_thread->th.th_info.ds.ds_thread))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
;
3925 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3926#endif /* KMP_OS_WINDOWS */
3927
3928#if OMPT_SUPPORT1
3929 if (ompt_enabled.ompt_callback_thread_end) {
3930 ompt_callbacks.ompt_callback(ompt_callback_thread_end)ompt_callback_thread_end_callback(
3931 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
3932 }
3933#endif
3934
3935 TCW_4(__kmp_nth,(__kmp_nth) = (__kmp_nth - 1)
3936 __kmp_nth - 1)(__kmp_nth) = (__kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
3937 root->r.r_cg_nthreads--;
3938
3939 __kmp_reap_thread(root->r.r_uber_thread, 1);
3940
3941 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead
3942 // of freeing.
3943 root->r.r_uber_thread = NULL__null;
3944 /* mark root as no longer in use */
3945 root->r.r_begin = FALSE0;
3946
3947 return n;
3948}
3949
3950void __kmp_unregister_root_current_thread(int gtid) {
3951 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: enter T#%d\n"
, gtid); }
;
3952 /* this lock should be ok, since unregister_root_current_thread is never
3953 called during an abort, only during a normal close. furthermore, if you
3954 have the forkjoin lock, you should never try to get the initz lock */
3955 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3956 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
3957 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
3958 "exiting T#%d\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
3959 gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
;
3960 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3961 return;
3962 }
3963 kmp_root_t *root = __kmp_root[gtid];
3964
3965 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])if (!(__kmp_threads && __kmp_threads[gtid])) { __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3965); }
;
3966 KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3966); }
;
3967 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)if (!(root == __kmp_threads[gtid]->th.th_root)) { __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3967); }
;
3968 KMP_ASSERT(root->r.r_active == FALSE)if (!(root->r.r_active == 0)) { __kmp_debug_assert("root->r.r_active == FALSE"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3968); }
;
3969
3970 KMP_MB();
3971
3972#if OMP_45_ENABLED(50 >= 45)
3973 kmp_info_t *thread = __kmp_threads[gtid];
3974 kmp_team_t *team = thread->th.th_team;
3975 kmp_task_team_t *task_team = thread->th.th_task_team;
3976
3977 // we need to wait for the proxy tasks before finishing the thread
3978 if (task_team != NULL__null && task_team->tt.tt_found_proxy_tasks) {
3979#if OMPT_SUPPORT1
3980 // the runtime is shutting down so we won't report any events
3981 thread->th.ompt_thread_info.state = omp_state_undefined;
3982#endif
3983 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL), __null);
3984 }
3985#endif
3986
3987 __kmp_reset_root(gtid, root);
3988
3989 /* free up this thread slot */
3990 __kmp_gtid_set_specific(KMP_GTID_DNE(-2));
3991#ifdef KMP_TDATA_GTID1
3992 __kmp_gtid = KMP_GTID_DNE(-2);
3993#endif
3994
3995 KMP_MB();
3996 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); }
3997 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); }
;
3998
3999 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4000}
4001
4002#if KMP_OS_WINDOWS0
4003/* __kmp_forkjoin_lock must be already held
4004 Unregisters a root thread that is not the current thread. Returns the number
4005 of __kmp_threads entries freed as a result. */
4006static int __kmp_unregister_root_other_thread(int gtid) {
4007 kmp_root_t *root = __kmp_root[gtid];
4008 int r;
4009
4010 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: enter T#%d\n"
, gtid); }
;
4011 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])if (!(__kmp_threads && __kmp_threads[gtid])) { __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4011); }
;
4012 KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4012); }
;
4013 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)if (!(root == __kmp_threads[gtid]->th.th_root)) { __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4013); }
;
4014 KMP_ASSERT(root->r.r_active == FALSE)if (!(root->r.r_active == 0)) { __kmp_debug_assert("root->r.r_active == FALSE"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4014); }
;
4015
4016 r = __kmp_reset_root(gtid, root);
4017 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); }
4018 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); }
;
4019 return r;
4020}
4021#endif
4022
4023#if KMP_DEBUG1
4024void __kmp_task_info() {
4025
4026 kmp_int32 gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
4027 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4028 kmp_info_t *this_thr = __kmp_threads[gtid];
4029 kmp_team_t *steam = this_thr->th.th_serial_team;
4030 kmp_team_t *team = this_thr->th.th_team;
4031
4032 __kmp_printf(
4033 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p "
4034 "ptask=%p\n",
4035 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4036 team->t.t_implicit_task_taskdata[tid].td_parent);
4037}
4038#endif // KMP_DEBUG
4039
4040/* TODO optimize with one big memclr, take out what isn't needed, split
4041 responsibility to workers as much as possible, and delay initialization of
4042 features as much as possible */
4043static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4044 int tid, int gtid) {
4045 /* this_thr->th.th_info.ds.ds_gtid is setup in
4046 kmp_allocate_thread/create_worker.
4047 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
4048 kmp_info_t *master = team->t.t_threads[0];
4049 KMP_DEBUG_ASSERT(this_thr != NULL)if (!(this_thr != __null)) { __kmp_debug_assert("this_thr != __null"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4049); }
;
4050 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team)if (!(this_thr->th.th_serial_team)) { __kmp_debug_assert("this_thr->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4050); }
;
4051 KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4051); }
;
4052 KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4052); }
;
4053 KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4053); }
;
4054 KMP_DEBUG_ASSERT(master)if (!(master)) { __kmp_debug_assert("master", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4054); }
;
4055 KMP_DEBUG_ASSERT(master->th.th_root)if (!(master->th.th_root)) { __kmp_debug_assert("master->th.th_root"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4055); }
;
4056
4057 KMP_MB();
4058
4059 TCW_SYNC_PTR(this_thr->th.th_team, team)((this_thr->th.th_team)) = ((team));
4060
4061 this_thr->th.th_info.ds.ds_tid = tid;
4062 this_thr->th.th_set_nproc = 0;
4063 if (__kmp_tasking_mode != tskm_immediate_exec)
4064 // When tasking is possible, threads are not safe to reap until they are
4065 // done tasking; this will be set when tasking code is exited in wait
4066 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP0;
4067 else // no tasking --> always safe to reap
4068 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP1;
4069#if OMP_40_ENABLED(50 >= 40)
4070 this_thr->th.th_set_proc_bind = proc_bind_default;
4071#if KMP_AFFINITY_SUPPORTED1
4072 this_thr->th.th_new_place = this_thr->th.th_current_place;
4073#endif
4074#endif
4075 this_thr->th.th_root = master->th.th_root;
4076
4077 /* setup the thread's cache of the team structure */
4078 this_thr->th.th_team_nproc = team->t.t_nproc;
4079 this_thr->th.th_team_master = master;
4080 this_thr->th.th_team_serialized = team->t.t_serialized;
4081 TCW_PTR(this_thr->th.th_sleep_loc, NULL)((this_thr->th.th_sleep_loc)) = ((__null));
4082
4083 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata)if (!(team->t.t_implicit_task_taskdata)) { __kmp_debug_assert
("team->t.t_implicit_task_taskdata", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4083); }
;
4084
4085 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
4086 tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
;
4087
4088 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4089 team, tid, TRUE(!0));
4090
4091 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
4092 tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
;
4093 // TODO: Initialize ICVs from parent; GEH - isn't that already done in
4094 // __kmp_initialize_team()?
4095
4096 /* TODO no worksharing in speculative threads */
4097 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4098
4099 this_thr->th.th_local.this_construct = 0;
4100
4101 if (!this_thr->th.th_pri_common) {
4102 this_thr->th.th_pri_common =
4103 (struct common_table *)__kmp_allocate(sizeof(struct common_table))___kmp_allocate((sizeof(struct common_table)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4103)
;
4104 if (__kmp_storage_map) {
4105 __kmp_print_storage_map_gtid(
4106 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4107 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
4108 }
4109 this_thr->th.th_pri_head = NULL__null;
4110 }
4111
4112 /* Initialize dynamic dispatch */
4113 {
4114 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4115 // Use team max_nproc since this will never change for the team.
4116 size_t disp_size =
4117 sizeof(dispatch_private_info_t) *
4118 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4119 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); }
4120 team->t.t_max_nproc))if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); }
;
4121 KMP_ASSERT(dispatch)if (!(dispatch)) { __kmp_debug_assert("dispatch", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4121); }
;
4122 KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4122); }
;
4123 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid])if (!(dispatch == &team->t.t_dispatch[tid])) { __kmp_debug_assert
("dispatch == &team->t.t_dispatch[tid]", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4123); }
;
4124
4125 dispatch->th_disp_index = 0;
4126#if OMP_45_ENABLED(50 >= 45)
4127 dispatch->th_doacross_buf_idx = 0;
4128#endif
4129 if (!dispatch->th_disp_buffer) {
4130 dispatch->th_disp_buffer =
4131 (dispatch_private_info_t *)__kmp_allocate(disp_size)___kmp_allocate((disp_size), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4131)
;
4132
4133 if (__kmp_storage_map) {
4134 __kmp_print_storage_map_gtid(
4135 gtid, &dispatch->th_disp_buffer[0],
4136 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4137 ? 1
4138 : __kmp_dispatch_num_buffers],
4139 disp_size, "th_%d.th_dispatch.th_disp_buffer "
4140 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4141 gtid, team->t.t_id, gtid);
4142 }
4143 } else {
4144 memset(&dispatch->th_disp_buffer[0], '\0', disp_size);
4145 }
4146
4147 dispatch->th_dispatch_pr_current = 0;
4148 dispatch->th_dispatch_sh_current = 0;
4149
4150 dispatch->th_deo_fcn = 0; /* ORDERED */
4151 dispatch->th_dxo_fcn = 0; /* END ORDERED */
4152 }
4153
4154 this_thr->th.th_next_pool = NULL__null;
4155
4156 if (!this_thr->th.th_task_state_memo_stack) {
4157 size_t i;
4158 this_thr->th.th_task_state_memo_stack =
4159 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8))___kmp_allocate((4 * sizeof(kmp_uint8)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4159)
;
4160 this_thr->th.th_task_state_top = 0;
4161 this_thr->th.th_task_state_stack_sz = 4;
4162 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4163 ++i) // zero init the stack
4164 this_thr->th.th_task_state_memo_stack[i] = 0;
4165 }
4166
4167 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here)if (!(!this_thr->th.th_spin_here)) { __kmp_debug_assert("!this_thr->th.th_spin_here"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4167); }
;
4168 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0)if (!(this_thr->th.th_next_waiting == 0)) { __kmp_debug_assert
("this_thr->th.th_next_waiting == 0", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4168); }
;
4169
4170 KMP_MB();
4171}
4172
4173/* allocate a new thread for the requesting team. this is only called from
4174 within a forkjoin critical section. we will first try to get an available
4175 thread from the thread pool. if none is available, we will fork a new one
4176 assuming we are able to create a new one. this should be assured, as the
4177 caller should check on this first. */
4178kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4179 int new_tid) {
4180 kmp_team_t *serial_team;
4181 kmp_info_t *new_thr;
4182 int new_gtid;
4183
4184 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d\n"
, __kmp_get_global_thread_id()); }
;
4185 KMP_DEBUG_ASSERT(root && team)if (!(root && team)) { __kmp_debug_assert("root && team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4185); }
;
4186#if !KMP_NESTED_HOT_TEAMS1
4187 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()))if (!((__kmp_tid_from_gtid((__kmp_get_global_thread_id())) ==
0))) { __kmp_debug_assert("(__kmp_tid_from_gtid((__kmp_get_global_thread_id())) == 0)"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4187); }
;
4188#endif
4189 KMP_MB();
4190
4191 /* first, try to get one from the thread pool */
4192 if (__kmp_thread_pool) {
4193
4194 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool);
4195 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;
4196 if (new_thr == __kmp_thread_pool_insert_pt) {
4197 __kmp_thread_pool_insert_pt = NULL__null;
4198 }
4199 TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
4200 // Don't touch th_active_in_pool or th_active.
4201 // The worker thread adjusts those flags as it sleeps/awakens.
4202 __kmp_thread_pool_nth--;
4203
4204 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); }
4205 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); }
;
4206 KMP_ASSERT(!new_thr->th.th_team)if (!(!new_thr->th.th_team)) { __kmp_debug_assert("!new_thr->th.th_team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4206); }
;
4207 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity)if (!(__kmp_nth < __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_nth < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4207); }
;
4208 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0)if (!(__kmp_thread_pool_nth >= 0)) { __kmp_debug_assert("__kmp_thread_pool_nth >= 0"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4208); }
;
4209
4210 /* setup the thread structure */
4211 __kmp_initialize_info(new_thr, team, new_tid,
4212 new_thr->th.th_info.ds.ds_gtid);
4213 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team)if (!(new_thr->th.th_serial_team)) { __kmp_debug_assert("new_thr->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4213); }
;
4214
4215 TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);
4216 root->r.r_cg_nthreads++;
4217
4218 new_thr->th.th_task_state = 0;
4219 new_thr->th.th_task_state_top = 0;
4220 new_thr->th.th_task_state_stack_sz = 4;
4221
4222#ifdef KMP_ADJUST_BLOCKTIME1
4223 /* Adjust blocktime back to zero if necessary */
4224 /* Middle initialization might not have occurred yet */
4225 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4226 if (__kmp_nth > __kmp_avail_proc) {
4227 __kmp_zero_bt = TRUE(!0);
4228 }
4229 }
4230#endif /* KMP_ADJUST_BLOCKTIME */
4231
4232#if KMP_DEBUG1
4233 // If thread entered pool via __kmp_free_thread, wait_flag should !=
4234 // KMP_BARRIER_PARENT_FLAG.
4235 int b;
4236 kmp_balign_t *balign = new_thr->th.th_bar;
4237 for (b = 0; b < bs_last_barrier; ++b)
4238 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4238); }
;
4239#endif
4240
4241 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); }
4242 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); }
;
4243
4244 KMP_MB();
4245 return new_thr;
4246 }
4247
4248 /* no, well fork a new one */
4249 KMP_ASSERT(__kmp_nth == __kmp_all_nth)if (!(__kmp_nth == __kmp_all_nth)) { __kmp_debug_assert("__kmp_nth == __kmp_all_nth"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4249); }
;
4250 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity)if (!(__kmp_all_nth < __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_all_nth < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4250); }
;
4251
4252#if KMP_USE_MONITOR
4253 // If this is the first worker thread the RTL is creating, then also
4254 // launch the monitor thread. We try to do this as early as possible.
4255 if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
4256 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4257 if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
4258 KF_TRACE(10, ("before __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("before __kmp_create_monitor\n"
); }
;
4259 TCW_4(__kmp_init_monitor, 1)(__kmp_init_monitor) = (1);
4260 __kmp_create_monitor(&__kmp_monitor);
4261 KF_TRACE(10, ("after __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after __kmp_create_monitor\n"
); }
;
4262#if KMP_OS_WINDOWS0
4263 // AC: wait until monitor has started. This is a fix for CQ232808.
4264 // The reason is that if the library is loaded/unloaded in a loop with
4265 // small (parallel) work in between, then there is high probability that
4266 // monitor thread started after the library shutdown. At shutdown it is
4267 // too late to cope with the problem, because when the master is in
4268 // DllMain (process detach) the monitor has no chances to start (it is
4269 // blocked), and master has no means to inform the monitor that the
4270 // library has gone, because all the memory which the monitor can access
4271 // is going to be released/reset.
4272 while (TCR_4(__kmp_init_monitor)(__kmp_init_monitor) < 2) {
4273 KMP_YIELD(TRUE){ __kmp_x86_pause(); __kmp_yield(((!0))); };
4274 }
4275 KF_TRACE(10, ("after monitor thread has started\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after monitor thread has started\n"
); }
;
4276#endif
4277 }
4278 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4279 }
4280#endif
4281
4282 KMP_MB();
4283 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid])((void *)(__kmp_threads[new_gtid])) != NULL__null; ++new_gtid) {
4284 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity)if (!(new_gtid < __kmp_threads_capacity)) { __kmp_debug_assert
("new_gtid < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4284); }
;
4285 }
4286
4287 /* allocate space for it. */
4288 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4288)
;
4289
4290 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr)((__kmp_threads[new_gtid])) = ((new_thr));
4291
4292 if (__kmp_storage_map) {
4293 __kmp_print_thread_storage_map(new_thr, new_gtid);
4294 }
4295
4296 // add the reserve serialized team, initialized from the team's master thread
4297 {
4298 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4299 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before th_serial/serial_team\n"
); }
;
4300 new_thr->th.th_serial_team = serial_team =
4301 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4302#if OMPT_SUPPORT1
4303 ompt_data_none, // root parallel id
4304#endif
4305#if OMP_40_ENABLED(50 >= 40)
4306 proc_bind_default,
4307#endif
4308 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL), __null);
4309 }
4310 KMP_ASSERT(serial_team)if (!(serial_team)) { __kmp_debug_assert("serial_team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4310); }
;
4311 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for
4312 // execution (it is unused for now).
4313 serial_team->t.t_threads[0] = new_thr;
4314 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
4315 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
4316 new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
;
4317
4318 /* setup the thread structures */
4319 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4320
4321#if USE_FAST_MEMORY3
4322 __kmp_initialize_fast_memory(new_thr);
4323#endif /* USE_FAST_MEMORY */
4324
4325#if KMP_USE_BGET1
4326 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL)if (!(new_thr->th.th_local.bget_data == __null)) { __kmp_debug_assert
("new_thr->th.th_local.bget_data == __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4326); }
;
4327 __kmp_initialize_bget(new_thr);
4328#endif
4329
4330 __kmp_init_random(new_thr); // Initialize random number generator
4331
4332 /* Initialize these only once when thread is grabbed for a team allocation */
4333 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
4334 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
4335 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
;
4336
4337 int b;
4338 kmp_balign_t *balign = new_thr->th.th_bar;
4339 for (b = 0; b < bs_last_barrier; ++b) {
4340 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE0;
4341 balign[b].bb.team = NULL__null;
4342 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING0;
4343 balign[b].bb.use_oncore_barrier = 0;
4344 }
4345
4346 new_thr->th.th_spin_here = FALSE0;
4347 new_thr->th.th_next_waiting = 0;
4348#if KMP_OS_UNIX1
4349 new_thr->th.th_blocking = false;
4350#endif
4351
4352#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
4353 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
4354 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
4355 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
4356 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
4357#endif
4358#if OMP_50_ENABLED(50 >= 50)
4359 new_thr->th.th_def_allocator = __kmp_def_allocator;
4360#endif
4361
4362 TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
4363 new_thr->th.th_active_in_pool = FALSE0;
4364 TCW_4(new_thr->th.th_active, TRUE)(new_thr->th.th_active) = ((!0));
4365
4366 /* adjust the global counters */
4367 __kmp_all_nth++;
4368 __kmp_nth++;
4369
4370 root->r.r_cg_nthreads++;
4371
4372 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
4373 // numbers of procs, and method #2 (keyed API call) for higher numbers.
4374 if (__kmp_adjust_gtid_mode) {
4375 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4376 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
4377 TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
4378 }
4379 } else {
4380 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
4381 TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
4382 }
4383 }
4384 }
4385
4386#ifdef KMP_ADJUST_BLOCKTIME1
4387 /* Adjust blocktime back to zero if necessary */
4388 /* Middle initialization might not have occurred yet */
4389 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4390 if (__kmp_nth > __kmp_avail_proc) {
4391 __kmp_zero_bt = TRUE(!0);
4392 }
4393 }
4394#endif /* KMP_ADJUST_BLOCKTIME */
4395
4396 /* actually fork it and create the new worker thread */
4397 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); }
4398 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); }
;
4399 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4400 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); }
4401 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); }
;
4402
4403 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); }
4404 new_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); }
;
4405 KMP_MB();
4406 return new_thr;
4407}
4408
4409/* Reinitialize team for reuse.
4410 The hot team code calls this case at every fork barrier, so EPCC barrier
4411 test are extremely sensitive to changes in it, esp. writes to the team
4412 struct, which cause a cache invalidation in all threads.
4413 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4414static void __kmp_reinitialize_team(kmp_team_t *team,
4415 kmp_internal_control_t *new_icvs,
4416 ident_t *loc) {
4417 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
4418 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
;
4419 KMP_DEBUG_ASSERT(team && new_icvs)if (!(team && new_icvs)) { __kmp_debug_assert("team && new_icvs"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4419); }
;
4420 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc)if (!((!(__kmp_init_parallel)) || new_icvs->nproc)) { __kmp_debug_assert
("(!(__kmp_init_parallel)) || new_icvs->nproc", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4420); }
;
4421 KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
)
;
4422
4423 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID())if ((team->t.t_id) != ((~0))) (team->t.t_id) = ((~0));
4424 // Copy ICVs to the master thread's implicit taskdata
4425 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE0);
4426 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4427
4428 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
4429 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
;
4430}
4431
4432/* Initialize the team data structure.
4433 This assumes the t_threads and t_max_nproc are already set.
4434 Also, we don't touch the arguments */
4435static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
4436 kmp_internal_control_t *new_icvs,
4437 ident_t *loc) {
4438 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: enter: team=%p\n"
, team); }
;
4439
4440 /* verify */
4441 KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4441); }
;
4442 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc)if (!(new_nproc <= team->t.t_max_nproc)) { __kmp_debug_assert
("new_nproc <= team->t.t_max_nproc", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4442); }
;
4443 KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4443); }
;
4444 KMP_MB();
4445
4446 team->t.t_master_tid = 0; /* not needed */
4447 /* team->t.t_master_bar; not needed */
4448 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4449 team->t.t_nproc = new_nproc;
4450
4451 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */
4452 team->t.t_next_pool = NULL__null;
4453 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess
4454 * up hot team */
4455
4456 TCW_SYNC_PTR(team->t.t_pkfn, NULL)((team->t.t_pkfn)) = ((__null)); /* not needed */
4457 team->t.t_invoke = NULL__null; /* not needed */
4458
4459 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4460 team->t.t_sched.sched = new_icvs->sched.sched;
4461
4462#if KMP_ARCH_X860 || KMP_ARCH_X86_641
4463 team->t.t_fp_control_saved = FALSE0; /* not needed */
4464 team->t.t_x87_fpu_control_word = 0; /* not needed */
4465 team->t.t_mxcsr = 0; /* not needed */
4466#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4467
4468 team->t.t_construct = 0;
4469
4470 team->t.t_ordered.dt.t_value = 0;
4471 team->t.t_master_active = FALSE0;
4472
4473 memset(&team->t.t_taskq, '\0', sizeof(kmp_taskq_t));
4474
4475#ifdef KMP_DEBUG1
4476 team->t.t_copypriv_data = NULL__null; /* not necessary, but nice for debugging */
4477#endif
4478#if KMP_OS_WINDOWS0
4479 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
4480#endif
4481
4482 team->t.t_control_stack_top = NULL__null;
4483
4484 __kmp_reinitialize_team(team, new_icvs, loc);
4485
4486 KMP_MB();
4487 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: exit: team=%p\n"
, team); }
;
4488}
4489
4490#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
4491/* Sets full mask for thread and returns old mask, no changes to structures. */
4492static void
4493__kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4494 if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
4495 int status;
4496 if (old_mask != NULL__null) {
4497 status = __kmp_get_system_affinity(old_mask, TRUE)(old_mask)->get_system_affinity((!0));
4498 int error = errno(*__errno_location ());
4499 if (status != 0) {
4500 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError)__kmp_msg_format(kmp_i18n_msg_ChangeThreadAffMaskError), KMP_ERR(error)__kmp_msg_error_code(error),
4501 __kmp_msg_null);
4502 }
4503 }
4504 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE)(__kmp_affin_fullMask)->set_system_affinity((!0));
4505 }
4506}
4507#endif
4508
4509#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
4510
4511// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
4512// It calculats the worker + master thread's partition based upon the parent
4513// thread's partition, and binds each worker to a thread in their partition.
4514// The master thread's partition should already include its current binding.
4515static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
4516 // Copy the master thread's place partion to the team struct
4517 kmp_info_t *master_th = team->t.t_threads[0];
4518 KMP_DEBUG_ASSERT(master_th != NULL)if (!(master_th != __null)) { __kmp_debug_assert("master_th != __null"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4518); }
;
4519 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4520 int first_place = master_th->th.th_first_place;
4521 int last_place = master_th->th.th_last_place;
4522 int masters_place = master_th->th.th_current_place;
4523 team->t.t_first_place = first_place;
4524 team->t.t_last_place = last_place;
4525
4526 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
4527 "bound to place %d partition = [%d,%d]\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
4528 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
4529 team->t.t_id, masters_place, first_place, last_place))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
;
4530
4531 switch (proc_bind) {
4532
4533 case proc_bind_default:
4534 // serial teams might have the proc_bind policy set to proc_bind_default. It
4535 // doesn't matter, as we don't rebind master thread for any proc_bind policy
4536 KMP_DEBUG_ASSERT(team->t.t_nproc == 1)if (!(team->t.t_nproc == 1)) { __kmp_debug_assert("team->t.t_nproc == 1"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4536); }
;
4537 break;
4538
4539 case proc_bind_master: {
4540 int f;
4541 int n_th = team->t.t_nproc;
4542 for (f = 1; f < n_th; f++) {
4543 kmp_info_t *th = team->t.t_threads[f];
4544 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4544); }
;
4545 th->th.th_first_place = first_place;
4546 th->th.th_last_place = last_place;
4547 th->th.th_new_place = masters_place;
4548
4549 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
4550 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
4551 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
4552 f, masters_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
;
4553 }
4554 } break;
4555
4556 case proc_bind_close: {
4557 int f;
4558 int n_th = team->t.t_nproc;
4559 int n_places;
4560 if (first_place <= last_place) {
4561 n_places = last_place - first_place + 1;
4562 } else {
4563 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4564 }
4565 if (n_th <= n_places) {
4566 int place = masters_place;
4567 for (f = 1; f < n_th; f++) {
4568 kmp_info_t *th = team->t.t_threads[f];
4569 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4569); }
;
4570
4571 if (place == last_place) {
4572 place = first_place;
4573 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4574 place = 0;
4575 } else {
4576 place++;
4577 }
4578 th->th.th_first_place = first_place;
4579 th->th.th_last_place = last_place;
4580 th->th.th_new_place = place;
4581
4582 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
4583 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
4584 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
4585 team->t.t_id, f, place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
;
4586 }
4587 } else {
4588 int S, rem, gap, s_count;
4589 S = n_th / n_places;
4590 s_count = 0;
4591 rem = n_th - (S * n_places);
4592 gap = rem > 0 ? n_places / rem : n_places;
4593 int place = masters_place;
4594 int gap_ct = gap;
4595 for (f = 0; f < n_th; f++) {
4596 kmp_info_t *th = team->t.t_threads[f];
4597 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4597); }
;
4598
4599 th->th.th_first_place = first_place;
4600 th->th.th_last_place = last_place;
4601 th->th.th_new_place = place;
4602 s_count++;
4603
4604 if ((s_count == S) && rem && (gap_ct == gap)) {
4605 // do nothing, add an extra thread to place on next iteration
4606 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4607 // we added an extra thread to this place; move to next place
4608 if (place == last_place) {
4609 place = first_place;
4610 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4611 place = 0;
4612 } else {
4613 place++;
4614 }
4615 s_count = 0;
4616 gap_ct = 1;
4617 rem--;
4618 } else if (s_count == S) { // place full; don't add extra
4619 if (place == last_place) {
4620 place = first_place;
4621 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4622 place = 0;
4623 } else {
4624 place++;
4625 }
4626 gap_ct++;
4627 s_count = 0;
4628 }
4629
4630 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4631 ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4632 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4633 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
4634 th->th.th_new_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
;
4635 }
4636 KMP_DEBUG_ASSERT(place == masters_place)if (!(place == masters_place)) { __kmp_debug_assert("place == masters_place"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4636); }
;
4637 }
4638 } break;
4639
4640 case proc_bind_spread: {
4641 int f;
4642 int n_th = team->t.t_nproc;
4643 int n_places;
4644 int thidx;
4645 if (first_place <= last_place) {
4646 n_places = last_place - first_place + 1;
4647 } else {
4648 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4649 }
4650 if (n_th <= n_places) {
4651 int place = -1;
4652
4653 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4654 int S = n_places / n_th;
4655 int s_count, rem, gap, gap_ct;
4656
4657 place = masters_place;
4658 rem = n_places - n_th * S;
4659 gap = rem ? n_th / rem : 1;
4660 gap_ct = gap;
4661 thidx = n_th;
4662 if (update_master_only == 1)
4663 thidx = 1;
4664 for (f = 0; f < thidx; f++) {
4665 kmp_info_t *th = team->t.t_threads[f];
4666 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4666); }
;
4667
4668 th->th.th_first_place = place;
4669 th->th.th_new_place = place;
4670 s_count = 1;
4671 while (s_count < S) {
4672 if (place == last_place) {
4673 place = first_place;
4674 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4675 place = 0;
4676 } else {
4677 place++;
4678 }
4679 s_count++;
4680 }
4681 if (rem && (gap_ct == gap)) {
4682 if (place == last_place) {
4683 place = first_place;
4684 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4685 place = 0;
4686 } else {
4687 place++;
4688 }
4689 rem--;
4690 gap_ct = 0;
4691 }
4692 th->th.th_last_place = place;
4693 gap_ct++;
4694
4695 if (place == last_place) {
4696 place = first_place;
4697 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4698 place = 0;
4699 } else {
4700 place++;
4701 }
4702
4703 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, __kmp_affinity_num_masks
); }
4704 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, __kmp_affinity_num_masks
); }
4705 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, __kmp_affinity_num_masks
); }
4706 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, __kmp_affinity_num_masks
); }
4707 f, th->th.th_new_place, th->th.th_first_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, __kmp_affinity_num_masks
); }
4708 th->th.th_last_place, __kmp_affinity_num_masks))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, __kmp_affinity_num_masks
); }
;
4709 }
4710 } else {
4711 /* Having uniform space of available computation places I can create
4712 T partitions of round(P/T) size and put threads into the first
4713 place of each partition. */
4714 double current = static_cast<double>(masters_place);
4715 double spacing =
4716 (static_cast<double>(n_places + 1) / static_cast<double>(n_th));
4717 int first, last;
4718 kmp_info_t *th;
4719
4720 thidx = n_th + 1;
4721 if (update_master_only == 1)
4722 thidx = 1;
4723 for (f = 0; f < thidx; f++) {
4724 first = static_cast<int>(current);
4725 last = static_cast<int>(current + spacing) - 1;
4726 KMP_DEBUG_ASSERT(last >= first)if (!(last >= first)) { __kmp_debug_assert("last >= first"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4726); }
;
4727 if (first >= n_places) {
4728 if (masters_place) {
4729 first -= n_places;
4730 last -= n_places;
4731 if (first == (masters_place + 1)) {
4732 KMP_DEBUG_ASSERT(f == n_th)if (!(f == n_th)) { __kmp_debug_assert("f == n_th", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4732); }
;
4733 first--;
4734 }
4735 if (last == masters_place) {
4736 KMP_DEBUG_ASSERT(f == (n_th - 1))if (!(f == (n_th - 1))) { __kmp_debug_assert("f == (n_th - 1)"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4736); }
;
4737 last--;
4738 }
4739 } else {
4740 KMP_DEBUG_ASSERT(f == n_th)if (!(f == n_th)) { __kmp_debug_assert("f == n_th", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4740); }
;
4741 first = 0;
4742 last = 0;
4743 }
4744 }
4745 if (last >= n_places) {
4746 last = (n_places - 1);
4747 }
4748 place = first;
4749 current += spacing;
4750 if (f < n_th) {
4751 KMP_DEBUG_ASSERT(0 <= first)if (!(0 <= first)) { __kmp_debug_assert("0 <= first", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4751); }
;
4752 KMP_DEBUG_ASSERT(n_places > first)if (!(n_places > first)) { __kmp_debug_assert("n_places > first"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4752); }
;
4753 KMP_DEBUG_ASSERT(0 <= last)if (!(0 <= last)) { __kmp_debug_assert("0 <= last", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4753); }
;
4754 KMP_DEBUG_ASSERT(n_places > last)if (!(n_places > last)) { __kmp_debug_assert("n_places > last"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4754); }
;
4755 KMP_DEBUG_ASSERT(last_place >= first_place)if (!(last_place >= first_place)) { __kmp_debug_assert("last_place >= first_place"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4755); }
;
4756 th = team->t.t_threads[f];
4757 KMP_DEBUG_ASSERT(th)if (!(th)) { __kmp_debug_assert("th", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4757); }
;
4758 th->th.th_first_place = first;
4759 th->th.th_new_place = place;
4760 th->th.th_last_place = last;
4761
4762 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
4763 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
4764 "partition = [%d,%d], spacing = %.4f\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
4765 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
4766 team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
4767 th->th.th_first_place, th->th.th_last_place, spacing))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
;
4768 }
4769 }
4770 }
4771 KMP_DEBUG_ASSERT(update_master_only || place == masters_place)if (!(update_master_only || place == masters_place)) { __kmp_debug_assert
("update_master_only || place == masters_place", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4771); }
;
4772 } else {
4773 int S, rem, gap, s_count;
4774 S = n_th / n_places;
4775 s_count = 0;
4776 rem = n_th - (S * n_places);
4777 gap = rem > 0 ? n_places / rem : n_places;
4778 int place = masters_place;
4779 int gap_ct = gap;
4780 thidx = n_th;
4781 if (update_master_only == 1)
4782 thidx = 1;
4783 for (f = 0; f < thidx; f++) {
4784 kmp_info_t *th = team->t.t_threads[f];
4785 KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4785); }
;
4786
4787 th->th.th_first_place = place;
4788 th->th.th_last_place = place;
4789 th->th.th_new_place = place;
4790 s_count++;
4791
4792 if ((s_count == S) && rem && (gap_ct == gap)) {
4793 // do nothing, add an extra thread to place on next iteration
4794 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4795 // we added an extra thread to this place; move on to next place
4796 if (place == last_place) {
4797 place = first_place;
4798 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4799 place = 0;
4800 } else {
4801 place++;
4802 }
4803 s_count = 0;
4804 gap_ct = 1;
4805 rem--;
4806 } else if (s_count == S) { // place is full; don't add extra thread
4807 if (place == last_place) {
4808 place = first_place;
4809 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4810 place = 0;
4811 } else {
4812 place++;
4813 }
4814 gap_ct++;
4815 s_count = 0;
4816 }
4817
4818 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4819 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4820 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4821 team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4822 th->th.th_first_place, th->th.th_last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
;
4823 }
4824 KMP_DEBUG_ASSERT(update_master_only || place == masters_place)if (!(update_master_only || place == masters_place)) { __kmp_debug_assert
("update_master_only || place == masters_place", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4824); }
;
4825 }
4826 } break;
4827
4828 default:
4829 break;
4830 }
4831
4832 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: exit T#%d\n"
, team->t.t_id); }
;
4833}
4834
4835#endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */
4836
4837/* allocate a new team data structure to use. take one off of the free pool if
4838 available */
4839kmp_team_t *
4840__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
4841#if OMPT_SUPPORT1
4842 ompt_data_t ompt_parallel_data,
4843#endif
4844#if OMP_40_ENABLED(50 >= 40)
4845 kmp_proc_bind_t new_proc_bind,
4846#endif
4847 kmp_internal_control_t *new_icvs,
4848 int argc USE_NESTED_HOT_ARG(kmp_info_t *master), kmp_info_t *master) {
4849 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team)((void)0);
4850 int f;
4851 kmp_team_t *team;
4852 int use_hot_team = !root->r.r_active;
4853 int level = 0;
4854
4855 KA_TRACE(20, ("__kmp_allocate_team: called\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: called\n"
); }
;
4856 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0)if (!(new_nproc >= 1 && argc >= 0)) { __kmp_debug_assert
("new_nproc >= 1 && argc >= 0", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4856); }
;
4857 KMP_DEBUG_ASSERT(max_nproc >= new_nproc)if (!(max_nproc >= new_nproc)) { __kmp_debug_assert("max_nproc >= new_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4857); }
;
4858 KMP_MB();
4859
4860#if KMP_NESTED_HOT_TEAMS1
4861 kmp_hot_team_ptr_t *hot_teams;
4862 if (master) {
4863 team = master->th.th_team;
4864 level = team->t.t_active_level;
4865 if (master->th.th_teams_microtask) { // in teams construct?
4866 if (master->th.th_teams_size.nteams > 1 &&
4867 ( // #teams > 1
4868 team->t.t_pkfn ==
4869 (microtask_t)__kmp_teams_master || // inner fork of the teams
4870 master->th.th_teams_level <
4871 team->t.t_level)) { // or nested parallel inside the teams
4872 ++level; // not increment if #teams==1, or for outer fork of the teams;
4873 // increment otherwise
4874 }
4875 }
4876 hot_teams = master->th.th_hot_teams;
4877 if (level < __kmp_hot_teams_max_level && hot_teams &&
4878 hot_teams[level]
4879 .hot_team) { // hot team has already been allocated for given level
4880 use_hot_team = 1;
4881 } else {
4882 use_hot_team = 0;
4883 }
4884 }
4885#endif
4886 // Optimization to use a "hot" team
4887 if (use_hot_team && new_nproc > 1) {
4888 KMP_DEBUG_ASSERT(new_nproc == max_nproc)if (!(new_nproc == max_nproc)) { __kmp_debug_assert("new_nproc == max_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4888); }
;
4889#if KMP_NESTED_HOT_TEAMS1
4890 team = hot_teams[level].hot_team;
4891#else
4892 team = root->r.r_hot_team;
4893#endif
4894#if KMP_DEBUG1
4895 if (__kmp_tasking_mode != tskm_immediate_exec) {
4896 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
4897 "task_team[1] = %p before reinit\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
4898 team->t.t_task_team[0], team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
;
4899 }
4900#endif
4901
4902 // Has the number of threads changed?
4903 /* Let's assume the most common case is that the number of threads is
4904 unchanged, and put that case first. */
4905 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
4906 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team\n"
); }
;
4907 // This case can mean that omp_set_num_threads() was called and the hot
4908 // team size was already reduced, so we check the special flag
4909 if (team->t.t_size_changed == -1) {
4910 team->t.t_size_changed = 1;
4911 } else {
4912 KMP_CHECK_UPDATE(team->t.t_size_changed, 0)if ((team->t.t_size_changed) != (0)) (team->t.t_size_changed
) = (0)
;
4913 }
4914
4915 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4916 kmp_r_sched_t new_sched = new_icvs->sched;
4917 // set master's schedule as new run-time schedule
4918 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched)if ((team->t.t_sched.sched) != (new_sched.sched)) (team->
t.t_sched.sched) = (new_sched.sched)
;
4919
4920 __kmp_reinitialize_team(team, new_icvs,
4921 root->r.r_uber_thread->th.th_ident);
4922
4923 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
4924 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
;
4925 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
4926
4927#if OMP_40_ENABLED(50 >= 40)
4928#if KMP_AFFINITY_SUPPORTED1
4929 if ((team->t.t_size_changed == 0) &&
4930 (team->t.t_proc_bind == new_proc_bind)) {
4931 if (new_proc_bind == proc_bind_spread) {
4932 __kmp_partition_places(
4933 team, 1); // add flag to update only master for spread
4934 }
4935 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: "if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
4936 "proc_bind = %d, partition = [%d,%d]\n",if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
4937 team->t.t_id, new_proc_bind, team->t.t_first_place,if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
4938 team->t.t_last_place))if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
;
4939 } else {
4940 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
4941 __kmp_partition_places(team);
4942 }
4943#else
4944 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
4945#endif /* KMP_AFFINITY_SUPPORTED */
4946#endif /* OMP_40_ENABLED */
4947 } else if (team->t.t_nproc > new_nproc) {
4948 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
4949 ("__kmp_allocate_team: decreasing hot team thread count to %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
4950 new_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
;
4951
4952 team->t.t_size_changed = 1;
4953#if KMP_NESTED_HOT_TEAMS1
4954 if (__kmp_hot_teams_mode == 0) {
4955 // AC: saved number of threads should correspond to team's value in this
4956 // mode, can be bigger in mode 1, when hot team has threads in reserve
4957 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc)if (!(hot_teams[level].hot_team_nth == team->t.t_nproc)) {
__kmp_debug_assert("hot_teams[level].hot_team_nth == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4957); }
;
4958 hot_teams[level].hot_team_nth = new_nproc;
4959#endif // KMP_NESTED_HOT_TEAMS
4960 /* release the extra threads we don't need any more */
4961 for (f = new_nproc; f < team->t.t_nproc; f++) {
4962 KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4962); }
;
4963 if (__kmp_tasking_mode != tskm_immediate_exec) {
4964 // When decreasing team size, threads no longer in the team should
4965 // unref task team.
4966 team->t.t_threads[f]->th.th_task_team = NULL__null;
4967 }
4968 __kmp_free_thread(team->t.t_threads[f]);
4969 team->t.t_threads[f] = NULL__null;
4970 }
4971#if KMP_NESTED_HOT_TEAMS1
4972 } // (__kmp_hot_teams_mode == 0)
4973 else {
4974 // When keeping extra threads in team, switch threads to wait on own
4975 // b_go flag
4976 for (f = new_nproc; f < team->t.t_nproc; ++f) {
4977 KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4977); }
;
4978 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
4979 for (int b = 0; b < bs_last_barrier; ++b) {
4980 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG2) {
4981 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG3;
4982 }
4983 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0)if ((balign[b].bb.leaf_kids) != (0)) (balign[b].bb.leaf_kids)
= (0)
;
4984 }
4985 }
4986 }
4987#endif // KMP_NESTED_HOT_TEAMS
4988 team->t.t_nproc = new_nproc;
4989 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4990 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched)if ((team->t.t_sched.sched) != (new_icvs->sched.sched))
(team->t.t_sched.sched) = (new_icvs->sched.sched)
;
4991 __kmp_reinitialize_team(team, new_icvs,
4992 root->r.r_uber_thread->th.th_ident);
4993
4994 /* update the remaining threads */
4995 for (f = 0; f < new_nproc; ++f) {
4996 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
4997 }
4998 // restore the current task state of the master thread: should be the
4999 // implicit task
5000 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
5001 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
;
5002
5003 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5004
5005#ifdef KMP_DEBUG1
5006 for (f = 0; f < team->t.t_nproc; f++) {
5007 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5009); }
5008 team->t.t_threads[f]->th.th_team_nproc ==if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5009); }
5009 team->t.t_nproc)if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5009); }
;
5010 }
5011#endif
5012
5013#if OMP_40_ENABLED(50 >= 40)
5014 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
5015#if KMP_AFFINITY_SUPPORTED1
5016 __kmp_partition_places(team);
5017#endif
5018#endif
5019 } else { // team->t.t_nproc < new_nproc
5020#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
5021 kmp_affin_mask_t *old_mask;
5022 if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
5023 KMP_CPU_ALLOC(old_mask)(old_mask = __kmp_affinity_dispatch->allocate_mask());
5024 }
5025#endif
5026
5027 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
5028 ("__kmp_allocate_team: increasing hot team thread count to %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
5029 new_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
;
5030
5031 team->t.t_size_changed = 1;
5032
5033#if KMP_NESTED_HOT_TEAMS1
5034 int avail_threads = hot_teams[level].hot_team_nth;
5035 if (new_nproc < avail_threads)
5036 avail_threads = new_nproc;
5037 kmp_info_t **other_threads = team->t.t_threads;
5038 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5039 // Adjust barrier data of reserved threads (if any) of the team
5040 // Other data will be set in __kmp_initialize_info() below.
5041 int b;
5042 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5043 for (b = 0; b < bs_last_barrier; ++b) {
5044 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5045 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5045); }
;
5046#if USE_DEBUGGER0
5047 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5048#endif
5049 }
5050 }
5051 if (hot_teams[level].hot_team_nth >= new_nproc) {
5052 // we have all needed threads in reserve, no need to allocate any
5053 // this only possible in mode 1, cannot have reserved threads in mode 0
5054 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1)if (!(__kmp_hot_teams_mode == 1)) { __kmp_debug_assert("__kmp_hot_teams_mode == 1"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5054); }
;
5055 team->t.t_nproc = new_nproc; // just get reserved threads involved
5056 } else {
5057 // we may have some threads in reserve, but not enough
5058 team->t.t_nproc =
5059 hot_teams[level]
5060 .hot_team_nth; // get reserved threads involved if any
5061 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size
5062#endif // KMP_NESTED_HOT_TEAMS
5063 if (team->t.t_max_nproc < new_nproc) {
5064 /* reallocate larger arrays */
5065 __kmp_reallocate_team_arrays(team, new_nproc);
5066 __kmp_reinitialize_team(team, new_icvs, NULL__null);
5067 }
5068
5069#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
5070 /* Temporarily set full mask for master thread before creation of
5071 workers. The reason is that workers inherit the affinity from master,
5072 so if a lot of workers are created on the single core quickly, they
5073 don't get a chance to set their own affinity for a long time. */
5074 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5075#endif
5076
5077 /* allocate new threads for the hot team */
5078 for (f = team->t.t_nproc; f < new_nproc; f++) {
5079 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5080 KMP_DEBUG_ASSERT(new_worker)if (!(new_worker)) { __kmp_debug_assert("new_worker", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5080); }
;
5081 team->t.t_threads[f] = new_worker;
5082
5083 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5084 ("__kmp_allocate_team: team %d init T#%d arrived: "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5085 "join=%llu, plain=%llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5086 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5087 team->t.t_bar[bs_forkjoin_barrier].b_arrived,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5088 team->t.t_bar[bs_plain_barrier].b_arrived))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
;
5089
5090 { // Initialize barrier data for new threads.
5091 int b;
5092 kmp_balign_t *balign = new_worker->th.th_bar;
5093 for (b = 0; b < bs_last_barrier; ++b) {
5094 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5095 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5096); }
5096 KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5096); }
;
5097#if USE_DEBUGGER0
5098 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5099#endif
5100 }
5101 }
5102 }
5103
5104#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
5105 if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
5106 /* Restore initial master thread's affinity mask */
5107 __kmp_set_system_affinity(old_mask, TRUE)(old_mask)->set_system_affinity((!0));
5108 KMP_CPU_FREE(old_mask)__kmp_affinity_dispatch->deallocate_mask(old_mask);
5109 }
5110#endif
5111#if KMP_NESTED_HOT_TEAMS1
5112 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
5113#endif // KMP_NESTED_HOT_TEAMS
5114 /* make sure everyone is syncronized */
5115 int old_nproc = team->t.t_nproc; // save old value and use to update only
5116 // new threads below
5117 __kmp_initialize_team(team, new_nproc, new_icvs,
5118 root->r.r_uber_thread->th.th_ident);
5119
5120 /* reinitialize the threads */
5121 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc)if (!(team->t.t_nproc == new_nproc)) { __kmp_debug_assert(
"team->t.t_nproc == new_nproc", "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5121); }
;
5122 for (f = 0; f < team->t.t_nproc; ++f)
5123 __kmp_initialize_info(team->t.t_threads[f], team, f,
5124 __kmp_gtid_from_tid(f, team));
5125 if (level) { // set th_task_state for new threads in nested hot team
5126 // __kmp_initialize_info() no longer zeroes th_task_state, so we should
5127 // only need to set the th_task_state for the new threads. th_task_state
5128 // for master thread will not be accurate until after this in
5129 // __kmp_fork_call(), so we look to the master's memo_stack to get the
5130 // correct value.
5131 for (f = old_nproc; f < team->t.t_nproc; ++f)
5132 team->t.t_threads[f]->th.th_task_state =
5133 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5134 } else { // set th_task_state for new threads in non-nested hot team
5135 int old_state =
5136 team->t.t_threads[0]->th.th_task_state; // copy master's state
5137 for (f = old_nproc; f < team->t.t_nproc; ++f)
5138 team->t.t_threads[f]->th.th_task_state = old_state;
5139 }
5140
5141#ifdef KMP_DEBUG1
5142 for (f = 0; f < team->t.t_nproc; ++f) {
5143 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5145); }
5144 team->t.t_threads[f]->th.th_team_nproc ==if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5145); }
5145 team->t.t_nproc)if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5145); }
;
5146 }
5147#endif
5148
5149#if OMP_40_ENABLED(50 >= 40)
5150 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
5151#if KMP_AFFINITY_SUPPORTED1
5152 __kmp_partition_places(team);
5153#endif
5154#endif
5155 } // Check changes in number of threads
5156
5157#if OMP_40_ENABLED(50 >= 40)
5158 kmp_info_t *master = team->t.t_threads[0];
5159 if (master->th.th_teams_microtask) {
5160 for (f = 1; f < new_nproc; ++f) {
5161 // propagate teams construct specific info to workers
5162 kmp_info_t *thr = team->t.t_threads[f];
5163 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5164 thr->th.th_teams_level = master->th.th_teams_level;
5165 thr->th.th_teams_size = master->th.th_teams_size;
5166 }
5167 }
5168#endif /* OMP_40_ENABLED */
5169#if KMP_NESTED_HOT_TEAMS1
5170 if (level) {
5171 // Sync barrier state for nested hot teams, not needed for outermost hot
5172 // team.
5173 for (f = 1; f < new_nproc; ++f) {
5174 kmp_info_t *thr = team->t.t_threads[f];
5175 int b;
5176 kmp_balign_t *balign = thr->th.th_bar;
5177 for (b = 0; b < bs_last_barrier; ++b) {
5178 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5179 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-8~svn345461/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5179); }
;
5180#if USE_DEBUGGER0
5181 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5182#endif
5183 }
5184 }
5185 }
5186#endif // KMP_NESTED_HOT_TEAMS
5187
5188 /* reallocate space for arguments if necessary */
5189 __kmp_alloc_argv_entries(argc, team, TRUE(!0));
5190 KMP_CHECK_UPDATE(team->t.t_argc, argc)if ((team->t.t_argc) != (argc)) (team->t.t_argc) = (argc
)
;
5191 // The hot team re-uses the previous task team,
5192 // if untouched during the previous release->gather phase.
5193
5194 KF_TRACE(10, (" hot_team = %p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf (" hot_team = %p\n"
, team); }
;
5195
5196#if KMP_DEBUG1
5197 if (__kmp_tasking_mode != tskm_immediate_exec) {
5198 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
5199 "task_team[1] = %p after reinit\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
5200 team->t.t_task_team[0], team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
;
5201 }
5202#endif
5203
5204#if OMPT_SUPPORT1
5205 __ompt_team_assign_id(team, ompt_parallel_data);
5206#endif
5207
5208 KMP_MB();
5209
5210 return team;
5211 }
5212
5213 /* next, let's try to take one from the team pool */
5214 KMP_MB();
5215 for (team = CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool); (team);) {
5216 /* TODO: consider resizing undersized teams instead of reaping them, now
5217 that we have a resizing mechanism */
5218 if (team->t.t_max_nproc >= max_nproc) {
5219 /* take this team from the team pool */
5220 __kmp_team_pool = team->t.t_next_pool;
5221
5222 /* setup the team for fresh use */
5223 __kmp_initialize_team(team, new_nproc, new_icvs, NULL__null);
5224
5225 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
"task_team[1] %p to NULL\n", &team->t.t_task_team[0],
&team->t.t_task_team[1]); }
5226 "task_team[1] %p to NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
"task_team[1] %p to NULL\n", &team->t.t_task_team[0],
&team->t.t_task_team[1]); }
5227 &team->t.t_task_team[0], &team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
"task_team[1] %p to NULL\n", &team->t.t_task_team[0],
&team->t.t_task_team[1]); }
;
5228 team->t.t_task_team[0] = NULL__null;
5229 team->t.t_task_team[1] = NULL__null;
5230
5231 /* reallocate space for arguments if necessary */
5232 __kmp_alloc_argv_entries(argc, team, TRUE(!0));
5233 KMP_CHECK_UPDATE(team->t.t_argc, argc)if ((team->t.t_argc) != (argc)) (team->t.t_argc) = (argc
)
;
5234
5235 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
5236 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
5237 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
;
5238 { // Initialize barrier data.
5239 int b;
5240 for (b = 0; b < bs_last_barrier; ++b) {
5241 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE0;
5242#if USE_DEBUGGER0
5243 team->t.t_bar[b].b_master_arrived = 0;
5244 team->t.t_bar[b].b_team_arrived = 0;
5245#endif
5246 }
5247 }
5248
5249#if OMP_40_ENABLED(50 >= 40)
5250 team->t.t_proc_bind = new_proc_bind;
5251#endif
5252
5253 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: using team from pool %d.\n"
, team->t.t_id); }
5254 team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: using team from pool %d.\n"
, team->t.t_id); }
;
5255
5256#if OMPT_SUPPORT1
5257 __ompt_team_assign_id(team, ompt_parallel_data);
5258#endif
5259
5260 KMP_MB();
5261
5262 return team;
5263 }
5264
5265 /* reap team if it is too small, then loop back and check the next one */
5266 // not sure if this is wise, but, will be redone during the hot-teams
5267 // rewrite.
5268 /* TODO: Use technique to find the right size hot-team, don't reap them */
5269 team = __kmp_reap_team(team);
5270 __kmp_team_pool = team;
5271 }
5272
5273 /* nothing available in the pool, no matter, make a new team! */
5274 KMP_MB();
5275 team = (kmp_team_t *)