Bug Summary

File:projects/openmp/runtime/src/kmp_runtime.cpp
Warning:line 1802, column 15
3rd function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name kmp_runtime.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D omp_EXPORTS -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/projects/openmp/runtime/src -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn338205/include -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/i18n -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/include/50 -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/thirdparty/ittnotify -U NDEBUG -D _GNU_SOURCE -D _REENTRANT -D _FORTIFY_SOURCE=2 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-comment -Wno-sign-compare -Wno-unused-function -Wno-unused-value -Wno-unused-variable -Wno-switch -Wno-unknown-pragmas -Wno-missing-field-initializers -Wno-missing-braces -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/projects/openmp/runtime/src -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fno-rtti -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-07-29-043837-17923-1 -x c++ /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp -faddrsig
1/*
2 * kmp_runtime.cpp -- KPTS runtime support library
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// The LLVM Compiler Infrastructure
8//
9// This file is dual licensed under the MIT and the University of Illinois Open
10// Source Licenses. See LICENSE.txt for details.
11//
12//===----------------------------------------------------------------------===//
13
14#include "kmp.h"
15#include "kmp_affinity.h"
16#include "kmp_atomic.h"
17#include "kmp_environment.h"
18#include "kmp_error.h"
19#include "kmp_i18n.h"
20#include "kmp_io.h"
21#include "kmp_itt.h"
22#include "kmp_settings.h"
23#include "kmp_stats.h"
24#include "kmp_str.h"
25#include "kmp_wait_release.h"
26#include "kmp_wrapper_getpid.h"
27#include "kmp_dispatch.h"
28#if KMP_USE_HIER_SCHED0
29#include "kmp_dispatch_hier.h"
30#endif
31
32#if OMPT_SUPPORT1
33#include "ompt-specific.h"
34#endif
35
36/* these are temporary issues to be dealt with */
37#define KMP_USE_PRCTL0 0
38
39#if KMP_OS_WINDOWS0
40#include <process.h>
41#endif
42
43#include "tsan_annotations.h"
44
45#if defined(KMP_GOMP_COMPAT)
46char const __kmp_version_alt_comp[] =
47 KMP_VERSION_PREFIX"\x00@(#) " "Intel(R) OMP " "alternative compiler support: yes";
48#endif /* defined(KMP_GOMP_COMPAT) */
49
50char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX"\x00@(#) " "Intel(R) OMP " "API version: "
51#if OMP_50_ENABLED(50 >= 50)
52 "5.0 (201611)";
53#elif OMP_45_ENABLED(50 >= 45)
54 "4.5 (201511)";
55#elif OMP_40_ENABLED(50 >= 40)
56 "4.0 (201307)";
57#else
58 "3.1 (201107)";
59#endif
60
61#ifdef KMP_DEBUG1
62char const __kmp_version_lock[] =
63 KMP_VERSION_PREFIX"\x00@(#) " "Intel(R) OMP " "lock type: run time selectable";
64#endif /* KMP_DEBUG */
65
66#define KMP_MIN(x, y)((x) < (y) ? (x) : (y)) ((x) < (y) ? (x) : (y))
67
68/* ------------------------------------------------------------------------ */
69
70#if KMP_USE_MONITOR
71kmp_info_t __kmp_monitor;
72#endif
73
74/* Forward declarations */
75
76void __kmp_cleanup(void);
77
78static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
79 int gtid);
80static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
81 kmp_internal_control_t *new_icvs,
82 ident_t *loc);
83#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
84static void __kmp_partition_places(kmp_team_t *team,
85 int update_master_only = 0);
86#endif
87static void __kmp_do_serial_initialize(void);
88void __kmp_fork_barrier(int gtid, int tid);
89void __kmp_join_barrier(int gtid);
90void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
91 kmp_internal_control_t *new_icvs, ident_t *loc);
92
93#ifdef USE_LOAD_BALANCE1
94static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
95#endif
96
97static int __kmp_expand_threads(int nNeed);
98#if KMP_OS_WINDOWS0
99static int __kmp_unregister_root_other_thread(int gtid);
100#endif
101static void __kmp_unregister_library(void); // called by __kmp_internal_end()
102static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
103kmp_info_t *__kmp_thread_pool_insert_pt = NULL__null;
104
105/* Calculate the identifier of the current thread */
106/* fast (and somewhat portable) way to get unique identifier of executing
107 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
108int __kmp_get_global_thread_id() {
109 int i;
110 kmp_info_t **other_threads;
111 size_t stack_data;
112 char *stack_addr;
113 size_t stack_size;
114 char *stack_base;
115
116 KA_TRACE(if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
117 1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
118 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
119 __kmp_nth, __kmp_all_nth))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
;
120
121 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
122 a parallel region, made it return KMP_GTID_DNE to force serial_initialize
123 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
124 __kmp_init_gtid for this to work. */
125
126 if (!TCR_4(__kmp_init_gtid)(__kmp_init_gtid))
127 return KMP_GTID_DNE(-2);
128
129#ifdef KMP_TDATA_GTID1
130 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
131 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using TDATA\n"
); }
;
132 return __kmp_gtid;
133 }
134#endif
135 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
136 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using keyed TLS\n"
); }
;
137 return __kmp_gtid_get_specific();
138 }
139 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using internal alg.\n"
); }
;
140
141 stack_addr = (char *)&stack_data;
142 other_threads = __kmp_threads;
143
144 /* ATT: The code below is a source of potential bugs due to unsynchronized
145 access to __kmp_threads array. For example:
146 1. Current thread loads other_threads[i] to thr and checks it, it is
147 non-NULL.
148 2. Current thread is suspended by OS.
149 3. Another thread unregisters and finishes (debug versions of free()
150 may fill memory with something like 0xEF).
151 4. Current thread is resumed.
152 5. Current thread reads junk from *thr.
153 TODO: Fix it. --ln */
154
155 for (i = 0; i < __kmp_threads_capacity; i++) {
156
157 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i])((void *)(other_threads[i]));
158 if (!thr)
159 continue;
160
161 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize)((void *)(thr->th.th_info.ds.ds_stacksize));
162 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase)((void *)(thr->th.th_info.ds.ds_stackbase));
163
164 /* stack grows down -- search through all of the active threads */
165
166 if (stack_addr <= stack_base) {
167 size_t stack_diff = stack_base - stack_addr;
168
169 if (stack_diff <= stack_size) {
170 /* The only way we can be closer than the allocated */
171 /* stack size is if we are running on this thread. */
172 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i)((__kmp_gtid_get_specific() == i) ? 0 : __kmp_debug_assert("__kmp_gtid_get_specific() == i"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 172))
;
173 return i;
174 }
175 }
176 }
177
178 /* get specific to try and determine our gtid */
179 KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
180 ("*** __kmp_get_global_thread_id: internal alg. failed to find "if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
181 "thread, using TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
"thread, using TLS\n"); }
;
182 i = __kmp_gtid_get_specific();
183
184 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */
185
186 /* if we havn't been assigned a gtid, then return code */
187 if (i < 0)
188 return i;
189
190 /* dynamically updated stack window for uber threads to avoid get_specific
191 call */
192 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
193 KMP_FATAL(StackOverflow, i)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_StackOverflow, i), __kmp_msg_null
)
;
194 }
195
196 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
197 if (stack_addr > stack_base) {
198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr)((other_threads[i]->th.th_info.ds.ds_stackbase)) = ((stack_addr
))
;
199 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
200 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
201 stack_base)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
;
202 } else {
203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
- stack_addr))
204 stack_base - stack_addr)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
- stack_addr))
;
205 }
206
207 /* Reprint stack bounds for ubermaster since they have been refined */
208 if (__kmp_storage_map) {
209 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
210 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
211 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
212 other_threads[i]->th.th_info.ds.ds_stacksize,
213 "th_%d stack (refinement)", i);
214 }
215 return i;
216}
217
218int __kmp_get_global_thread_id_reg() {
219 int gtid;
220
221 if (!__kmp_init_serial) {
222 gtid = KMP_GTID_DNE(-2);
223 } else
224#ifdef KMP_TDATA_GTID1
225 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
226 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using TDATA\n"
); }
;
227 gtid = __kmp_gtid;
228 } else
229#endif
230 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
231 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"
); }
;
232 gtid = __kmp_gtid_get_specific();
233 } else {
234 KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); }
235 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); }
;
236 gtid = __kmp_get_global_thread_id();
237 }
238
239 /* we must be a new uber master sibling thread */
240 if (gtid == KMP_GTID_DNE(-2)) {
241 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
242 ("__kmp_get_global_thread_id_reg: Encountered new root thread. "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
243 "Registering a new gtid.\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
"Registering a new gtid.\n"); }
;
244 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
245 if (!__kmp_init_serial) {
246 __kmp_do_serial_initialize();
247 gtid = __kmp_gtid_get_specific();
248 } else {
249 gtid = __kmp_register_root(FALSE0);
250 }
251 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
252 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
253 }
254
255 KMP_DEBUG_ASSERT(gtid >= 0)((gtid >= 0) ? 0 : __kmp_debug_assert("gtid >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 255))
;
256
257 return gtid;
258}
259
260/* caller must hold forkjoin_lock */
261void __kmp_check_stack_overlap(kmp_info_t *th) {
262 int f;
263 char *stack_beg = NULL__null;
264 char *stack_end = NULL__null;
265 int gtid;
266
267 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: called\n"
); }
;
268 if (__kmp_storage_map) {
269 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
270 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
271
272 gtid = __kmp_gtid_from_thread(th)((((th) != __null) ? 0 : __kmp_debug_assert("(th) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 272)), (th)->th.th_info.ds.ds_gtid)
;
273
274 if (gtid == KMP_GTID_MONITOR(-4)) {
275 __kmp_print_storage_map_gtid(
276 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
277 "th_%s stack (%s)", "mon",
278 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
279 } else {
280 __kmp_print_storage_map_gtid(
281 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
282 "th_%d stack (%s)", gtid,
283 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
284 }
285 }
286
287 /* No point in checking ubermaster threads since they use refinement and
288 * cannot overlap */
289 gtid = __kmp_gtid_from_thread(th)((((th) != __null) ? 0 : __kmp_debug_assert("(th) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 289)), (th)->th.th_info.ds.ds_gtid)
;
290 if (__kmp_env_checks == TRUE(!0) && !KMP_UBER_GTID(gtid)((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 290)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 290)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))
) {
291 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); }
292 ("__kmp_check_stack_overlap: performing extensive checking\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); }
;
293 if (stack_beg == NULL__null) {
294 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
295 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
296 }
297
298 for (f = 0; f < __kmp_threads_capacity; f++) {
299 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f])((void *)(__kmp_threads[f]));
300
301 if (f_th && f_th != th) {
302 char *other_stack_end =
303 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase)((void *)(f_th->th.th_info.ds.ds_stackbase));
304 char *other_stack_beg =
305 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize));
306 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
307 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
308
309 /* Print the other stack values before the abort */
310 if (__kmp_storage_map)
311 __kmp_print_storage_map_gtid(
312 -1, other_stack_beg, other_stack_end,
313 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize)),
314 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th)((((f_th) != __null) ? 0 : __kmp_debug_assert("(f_th) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 314)), (f_th)->th.th_info.ds.ds_gtid)
);
315
316 __kmp_fatal(KMP_MSG(StackOverlap)__kmp_msg_format(kmp_i18n_msg_StackOverlap), KMP_HNT(ChangeStackLimit)__kmp_msg_format(kmp_i18n_hnt_ChangeStackLimit),
317 __kmp_msg_null);
318 }
319 }
320 }
321 }
322 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: returning\n"
); }
;
323}
324
325/* ------------------------------------------------------------------------ */
326
327void __kmp_infinite_loop(void) {
328 static int done = FALSE0;
329
330 while (!done) {
331 KMP_YIELD(1){ __kmp_x86_pause(); __kmp_yield((1)); };
332 }
333}
334
335#define MAX_MESSAGE512 512
336
337void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
338 char const *format, ...) {
339 char buffer[MAX_MESSAGE512];
340 va_list ap;
341
342 va_start(ap, format)__builtin_va_start(ap, format);
343 KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
344 p2, (unsigned long)size, format);
345 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
346 __kmp_vprintf(kmp_err, buffer, ap);
347#if KMP_PRINT_DATA_PLACEMENT
348 int node;
349 if (gtid >= 0) {
350 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
351 if (__kmp_storage_map_verbose) {
352 node = __kmp_get_host_node(p1);
353 if (node < 0) /* doesn't work, so don't try this next time */
354 __kmp_storage_map_verbose = FALSE0;
355 else {
356 char *last;
357 int lastNode;
358 int localProc = __kmp_get_cpu_from_gtid(gtid);
359
360 const int page_size = KMP_GET_PAGE_SIZE()getpagesize();
361
362 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
363 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
364 if (localProc >= 0)
365 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid,
366 localProc >> 1);
367 else
368 __kmp_printf_no_lock(" GTID %d\n", gtid);
369#if KMP_USE_PRCTL0
370 /* The more elaborate format is disabled for now because of the prctl
371 * hanging bug. */
372 do {
373 last = p1;
374 lastNode = node;
375 /* This loop collates adjacent pages with the same host node. */
376 do {
377 (char *)p1 += page_size;
378 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
379 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1,
380 lastNode);
381 } while (p1 <= p2);
382#else
383 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1,
384 (char *)p1 + (page_size - 1),
385 __kmp_get_host_node(p1));
386 if (p1 < p2) {
387 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2,
388 (char *)p2 + (page_size - 1),
389 __kmp_get_host_node(p2));
390 }
391#endif
392 }
393 }
394 } else
395 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)__kmp_i18n_catgets(kmp_i18n_str_StorageMapWarning));
396 }
397#endif /* KMP_PRINT_DATA_PLACEMENT */
398 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
399}
400
401void __kmp_warn(char const *format, ...) {
402 char buffer[MAX_MESSAGE512];
403 va_list ap;
404
405 if (__kmp_generate_warnings == kmp_warnings_off) {
406 return;
407 }
408
409 va_start(ap, format)__builtin_va_start(ap, format);
410
411 KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP warning: %s\n", format);
412 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
413 __kmp_vprintf(kmp_err, buffer, ap);
414 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
415
416 va_end(ap)__builtin_va_end(ap);
417}
418
419void __kmp_abort_process() {
420 // Later threads may stall here, but that's ok because abort() will kill them.
421 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
422
423 if (__kmp_debug_buf) {
424 __kmp_dump_debug_buffer();
425 }
426
427 if (KMP_OS_WINDOWS0) {
428 // Let other threads know of abnormal termination and prevent deadlock
429 // if abort happened during library initialization or shutdown
430 __kmp_global.g.g_abort = SIGABRT6;
431
432 /* On Windows* OS by default abort() causes pop-up error box, which stalls
433 nightly testing. Unfortunately, we cannot reliably suppress pop-up error
434 boxes. _set_abort_behavior() works well, but this function is not
435 available in VS7 (this is not problem for DLL, but it is a problem for
436 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
437 help, at least in some versions of MS C RTL.
438
439 It seems following sequence is the only way to simulate abort() and
440 avoid pop-up error box. */
441 raise(SIGABRT6);
442 _exit(3); // Just in case, if signal ignored, exit anyway.
443 } else {
444 abort();
445 }
446
447 __kmp_infinite_loop();
448 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
449
450} // __kmp_abort_process
451
452void __kmp_abort_thread(void) {
453 // TODO: Eliminate g_abort global variable and this function.
454 // In case of abort just call abort(), it will kill all the threads.
455 __kmp_infinite_loop();
456} // __kmp_abort_thread
457
458/* Print out the storage map for the major kmp_info_t thread data structures
459 that are allocated together. */
460
461static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
462 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
463 gtid);
464
465 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
466 sizeof(kmp_desc_t), "th_%d.th_info", gtid);
467
468 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
469 sizeof(kmp_local_t), "th_%d.th_local", gtid);
470
471 __kmp_print_storage_map_gtid(
472 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
473 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);
474
475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
476 &thr->th.th_bar[bs_plain_barrier + 1],
477 sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
478 gtid);
479
480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
481 &thr->th.th_bar[bs_forkjoin_barrier + 1],
482 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
483 gtid);
484
485#if KMP_FAST_REDUCTION_BARRIER1
486 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
487 &thr->th.th_bar[bs_reduction_barrier + 1],
488 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
489 gtid);
490#endif // KMP_FAST_REDUCTION_BARRIER
491}
492
493/* Print out the storage map for the major kmp_team_t team data structures
494 that are allocated together. */
495
496static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
497 int team_id, int num_thr) {
498 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
499 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
500 header, team_id);
501
502 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
503 &team->t.t_bar[bs_last_barrier],
504 sizeof(kmp_balign_team_t) * bs_last_barrier,
505 "%s_%d.t_bar", header, team_id);
506
507 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
508 &team->t.t_bar[bs_plain_barrier + 1],
509 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
510 header, team_id);
511
512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
513 &team->t.t_bar[bs_forkjoin_barrier + 1],
514 sizeof(kmp_balign_team_t),
515 "%s_%d.t_bar[forkjoin]", header, team_id);
516
517#if KMP_FAST_REDUCTION_BARRIER1
518 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
519 &team->t.t_bar[bs_reduction_barrier + 1],
520 sizeof(kmp_balign_team_t),
521 "%s_%d.t_bar[reduction]", header, team_id);
522#endif // KMP_FAST_REDUCTION_BARRIER
523
524 __kmp_print_storage_map_gtid(
525 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
526 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);
527
528 __kmp_print_storage_map_gtid(
529 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
530 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);
531
532 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
533 &team->t.t_disp_buffer[num_disp_buff],
534 sizeof(dispatch_shared_info_t) * num_disp_buff,
535 "%s_%d.t_disp_buffer", header, team_id);
536
537 __kmp_print_storage_map_gtid(-1, &team->t.t_taskq, &team->t.t_copypriv_data,
538 sizeof(kmp_taskq_t), "%s_%d.t_taskq", header,
539 team_id);
540}
541
542static void __kmp_init_allocator() {}
543static void __kmp_fini_allocator() {}
544
545/* ------------------------------------------------------------------------ */
546
547#ifdef KMP_DYNAMIC_LIB1
548#if KMP_OS_WINDOWS0
549
550static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
551 // TODO: Change to __kmp_break_bootstrap_lock().
552 __kmp_init_bootstrap_lock(lck); // make the lock released
553}
554
555static void __kmp_reset_locks_on_process_detach(int gtid_req) {
556 int i;
557 int thread_count;
558
559 // PROCESS_DETACH is expected to be called by a thread that executes
560 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one
561 // calling ProcessExit or FreeLibrary). So, it might be safe to access the
562 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some
563 // threads can be still alive here, although being about to be terminated. The
564 // threads in the array with ds_thread==0 are most suspicious. Actually, it
565 // can be not safe to access the __kmp_threads[].
566
567 // TODO: does it make sense to check __kmp_roots[] ?
568
569 // Let's check that there are no other alive threads registered with the OMP
570 // lib.
571 while (1) {
572 thread_count = 0;
573 for (i = 0; i < __kmp_threads_capacity; ++i) {
574 if (!__kmp_threads)
575 continue;
576 kmp_info_t *th = __kmp_threads[i];
577 if (th == NULL__null)
578 continue;
579 int gtid = th->th.th_info.ds.ds_gtid;
580 if (gtid == gtid_req)
581 continue;
582 if (gtid < 0)
583 continue;
584 DWORD exit_val;
585 int alive = __kmp_is_thread_alive(th, &exit_val);
586 if (alive) {
587 ++thread_count;
588 }
589 }
590 if (thread_count == 0)
591 break; // success
592 }
593
594 // Assume that I'm alone. Now it might be safe to check and reset locks.
595 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset.
596 __kmp_reset_lock(&__kmp_forkjoin_lock);
597#ifdef KMP_DEBUG1
598 __kmp_reset_lock(&__kmp_stdio_lock);
599#endif // KMP_DEBUG
600}
601
602BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
603 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
604
605 switch (fdwReason) {
606
607 case DLL_PROCESS_ATTACH:
608 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_ATTACH\n"
); }
;
609
610 return TRUE(!0);
611
612 case DLL_PROCESS_DETACH:
613 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_DETACH T#%d\n"
, __kmp_gtid_get_specific()); }
;
614
615 if (lpReserved != NULL__null) {
616 // lpReserved is used for telling the difference:
617 // lpReserved == NULL when FreeLibrary() was called,
618 // lpReserved != NULL when the process terminates.
619 // When FreeLibrary() is called, worker threads remain alive. So they will
620 // release the forkjoin lock by themselves. When the process terminates,
621 // worker threads disappear triggering the problem of unreleased forkjoin
622 // lock as described below.
623
624 // A worker thread can take the forkjoin lock. The problem comes up if
625 // that worker thread becomes dead before it releases the forkjoin lock.
626 // The forkjoin lock remains taken, while the thread executing
627 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try
628 // to take the forkjoin lock and will always fail, so that the application
629 // will never finish [normally]. This scenario is possible if
630 // __kmpc_end() has not been executed. It looks like it's not a corner
631 // case, but common cases:
632 // - the main function was compiled by an alternative compiler;
633 // - the main function was compiled by icl but without /Qopenmp
634 // (application with plugins);
635 // - application terminates by calling C exit(), Fortran CALL EXIT() or
636 // Fortran STOP.
637 // - alive foreign thread prevented __kmpc_end from doing cleanup.
638 //
639 // This is a hack to work around the problem.
640 // TODO: !!! figure out something better.
641 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
642 }
643
644 __kmp_internal_end_library(__kmp_gtid_get_specific());
645
646 return TRUE(!0);
647
648 case DLL_THREAD_ATTACH:
649 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_ATTACH\n"
); }
;
650
651 /* if we want to register new siblings all the time here call
652 * __kmp_get_gtid(); */
653 return TRUE(!0);
654
655 case DLL_THREAD_DETACH:
656 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_DETACH T#%d\n"
, __kmp_gtid_get_specific()); }
;
657
658 __kmp_internal_end_thread(__kmp_gtid_get_specific());
659 return TRUE(!0);
660 }
661
662 return TRUE(!0);
663}
664
665#endif /* KMP_OS_WINDOWS */
666#endif /* KMP_DYNAMIC_LIB */
667
668/* Change the library type to "status" and return the old type */
669/* called from within initialization routines where __kmp_initz_lock is held */
670int __kmp_change_library(int status) {
671 int old_status;
672
673 old_status = __kmp_yield_init &
674 1; // check whether KMP_LIBRARY=throughput (even init count)
675
676 if (status) {
677 __kmp_yield_init |= 1; // throughput => turnaround (odd init count)
678 } else {
679 __kmp_yield_init &= ~1; // turnaround => throughput (even init count)
680 }
681
682 return old_status; // return previous setting of whether
683 // KMP_LIBRARY=throughput
684}
685
686/* __kmp_parallel_deo -- Wait until it's our turn. */
687void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
688 int gtid = *gtid_ref;
689#ifdef BUILD_PARALLEL_ORDERED1
690 kmp_team_t *team = __kmp_team_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 690)), __kmp_threads[(gtid)]->th.th_team)
;
691#endif /* BUILD_PARALLEL_ORDERED */
692
693 if (__kmp_env_consistency_check) {
694 if (__kmp_threads[gtid]->th.th_root->r.r_active)
695#if KMP_USE_DYNAMIC_LOCK1
696 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null, 0);
697#else
698 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null);
699#endif
700 }
701#ifdef BUILD_PARALLEL_ORDERED1
702 if (!team->t.t_serialized) {
703 KMP_MB();
704 KMP_WAIT_YIELD__kmp_wait_yield_4(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 704)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
,
705 KMP_EQ__kmp_eq_4, NULL__null);
706 KMP_MB();
707 }
708#endif /* BUILD_PARALLEL_ORDERED */
709}
710
711/* __kmp_parallel_dxo -- Signal the next task. */
712void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
713 int gtid = *gtid_ref;
714#ifdef BUILD_PARALLEL_ORDERED1
715 int tid = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 715)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
716 kmp_team_t *team = __kmp_team_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 716)), __kmp_threads[(gtid)]->th.th_team)
;
717#endif /* BUILD_PARALLEL_ORDERED */
718
719 if (__kmp_env_consistency_check) {
720 if (__kmp_threads[gtid]->th.th_root->r.r_active)
721 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
722 }
723#ifdef BUILD_PARALLEL_ORDERED1
724 if (!team->t.t_serialized) {
725 KMP_MB(); /* Flush all pending memory write invalidates. */
726
727 /* use the tid of the next thread in this team */
728 /* TODO replace with general release procedure */
729 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
730
731 KMP_MB(); /* Flush all pending memory write invalidates. */
732 }
733#endif /* BUILD_PARALLEL_ORDERED */
734}
735
736/* ------------------------------------------------------------------------ */
737/* The BARRIER for a SINGLE process section is always explicit */
738
739int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
740 int status;
741 kmp_info_t *th;
742 kmp_team_t *team;
743
744 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
745 __kmp_parallel_initialize();
746
747 th = __kmp_threads[gtid];
748 team = th->th.th_team;
749 status = 0;
750
751 th->th.th_ident = id_ref;
752
753 if (team->t.t_serialized) {
754 status = 1;
755 } else {
756 kmp_int32 old_this = th->th.th_local.this_construct;
757
758 ++th->th.th_local.this_construct;
759 /* try to set team count to thread count--success means thread got the
760 single block */
761 /* TODO: Should this be acquire or release? */
762 if (team->t.t_construct == old_this) {
763 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
764 th->th.th_local.this_construct);
765 }
766#if USE_ITT_BUILD1
767 if (__itt_metadata_add_ptr__kmp_itt_metadata_add_ptr__3_0 && __kmp_forkjoin_frames_mode == 3 &&
768 KMP_MASTER_GTID(gtid)((((((gtid)) >= 0) ? 0 : __kmp_debug_assert("((gtid)) >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 768)), __kmp_threads[((gtid))]->th.th_info.ds.ds_tid) ==
0)
&&
769#if OMP_40_ENABLED(50 >= 40)
770 th->th.th_teams_microtask == NULL__null &&
771#endif
772 team->t.t_active_level ==
773 1) { // Only report metadata by master of active team at level 1
774 __kmp_itt_metadata_single(id_ref);
775 }
776#endif /* USE_ITT_BUILD */
777 }
778
779 if (__kmp_env_consistency_check) {
780 if (status && push_ws) {
781 __kmp_push_workshare(gtid, ct_psingle, id_ref);
782 } else {
783 __kmp_check_workshare(gtid, ct_psingle, id_ref);
784 }
785 }
786#if USE_ITT_BUILD1
787 if (status) {
788 __kmp_itt_single_start(gtid);
789 }
790#endif /* USE_ITT_BUILD */
791 return status;
792}
793
794void __kmp_exit_single(int gtid) {
795#if USE_ITT_BUILD1
796 __kmp_itt_single_end(gtid);
797#endif /* USE_ITT_BUILD */
798 if (__kmp_env_consistency_check)
799 __kmp_pop_workshare(gtid, ct_psingle, NULL__null);
800}
801
802/* determine if we can go parallel or must use a serialized parallel region and
803 * how many threads we can use
804 * set_nproc is the number of threads requested for the team
805 * returns 0 if we should serialize or only use one thread,
806 * otherwise the number of threads to use
807 * The forkjoin lock is held by the caller. */
808static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
809 int master_tid, int set_nthreads
810#if OMP_40_ENABLED(50 >= 40)
811 ,
812 int enter_teams
813#endif /* OMP_40_ENABLED */
814 ) {
815 int capacity;
816 int new_nthreads;
817 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 817))
;
818 KMP_DEBUG_ASSERT(root && parent_team)((root && parent_team) ? 0 : __kmp_debug_assert("root && parent_team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 818))
;
819
820 // If dyn-var is set, dynamically adjust the number of desired threads,
821 // according to the method specified by dynamic_mode.
822 new_nthreads = set_nthreads;
823 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
) {
824 ;
825 }
826#ifdef USE_LOAD_BALANCE1
827 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
828 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
829 if (new_nthreads == 1) {
830 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
831 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
832 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to 1 thread\n", master_tid); }
;
833 return 1;
834 }
835 if (new_nthreads < set_nthreads) {
836 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
837 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
838 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
839 }
840 }
841#endif /* USE_LOAD_BALANCE */
842 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
843 new_nthreads = __kmp_avail_proc - __kmp_nth +
844 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
845 if (new_nthreads <= 1) {
846 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
847 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
848 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to 1 thread\n", master_tid); }
;
849 return 1;
850 }
851 if (new_nthreads < set_nthreads) {
852 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
853 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
854 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
855 } else {
856 new_nthreads = set_nthreads;
857 }
858 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
859 if (set_nthreads > 2) {
860 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
861 new_nthreads = (new_nthreads % set_nthreads) + 1;
862 if (new_nthreads == 1) {
863 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
864 "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
865 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to 1 thread\n", master_tid); }
;
866 return 1;
867 }
868 if (new_nthreads < set_nthreads) {
869 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
870 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
871 master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
"reservation to %d threads\n", master_tid, new_nthreads); }
;
872 }
873 }
874 } else {
875 KMP_ASSERT(0)((0) ? 0 : __kmp_debug_assert("0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 875))
;
876 }
877
878 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
879 if (__kmp_nth + new_nthreads -
880 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
881 __kmp_max_nth) {
882 int tl_nthreads = __kmp_max_nth - __kmp_nth +
883 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
884 if (tl_nthreads <= 0) {
885 tl_nthreads = 1;
886 }
887
888 // If dyn-var is false, emit a 1-time warning.
889 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
890 __kmp_reserve_warn = 1;
891 __kmp_msg(kmp_ms_warning,
892 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
)
,
893 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
894 }
895 if (tl_nthreads == 1) {
896 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
897 "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
898 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
;
899 return 1;
900 }
901 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
902 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
903 master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
;
904 new_nthreads = tl_nthreads;
905 }
906
907 // Respect OMP_THREAD_LIMIT
908 if (root->r.r_cg_nthreads + new_nthreads -
909 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
910 __kmp_cg_max_nth) {
911 int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads +
912 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
913 if (tl_nthreads <= 0) {
914 tl_nthreads = 1;
915 }
916
917 // If dyn-var is false, emit a 1-time warning.
918 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
919 __kmp_reserve_warn = 1;
920 __kmp_msg(kmp_ms_warning,
921 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
)
,
922 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
923 }
924 if (tl_nthreads == 1) {
925 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
926 "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
927 master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
"reduced reservation to 1 thread\n", master_tid); }
;
928 return 1;
929 }
930 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
931 "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
932 master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
"reservation to %d threads\n", master_tid, tl_nthreads); }
;
933 new_nthreads = tl_nthreads;
934 }
935
936 // Check if the threads array is large enough, or needs expanding.
937 // See comment in __kmp_register_root() about the adjustment if
938 // __kmp_threads[0] == NULL.
939 capacity = __kmp_threads_capacity;
940 if (TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
941 --capacity;
942 }
943 if (__kmp_nth + new_nthreads -
944 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
945 capacity) {
946 // Expand the threads array.
947 int slotsRequired = __kmp_nth + new_nthreads -
948 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
949 capacity;
950 int slotsAdded = __kmp_expand_threads(slotsRequired);
951 if (slotsAdded < slotsRequired) {
952 // The threads array was not expanded enough.
953 new_nthreads -= (slotsRequired - slotsAdded);
954 KMP_ASSERT(new_nthreads >= 1)((new_nthreads >= 1) ? 0 : __kmp_debug_assert("new_nthreads >= 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 954))
;
955
956 // If dyn-var is false, emit a 1-time warning.
957 if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)
&& (!__kmp_reserve_warn)) {
958 __kmp_reserve_warn = 1;
959 if (__kmp_tp_cached) {
960 __kmp_msg(kmp_ms_warning,
961 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
)
,
962 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
)
,
963 KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
964 } else {
965 __kmp_msg(kmp_ms_warning,
966 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
)
,
967 KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads), __kmp_msg_null);
968 }
969 }
970 }
971 }
972
973#ifdef KMP_DEBUG1
974 if (new_nthreads == 1) {
975 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
976 ("__kmp_reserve_threads: T#%d serializing team after reclaiming "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
977 "dead roots and rechecking; requested %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
978 __kmp_get_gtid(), set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
"dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
;
979 } else {
980 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
981 " %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
982 __kmp_get_gtid(), new_nthreads, set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
" %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
set_nthreads); }
;
983 }
984#endif // KMP_DEBUG
985 return new_nthreads;
986}
987
988/* Allocate threads from the thread pool and assign them to the new team. We are
989 assured that there are enough threads available, because we checked on that
990 earlier within critical section forkjoin */
991static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
992 kmp_info_t *master_th, int master_gtid) {
993 int i;
994 int use_hot_team;
995
996 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_fork_team_threads: new_nprocs = %d\n"
, team->t.t_nproc); }
;
997 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid())((master_gtid == __kmp_get_global_thread_id()) ? 0 : __kmp_debug_assert
("master_gtid == __kmp_get_global_thread_id()", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 997))
;
998 KMP_MB();
999
1000 /* first, let's setup the master thread */
1001 master_th->th.th_info.ds.ds_tid = 0;
1002 master_th->th.th_team = team;
1003 master_th->th.th_team_nproc = team->t.t_nproc;
1004 master_th->th.th_team_master = master_th;
1005 master_th->th.th_team_serialized = FALSE0;
1006 master_th->th.th_dispatch = &team->t.t_dispatch[0];
1007
1008/* make sure we are not the optimized hot team */
1009#if KMP_NESTED_HOT_TEAMS1
1010 use_hot_team = 0;
1011 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
1012 if (hot_teams) { // hot teams array is not allocated if
1013 // KMP_HOT_TEAMS_MAX_LEVEL=0
1014 int level = team->t.t_active_level - 1; // index in array of hot teams
1015 if (master_th->th.th_teams_microtask) { // are we inside the teams?
1016 if (master_th->th.th_teams_size.nteams > 1) {
1017 ++level; // level was not increased in teams construct for
1018 // team_of_masters
1019 }
1020 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1021 master_th->th.th_teams_level == team->t.t_level) {
1022 ++level; // level was not increased in teams construct for
1023 // team_of_workers before the parallel
1024 } // team->t.t_level will be increased inside parallel
1025 }
1026 if (level < __kmp_hot_teams_max_level) {
1027 if (hot_teams[level].hot_team) {
1028 // hot team has already been allocated for given level
1029 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team)((hot_teams[level].hot_team == team) ? 0 : __kmp_debug_assert
("hot_teams[level].hot_team == team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1029))
;
1030 use_hot_team = 1; // the team is ready to use
1031 } else {
1032 use_hot_team = 0; // AC: threads are not allocated yet
1033 hot_teams[level].hot_team = team; // remember new hot team
1034 hot_teams[level].hot_team_nth = team->t.t_nproc;
1035 }
1036 } else {
1037 use_hot_team = 0;
1038 }
1039 }
1040#else
1041 use_hot_team = team == root->r.r_hot_team;
1042#endif
1043 if (!use_hot_team) {
1044
1045 /* install the master thread */
1046 team->t.t_threads[0] = master_th;
1047 __kmp_initialize_info(master_th, team, 0, master_gtid);
1048
1049 /* now, install the worker threads */
1050 for (i = 1; i < team->t.t_nproc; i++) {
1051
1052 /* fork or reallocate a new thread and install it in team */
1053 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1054 team->t.t_threads[i] = thr;
1055 KMP_DEBUG_ASSERT(thr)((thr) ? 0 : __kmp_debug_assert("thr", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1055))
;
1056 KMP_DEBUG_ASSERT(thr->th.th_team == team)((thr->th.th_team == team) ? 0 : __kmp_debug_assert("thr->th.th_team == team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1056))
;
1057 /* align team and thread arrived states */
1058 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", ((((0) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1060)), team->t.t_threads[(0)]->th.th_info.ds.ds_gtid
), team->t.t_id, 0, ((((i) >= 0 && (team) != __null
) ? 0 : __kmp_debug_assert("(i) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1061)), team->t.t_threads[(i)]->th.th_info.ds.ds_gtid
), team->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
1059 "T#%d(%d:%d) join =%llu, plain=%llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", ((((0) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1060)), team->t.t_threads[(0)]->th.th_info.ds.ds_gtid
), team->t.t_id, 0, ((((i) >= 0 && (team) != __null
) ? 0 : __kmp_debug_assert("(i) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1061)), team->t.t_threads[(i)]->th.th_info.ds.ds_gtid
), team->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
1060 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", ((((0) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1060)), team->t.t_threads[(0)]->th.th_info.ds.ds_gtid
), team->t.t_id, 0, ((((i) >= 0 && (team) != __null
) ? 0 : __kmp_debug_assert("(i) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1061)), team->t.t_threads[(i)]->th.th_info.ds.ds_gtid
), team->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
1061 __kmp_gtid_from_tid(i, team), team->t.t_id, i,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", ((((0) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1060)), team->t.t_threads[(0)]->th.th_info.ds.ds_gtid
), team->t.t_id, 0, ((((i) >= 0 && (team) != __null
) ? 0 : __kmp_debug_assert("(i) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1061)), team->t.t_threads[(i)]->th.th_info.ds.ds_gtid
), team->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
1062 team->t.t_bar[bs_forkjoin_barrier].b_arrived,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", ((((0) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1060)), team->t.t_threads[(0)]->th.th_info.ds.ds_gtid
), team->t.t_id, 0, ((((i) >= 0 && (team) != __null
) ? 0 : __kmp_debug_assert("(i) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1061)), team->t.t_threads[(i)]->th.th_info.ds.ds_gtid
), team->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
1063 team->t.t_bar[bs_plain_barrier].b_arrived))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
"T#%d(%d:%d) join =%llu, plain=%llu\n", ((((0) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1060)), team->t.t_threads[(0)]->th.th_info.ds.ds_gtid
), team->t.t_id, 0, ((((i) >= 0 && (team) != __null
) ? 0 : __kmp_debug_assert("(i) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1061)), team->t.t_threads[(i)]->th.th_info.ds.ds_gtid
), team->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
;
1064#if OMP_40_ENABLED(50 >= 40)
1065 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1066 thr->th.th_teams_level = master_th->th.th_teams_level;
1067 thr->th.th_teams_size = master_th->th.th_teams_size;
1068#endif
1069 { // Initialize threads' barrier data.
1070 int b;
1071 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1072 for (b = 0; b < bs_last_barrier; ++b) {
1073 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1074 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)((balign[b].bb.wait_flag != 2) ? 0 : __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1074))
;
1075#if USE_DEBUGGER0
1076 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1077#endif
1078 }
1079 }
1080 }
1081
1082#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
1083 __kmp_partition_places(team);
1084#endif
1085 }
1086
1087 KMP_MB();
1088}
1089
1090#if KMP_ARCH_X860 || KMP_ARCH_X86_641
1091// Propagate any changes to the floating point control registers out to the team
1092// We try to avoid unnecessary writes to the relevant cache line in the team
1093// structure, so we don't make changes unless they are needed.
1094inline static void propagateFPControl(kmp_team_t *team) {
1095 if (__kmp_inherit_fp_control) {
1096 kmp_int16 x87_fpu_control_word;
1097 kmp_uint32 mxcsr;
1098
1099 // Get master values of FPU control flags (both X87 and vector)
1100 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1101 __kmp_store_mxcsr(&mxcsr);
1102 mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
1103
1104 // There is no point looking at t_fp_control_saved here.
1105 // If it is TRUE, we still have to update the values if they are different
1106 // from those we now have. If it is FALSE we didn't save anything yet, but
1107 // our objective is the same. We have to ensure that the values in the team
1108 // are the same as those we have.
1109 // So, this code achieves what we need whether or not t_fp_control_saved is
1110 // true. By checking whether the value needs updating we avoid unnecessary
1111 // writes that would put the cache-line into a written state, causing all
1112 // threads in the team to have to read it again.
1113 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word)if ((team->t.t_x87_fpu_control_word) != (x87_fpu_control_word
)) (team->t.t_x87_fpu_control_word) = (x87_fpu_control_word
)
;
1114 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr)if ((team->t.t_mxcsr) != (mxcsr)) (team->t.t_mxcsr) = (
mxcsr)
;
1115 // Although we don't use this value, other code in the runtime wants to know
1116 // whether it should restore them. So we must ensure it is correct.
1117 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE)if ((team->t.t_fp_control_saved) != ((!0))) (team->t.t_fp_control_saved
) = ((!0))
;
1118 } else {
1119 // Similarly here. Don't write to this cache-line in the team structure
1120 // unless we have to.
1121 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE)if ((team->t.t_fp_control_saved) != (0)) (team->t.t_fp_control_saved
) = (0)
;
1122 }
1123}
1124
1125// Do the opposite, setting the hardware registers to the updated values from
1126// the team.
1127inline static void updateHWFPControl(kmp_team_t *team) {
1128 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1129 // Only reset the fp control regs if they have been changed in the team.
1130 // the parallel region that we are exiting.
1131 kmp_int16 x87_fpu_control_word;
1132 kmp_uint32 mxcsr;
1133 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1134 __kmp_store_mxcsr(&mxcsr);
1135 mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
1136
1137 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1138 __kmp_clear_x87_fpu_status_word();
1139 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1140 }
1141
1142 if (team->t.t_mxcsr != mxcsr) {
1143 __kmp_load_mxcsr(&team->t.t_mxcsr)_mm_setcsr(*(&team->t.t_mxcsr));
1144 }
1145 }
1146}
1147#else
1148#define propagateFPControl(x) ((void)0)
1149#define updateHWFPControl(x) ((void)0)
1150#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1151
1152static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1153 int realloc); // forward declaration
1154
1155/* Run a parallel region that has been serialized, so runs only in a team of the
1156 single master thread. */
1157void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
1158 kmp_info_t *this_thr;
1159 kmp_team_t *serial_team;
1160
1161 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: called by T#%d\n"
, global_tid); }
;
1162
1163 /* Skip all this code for autopar serialized loops since it results in
1164 unacceptable overhead */
1165 if (loc != NULL__null && (loc->flags & KMP_IDENT_AUTOPAR0x08))
1166 return;
1167
1168 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
1169 __kmp_parallel_initialize();
1170
1171 this_thr = __kmp_threads[global_tid];
1172 serial_team = this_thr->th.th_serial_team;
1173
1174 /* utilize the serialized team held by this thread */
1175 KMP_DEBUG_ASSERT(serial_team)((serial_team) ? 0 : __kmp_debug_assert("serial_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1175))
;
1176 KMP_MB();
1177
1178 if (__kmp_tasking_mode != tskm_immediate_exec) {
1179 KMP_DEBUG_ASSERT(((this_thr->th.th_task_team == this_thr->th.th_team->
t.t_task_team[this_thr->th.th_task_state]) ? 0 : __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1181))
1180 this_thr->th.th_task_team ==((this_thr->th.th_task_team == this_thr->th.th_team->
t.t_task_team[this_thr->th.th_task_state]) ? 0 : __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1181))
1181 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state])((this_thr->th.th_task_team == this_thr->th.th_team->
t.t_task_team[this_thr->th.th_task_state]) ? 0 : __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1181))
;
1182 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==((serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null) ? 0 : __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1183))
1183 NULL)((serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null) ? 0 : __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1183))
;
1184 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
1185 "team %p, new task_team = NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
1186 global_tid, this_thr->th.th_task_team, this_thr->th.th_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
"team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
;
1187 this_thr->th.th_task_team = NULL__null;
1188 }
1189
1190#if OMP_40_ENABLED(50 >= 40)
1191 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1192 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1193 proc_bind = proc_bind_false;
1194 } else if (proc_bind == proc_bind_default) {
1195 // No proc_bind clause was specified, so use the current value
1196 // of proc-bind-var for this parallel region.
1197 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1198 }
1199 // Reset for next parallel region
1200 this_thr->th.th_set_proc_bind = proc_bind_default;
1201#endif /* OMP_40_ENABLED */
1202
1203#if OMPT_SUPPORT1
1204 ompt_data_t ompt_parallel_data;
1205 ompt_parallel_data.ptr = NULL__null;
1206 ompt_data_t *implicit_task_data;
1207 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid)__ompt_load_return_address(global_tid);
1208 if (ompt_enabled.enabled &&
1209 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
1210
1211 ompt_task_info_t *parent_task_info;
1212 parent_task_info = OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info));
1213
1214 parent_task_info->frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1215 if (ompt_enabled.ompt_callback_parallel_begin) {
1216 int team_size = 1;
1217
1218 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
1219 &(parent_task_info->task_data), &(parent_task_info->frame),
1220 &ompt_parallel_data, team_size, ompt_invoker_program, codeptr);
1221 }
1222 }
1223#endif // OMPT_SUPPORT
1224
1225 if (this_thr->th.th_team != serial_team) {
1226 // Nested level will be an index in the nested nthreads array
1227 int level = this_thr->th.th_team->t.t_level;
1228
1229 if (serial_team->t.t_serialized) {
1230 /* this serial team was already used
1231 TODO increase performance by making this locks more specific */
1232 kmp_team_t *new_team;
1233
1234 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1235
1236 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1237#if OMPT_SUPPORT1
1238 ompt_parallel_data,
1239#endif
1240#if OMP_40_ENABLED(50 >= 40)
1241 proc_bind,
1242#endif
1243 &this_thr->th.th_current_task->td_icvs,
1244 0 USE_NESTED_HOT_ARG(NULL), __null);
1245 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1246 KMP_ASSERT(new_team)((new_team) ? 0 : __kmp_debug_assert("new_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1246))
;
1247
1248 /* setup new serialized team and install it */
1249 new_team->t.t_threads[0] = this_thr;
1250 new_team->t.t_parent = this_thr->th.th_team;
1251 serial_team = new_team;
1252 this_thr->th.th_serial_team = serial_team;
1253
1254 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1255 10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1256 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
1257 global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
;
1258
1259 /* TODO the above breaks the requirement that if we run out of resources,
1260 then we can still guarantee that serialized teams are ok, since we may
1261 need to allocate a new one */
1262 } else {
1263 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1264 10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1265 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
1266 global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
;
1267 }
1268
1269 /* we have to initialize this serial team */
1270 KMP_DEBUG_ASSERT(serial_team->t.t_threads)((serial_team->t.t_threads) ? 0 : __kmp_debug_assert("serial_team->t.t_threads"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1270))
;
1271 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)((serial_team->t.t_threads[0] == this_thr) ? 0 : __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1271))
;
1272 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team)((this_thr->th.th_team != serial_team) ? 0 : __kmp_debug_assert
("this_thr->th.th_team != serial_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1272))
;
1273 serial_team->t.t_ident = loc;
1274 serial_team->t.t_serialized = 1;
1275 serial_team->t.t_nproc = 1;
1276 serial_team->t.t_parent = this_thr->th.th_team;
1277 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1278 this_thr->th.th_team = serial_team;
1279 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1280
1281 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); }
1282 this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); }
;
1283 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1)((this_thr->th.th_current_task->td_flags.executing == 1
) ? 0 : __kmp_debug_assert("this_thr->th.th_current_task->td_flags.executing == 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1283))
;
1284 this_thr->th.th_current_task->td_flags.executing = 0;
1285
1286 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1287
1288 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
1289 implicit task for each serialized task represented by
1290 team->t.t_serialized? */
1291 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1292 &this_thr->th.th_current_task->td_parent->td_icvs);
1293
1294 // Thread value exists in the nested nthreads array for the next nested
1295 // level
1296 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1297 this_thr->th.th_current_task->td_icvs.nproc =
1298 __kmp_nested_nth.nth[level + 1];
1299 }
1300
1301#if OMP_40_ENABLED(50 >= 40)
1302 if (__kmp_nested_proc_bind.used &&
1303 (level + 1 < __kmp_nested_proc_bind.used)) {
1304 this_thr->th.th_current_task->td_icvs.proc_bind =
1305 __kmp_nested_proc_bind.bind_types[level + 1];
1306 }
1307#endif /* OMP_40_ENABLED */
1308
1309#if USE_DEBUGGER0
1310 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
1311#endif
1312 this_thr->th.th_info.ds.ds_tid = 0;
1313
1314 /* set thread cache values */
1315 this_thr->th.th_team_nproc = 1;
1316 this_thr->th.th_team_master = this_thr;
1317 this_thr->th.th_team_serialized = 1;
1318
1319 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1320 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1321
1322 propagateFPControl(serial_team);
1323
1324 /* check if we need to allocate dispatch buffers stack */
1325 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)((serial_team->t.t_dispatch) ? 0 : __kmp_debug_assert("serial_team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1325))
;
1326 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1327 serial_team->t.t_dispatch->th_disp_buffer =
1328 (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1329)
1329 sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1329)
;
1330 }
1331 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1332
1333 KMP_MB();
1334
1335 } else {
1336 /* this serialized team is already being used,
1337 * that's fine, just add another nested level */
1338 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team)((this_thr->th.th_team == serial_team) ? 0 : __kmp_debug_assert
("this_thr->th.th_team == serial_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1338))
;
1339 KMP_DEBUG_ASSERT(serial_team->t.t_threads)((serial_team->t.t_threads) ? 0 : __kmp_debug_assert("serial_team->t.t_threads"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1339))
;
1340 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)((serial_team->t.t_threads[0] == this_thr) ? 0 : __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1340))
;
1341 ++serial_team->t.t_serialized;
1342 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1343
1344 // Nested level will be an index in the nested nthreads array
1345 int level = this_thr->th.th_team->t.t_level;
1346 // Thread value exists in the nested nthreads array for the next nested
1347 // level
1348 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1349 this_thr->th.th_current_task->td_icvs.nproc =
1350 __kmp_nested_nth.nth[level + 1];
1351 }
1352 serial_team->t.t_level++;
1353 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
1354 "of serial team %p to %d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
1355 global_tid, serial_team, serial_team->t.t_level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
"of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
;
1356
1357 /* allocate/push dispatch buffers stack */
1358 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)((serial_team->t.t_dispatch) ? 0 : __kmp_debug_assert("serial_team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1358))
;
1359 {
1360 dispatch_private_info_t *disp_buffer =
1361 (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1362)
1362 sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1362)
;
1363 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1364 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1365 }
1366 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1367
1368 KMP_MB();
1369 }
1370#if OMP_40_ENABLED(50 >= 40)
1371 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq)if ((serial_team->t.t_cancel_request) != (cancel_noreq)) (
serial_team->t.t_cancel_request) = (cancel_noreq)
;
1372#endif
1373
1374 if (__kmp_env_consistency_check)
1375 __kmp_push_parallel(global_tid, NULL__null);
1376#if OMPT_SUPPORT1
1377 serial_team->t.ompt_team_info.master_return_address = codeptr;
1378 if (ompt_enabled.enabled &&
1379 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
1380 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1381
1382 ompt_lw_taskteam_t lw_taskteam;
1383 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1384 &ompt_parallel_data, codeptr);
1385
1386 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1387 // don't use lw_taskteam after linking. content was swaped
1388
1389 /* OMPT implicit task begin */
1390 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
))
;
1391 if (ompt_enabled.ompt_callback_implicit_task) {
1392 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1393 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr)(&(this_thr->th.th_team->t.ompt_team_info.parallel_data
))
,
1394 OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
))
, 1, __kmp_tid_from_gtid(global_tid)((((global_tid) >= 0) ? 0 : __kmp_debug_assert("(global_tid) >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1394)), __kmp_threads[(global_tid)]->th.th_info.ds.ds_tid
)
);
1395 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))
1396 ->thread_num = __kmp_tid_from_gtid(global_tid)((((global_tid) >= 0) ? 0 : __kmp_debug_assert("(global_tid) >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1396)), __kmp_threads[(global_tid)]->th.th_info.ds.ds_tid
)
;
1397 }
1398
1399 /* OMPT state */
1400 this_thr->th.ompt_thread_info.state = omp_state_work_parallel;
1401 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1402 }
1403#endif
1404}
1405
1406/* most of the work for a fork */
1407/* return true if we really went parallel, false if serialized */
1408int __kmp_fork_call(ident_t *loc, int gtid,
1409 enum fork_context_e call_context, // Intel, GNU, ...
1410 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1411/* TODO: revert workaround for Intel(R) 64 tracker #96 */
1412#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1413 va_list *ap
1414#else
1415 va_list ap
1416#endif
1417 ) {
1418 void **argv;
1419 int i;
1420 int master_tid;
1421 int master_this_cons;
1422 kmp_team_t *team;
1423 kmp_team_t *parent_team;
1424 kmp_info_t *master_th;
1425 kmp_root_t *root;
1426 int nthreads;
1427 int master_active;
1428 int master_set_numthreads;
1429 int level;
1430#if OMP_40_ENABLED(50 >= 40)
1431 int active_level;
1432 int teams_level;
1433#endif
1434#if KMP_NESTED_HOT_TEAMS1
1435 kmp_hot_team_ptr_t **p_hot_teams;
1436#endif
1437 { // KMP_TIME_BLOCK
1438 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call)((void)0);
1439 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc)((void)0);
1440
1441 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: enter T#%d\n"
, gtid); }
;
1442 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL__null) {
1
Assuming '__kmp_stkpadding' is <= 0
1443 /* Some systems prefer the stack for the root thread(s) to start with */
1444 /* some gap from the parent stack to prevent false sharing. */
1445 void *dummy = KMP_ALLOCA(__kmp_stkpadding)__builtin_alloca (__kmp_stkpadding);
1446 /* These 2 lines below are so this does not get optimized out */
1447 if (__kmp_stkpadding > KMP_MAX_STKPADDING(2 * 1024 * 1024))
1448 __kmp_stkpadding += (short)((kmp_int64)dummy);
1449 }
1450
1451 /* initialize if needed */
1452 KMP_DEBUG_ASSERT(((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1453))
1453 __kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1453))
; // AC: potentially unsafe, not in sync with shutdown
1454 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
2
Assuming '__kmp_init_parallel' is not equal to 0
3
Taking false branch
1455 __kmp_parallel_initialize();
1456
1457 /* setup current data */
1458 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
1459 // shutdown
1460 parent_team = master_th->th.th_team;
1461 master_tid = master_th->th.th_info.ds.ds_tid;
1462 master_this_cons = master_th->th.th_local.this_construct;
1463 root = master_th->th.th_root;
1464 master_active = root->r.r_active;
1465 master_set_numthreads = master_th->th.th_set_nproc;
1466
1467#if OMPT_SUPPORT1
1468 ompt_data_t ompt_parallel_data;
1469 ompt_parallel_data.ptr = NULL__null;
1470 ompt_data_t *parent_task_data;
1471 omp_frame_t *ompt_frame;
1472 ompt_data_t *implicit_task_data;
1473 void *return_address = NULL__null;
1474
1475 if (ompt_enabled.enabled) {
4
Assuming the condition is false
5
Taking false branch
1476 __ompt_get_task_info_internal(0, NULL__null, &parent_task_data, &ompt_frame,
1477 NULL__null, NULL__null);
1478 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid);
1479 }
1480#endif
1481
1482 // Nested level will be an index in the nested nthreads array
1483 level = parent_team->t.t_level;
1484 // used to launch non-serial teams even if nested is not allowed
1485 active_level = parent_team->t.t_active_level;
1486#if OMP_40_ENABLED(50 >= 40)
1487 // needed to check nesting inside the teams
1488 teams_level = master_th->th.th_teams_level;
1489#endif
1490#if KMP_NESTED_HOT_TEAMS1
1491 p_hot_teams = &master_th->th.th_hot_teams;
1492 if (*p_hot_teams == NULL__null && __kmp_hot_teams_max_level > 0) {
6
Assuming the condition is false
1493 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1494)
1494 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level)___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1494)
;
1495 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1496 // it is either actual or not needed (when active_level > 0)
1497 (*p_hot_teams)[0].hot_team_nth = 1;
1498 }
1499#endif
1500
1501#if OMPT_SUPPORT1
1502 if (ompt_enabled.enabled) {
7
Taking false branch
1503 if (ompt_enabled.ompt_callback_parallel_begin) {
1504 int team_size = master_set_numthreads
1505 ? master_set_numthreads
1506 : get__nproc_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
;
1507 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
1508 parent_task_data, ompt_frame, &ompt_parallel_data, team_size,
1509 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_invoker_program : ompt_invoker_runtime
)
, return_address);
1510 }
1511 master_th->th.ompt_thread_info.state = omp_state_overhead;
1512 }
1513#endif
1514
1515 master_th->th.th_ident = loc;
1516
1517#if OMP_40_ENABLED(50 >= 40)
1518 if (master_th->th.th_teams_microtask && ap &&
1519 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1520 // AC: This is start of parallel that is nested inside teams construct.
1521 // The team is actual (hot), all workers are ready at the fork barrier.
1522 // No lock needed to initialize the team a bit, then free workers.
1523 parent_team->t.t_ident = loc;
1524 __kmp_alloc_argv_entries(argc, parent_team, TRUE(!0));
1525 parent_team->t.t_argc = argc;
1526 argv = (void **)parent_team->t.t_argv;
1527 for (i = argc - 1; i >= 0; --i)
1528/* TODO: revert workaround for Intel(R) 64 tracker #96 */
1529#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1530 *argv++ = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
1531#else
1532 *argv++ = va_arg(ap, void *)__builtin_va_arg(ap, void *);
1533#endif
1534 // Increment our nested depth levels, but not increase the serialization
1535 if (parent_team == master_th->th.th_serial_team) {
1536 // AC: we are in serialized parallel
1537 __kmpc_serialized_parallel(loc, gtid);
1538 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1)((parent_team->t.t_serialized > 1) ? 0 : __kmp_debug_assert
("parent_team->t.t_serialized > 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1538))
;
1539 // AC: need this in order enquiry functions work
1540 // correctly, will restore at join time
1541 parent_team->t.t_serialized--;
1542#if OMPT_SUPPORT1
1543 void *dummy;
1544 void **exit_runtime_p;
1545
1546 ompt_lw_taskteam_t lw_taskteam;
1547
1548 if (ompt_enabled.enabled) {
1549 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1550 &ompt_parallel_data, return_address);
1551 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame);
1552
1553 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1554 // don't use lw_taskteam after linking. content was swaped
1555
1556 /* OMPT implicit task begin */
1557 implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
;
1558 if (ompt_enabled.ompt_callback_implicit_task) {
1559 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1560 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1561 implicit_task_data, 1, __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1561)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
);
1562 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))
1563 ->thread_num = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1563)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
1564 }
1565
1566 /* OMPT state */
1567 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
1568 } else {
1569 exit_runtime_p = &dummy;
1570 }
1571#endif
1572
1573 {
1574 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1575 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1576 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1577#if OMPT_SUPPORT1
1578 ,
1579 exit_runtime_p
1580#endif
1581 );
1582 }
1583
1584#if OMPT_SUPPORT1
1585 *exit_runtime_p = NULL__null;
1586 if (ompt_enabled.enabled) {
1587 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->frame.exit_frame = NULL__null;
1588 if (ompt_enabled.ompt_callback_implicit_task) {
1589 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1590 ompt_scope_end, NULL__null, implicit_task_data, 1,
1591 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
1592 }
1593 __ompt_lw_taskteam_unlink(master_th);
1594
1595 if (ompt_enabled.ompt_callback_parallel_end) {
1596 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1597 OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
, OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
,
1598 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_invoker_program : ompt_invoker_runtime
)
, return_address);
1599 }
1600 master_th->th.ompt_thread_info.state = omp_state_overhead;
1601 }
1602#endif
1603 return TRUE(!0);
1604 }
1605
1606 parent_team->t.t_pkfn = microtask;
1607 parent_team->t.t_invoke = invoker;
1608 KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
)
;
1609 parent_team->t.t_active_level++;
1610 parent_team->t.t_level++;
1611
1612 /* Change number of threads in the team if requested */
1613 if (master_set_numthreads) { // The parallel has num_threads clause
1614 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1615 // AC: only can reduce number of threads dynamically, can't increase
1616 kmp_info_t **other_threads = parent_team->t.t_threads;
1617 parent_team->t.t_nproc = master_set_numthreads;
1618 for (i = 0; i < master_set_numthreads; ++i) {
1619 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1620 }
1621 // Keep extra threads hot in the team for possible next parallels
1622 }
1623 master_th->th.th_set_nproc = 0;
1624 }
1625
1626#if USE_DEBUGGER0
1627 if (__kmp_debugging) { // Let debugger override number of threads.
1628 int nth = __kmp_omp_num_threads(loc);
1629 if (nth > 0) { // 0 means debugger doesn't want to change num threads
1630 master_set_numthreads = nth;
1631 }
1632 }
1633#endif
1634
1635 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1636 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1637 root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
;
1638 __kmp_internal_fork(loc, gtid, parent_team);
1639 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1640 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
1641 root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
;
1642
1643 /* Invoke microtask for MASTER thread */
1644 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
1645 parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
;
1646
1647 {
1648 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1649 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1650 if (!parent_team->t.t_invoke(gtid)) {
1651 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread")((0) ? 0 : __kmp_debug_assert(("cannot invoke microtask for MASTER thread"
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1651))
;
1652 }
1653 }
1654 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
1655 parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
;
1656 KMP_MB(); /* Flush all pending memory write invalidates. */
1657
1658 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
1659
1660 return TRUE(!0);
1661 } // Parallel closely nested in teams construct
1662#endif /* OMP_40_ENABLED */
1663
1664#if KMP_DEBUG1
1665 if (__kmp_tasking_mode != tskm_immediate_exec) {
8
Assuming '__kmp_tasking_mode' is equal to tskm_immediate_exec
9
Taking false branch
1666 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==((master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state]) ? 0 : __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1667))
1667 parent_team->t.t_task_team[master_th->th.th_task_state])((master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state]) ? 0 : __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1667))
;
1668 }
1669#endif
1670
1671 if (parent_team->t.t_active_level >=
10
Taking true branch
1672 master_th->th.th_current_task->td_icvs.max_active_levels) {
1673 nthreads = 1;
1674 } else {
1675#if OMP_40_ENABLED(50 >= 40)
1676 int enter_teams = ((ap == NULL__null && active_level == 0) ||
1677 (ap && teams_level > 0 && teams_level == level));
1678#endif
1679 nthreads =
1680 master_set_numthreads
1681 ? master_set_numthreads
1682 : get__nproc_2(((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
1683 parent_team,((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
1684 master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc)
; // TODO: get nproc directly from current task
1685
1686 // Check if we need to take forkjoin lock? (no need for serialized
1687 // parallel out of teams construct). This code moved here from
1688 // __kmp_reserve_threads() to speedup nested serialized parallels.
1689 if (nthreads > 1) {
1690 if ((!get__nested(master_th)(((master_th)->th.th_current_task->td_icvs.nested) ? ((
!0)) : (0))
&& (root->r.r_in_parallel
1691#if OMP_40_ENABLED(50 >= 40)
1692 && !enter_teams
1693#endif /* OMP_40_ENABLED */
1694 )) ||
1695 (__kmp_library == library_serial)) {
1696 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
1697 " threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
1698 gtid, nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team; requested %d"
" threads\n", gtid, nthreads); }
;
1699 nthreads = 1;
1700 }
1701 }
1702 if (nthreads > 1) {
1703 /* determine how many new threads we can use */
1704 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1705 nthreads = __kmp_reserve_threads(
1706 root, parent_team, master_tid, nthreads
1707#if OMP_40_ENABLED(50 >= 40)
1708 /* AC: If we execute teams from parallel region (on host), then
1709 teams should be created but each can only have 1 thread if
1710 nesting is disabled. If teams called from serial region, then
1711 teams and their threads should be created regardless of the
1712 nesting setting. */
1713 ,
1714 enter_teams
1715#endif /* OMP_40_ENABLED */
1716 );
1717 if (nthreads == 1) {
1718 // Free lock for single thread execution here; for multi-thread
1719 // execution it will be freed later after team of threads created
1720 // and initialized
1721 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1722 }
1723 }
1724 }
1725 KMP_DEBUG_ASSERT(nthreads > 0)((nthreads > 0) ? 0 : __kmp_debug_assert("nthreads > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1725))
;
1726
1727 // If we temporarily changed the set number of threads then restore it now
1728 master_th->th.th_set_nproc = 0;
1729
1730 /* create a serialized parallel region? */
1731 if (nthreads == 1) {
11
Taking true branch
1732/* josh todo: hypothetical question: what do we do for OS X*? */
1733#if KMP_OS_LINUX1 && \
1734 (KMP_ARCH_X860 || KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640)
1735 void *args[argc];
1736#else
1737 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *))__builtin_alloca (argc * sizeof(void *));
1738#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \
1739 KMP_ARCH_AARCH64) */
1740
1741 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing parallel region\n"
, gtid); }
1742 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing parallel region\n"
, gtid); }
;
1743
1744 __kmpc_serialized_parallel(loc, gtid);
1745
1746 if (call_context == fork_context_intel) {
12
Assuming 'call_context' is equal to fork_context_intel
13
Taking true branch
1747 /* TODO this sucks, use the compiler itself to pass args! :) */
1748 master_th->th.th_serial_team->t.t_ident = loc;
1749#if OMP_40_ENABLED(50 >= 40)
1750 if (!ap) {
14
Assuming 'ap' is null
15
Taking true branch
1751 // revert change made in __kmpc_serialized_parallel()
1752 master_th->th.th_serial_team->t.t_level--;
1753// Get args from parent team for teams construct
1754
1755#if OMPT_SUPPORT1
1756 void *dummy;
1757 void **exit_runtime_p;
1758 ompt_task_info_t *task_info;
16
'task_info' declared without an initial value
1759
1760 ompt_lw_taskteam_t lw_taskteam;
1761
1762 if (ompt_enabled.enabled) {
17
Assuming the condition is false
18
Taking false branch
1763 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1764 &ompt_parallel_data, return_address);
1765
1766 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1767 // don't use lw_taskteam after linking. content was swaped
1768
1769 task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
1770 exit_runtime_p = &(task_info->frame.exit_frame);
1771 if (ompt_enabled.ompt_callback_implicit_task) {
1772 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1773 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1774 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1774)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
);
1775 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))
1776 ->thread_num = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1776)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
1777 }
1778
1779 /* OMPT state */
1780 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
1781 } else {
1782 exit_runtime_p = &dummy;
1783 }
1784#endif
1785
1786 {
1787 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1788 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1789 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1790 parent_team->t.t_argv
1791#if OMPT_SUPPORT1
1792 ,
1793 exit_runtime_p
1794#endif
1795 );
1796 }
1797
1798#if OMPT_SUPPORT1
1799 if (ompt_enabled.enabled) {
19
Assuming the condition is true
20
Taking true branch
1800 exit_runtime_p = NULL__null;
1801 if (ompt_enabled.ompt_callback_implicit_task) {
21
Assuming the condition is true
22
Taking true branch
1802 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
23
3rd function call argument is an uninitialized value
1803 ompt_scope_end, NULL__null, &(task_info->task_data), 1,
1804 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
1805 }
1806
1807 __ompt_lw_taskteam_unlink(master_th);
1808 if (ompt_enabled.ompt_callback_parallel_end) {
1809 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1810 OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
, parent_task_data,
1811 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_invoker_program : ompt_invoker_runtime
)
, return_address);
1812 }
1813 master_th->th.ompt_thread_info.state = omp_state_overhead;
1814 }
1815#endif
1816 } else if (microtask == (microtask_t)__kmp_teams_master) {
1817 KMP_DEBUG_ASSERT(master_th->th.th_team ==((master_th->th.th_team == master_th->th.th_serial_team
) ? 0 : __kmp_debug_assert("master_th->th.th_team == master_th->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1818))
1818 master_th->th.th_serial_team)((master_th->th.th_team == master_th->th.th_serial_team
) ? 0 : __kmp_debug_assert("master_th->th.th_team == master_th->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1818))
;
1819 team = master_th->th.th_team;
1820 // team->t.t_pkfn = microtask;
1821 team->t.t_invoke = invoker;
1822 __kmp_alloc_argv_entries(argc, team, TRUE(!0));
1823 team->t.t_argc = argc;
1824 argv = (void **)team->t.t_argv;
1825 if (ap) {
1826 for (i = argc - 1; i >= 0; --i)
1827// TODO: revert workaround for Intel(R) 64 tracker #96
1828#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1829 *argv++ = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
1830#else
1831 *argv++ = va_arg(ap, void *)__builtin_va_arg(ap, void *);
1832#endif
1833 } else {
1834 for (i = 0; i < argc; ++i)
1835 // Get args from parent team for teams construct
1836 argv[i] = parent_team->t.t_argv[i];
1837 }
1838 // AC: revert change made in __kmpc_serialized_parallel()
1839 // because initial code in teams should have level=0
1840 team->t.t_level--;
1841 // AC: call special invoker for outer "parallel" of teams construct
1842 {
1843 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1844 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1845 invoker(gtid);
1846 }
1847 } else {
1848#endif /* OMP_40_ENABLED */
1849 argv = args;
1850 for (i = argc - 1; i >= 0; --i)
1851// TODO: revert workaround for Intel(R) 64 tracker #96
1852#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
1853 *argv++ = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
1854#else
1855 *argv++ = va_arg(ap, void *)__builtin_va_arg(ap, void *);
1856#endif
1857 KMP_MB();
1858
1859#if OMPT_SUPPORT1
1860 void *dummy;
1861 void **exit_runtime_p;
1862 ompt_task_info_t *task_info;
1863
1864 ompt_lw_taskteam_t lw_taskteam;
1865
1866 if (ompt_enabled.enabled) {
1867 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1868 &ompt_parallel_data, return_address);
1869 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1870 // don't use lw_taskteam after linking. content was swaped
1871 task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
1872 exit_runtime_p = &(task_info->frame.exit_frame);
1873
1874 /* OMPT implicit task begin */
1875 implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
))
;
1876 if (ompt_enabled.ompt_callback_implicit_task) {
1877 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1878 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
,
1879 implicit_task_data, 1, __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1879)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
);
1880 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))
1881 ->thread_num = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1881)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
1882 }
1883
1884 /* OMPT state */
1885 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
1886 } else {
1887 exit_runtime_p = &dummy;
1888 }
1889#endif
1890
1891 {
1892 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
1893 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
1894 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1895#if OMPT_SUPPORT1
1896 ,
1897 exit_runtime_p
1898#endif
1899 );
1900 }
1901
1902#if OMPT_SUPPORT1
1903 if (ompt_enabled.enabled) {
1904 *exit_runtime_p = NULL__null;
1905 if (ompt_enabled.ompt_callback_implicit_task) {
1906 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
1907 ompt_scope_end, NULL__null, &(task_info->task_data), 1,
1908 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
1909 }
1910
1911 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
))
;
1912 __ompt_lw_taskteam_unlink(master_th);
1913 if (ompt_enabled.ompt_callback_parallel_end) {
1914 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
1915 &ompt_parallel_data, parent_task_data,
1916 OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_invoker_program : ompt_invoker_runtime
)
, return_address);
1917 }
1918 master_th->th.ompt_thread_info.state = omp_state_overhead;
1919 }
1920#endif
1921#if OMP_40_ENABLED(50 >= 40)
1922 }
1923#endif /* OMP_40_ENABLED */
1924 } else if (call_context == fork_context_gnu) {
1925#if OMPT_SUPPORT1
1926 ompt_lw_taskteam_t lwt;
1927 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1928 return_address);
1929
1930 lwt.ompt_task_info.frame.exit_frame = NULL__null;
1931 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1932// don't use lw_taskteam after linking. content was swaped
1933#endif
1934
1935 // we were called from GNU native code
1936 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serial exit\n"
, gtid); }
;
1937 return FALSE0;
1938 } else {
1939 KMP_ASSERT2(call_context < fork_context_last,((call_context < fork_context_last) ? 0 : __kmp_debug_assert
(("__kmp_fork_call: unknown fork_context parameter"), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1940))
1940 "__kmp_fork_call: unknown fork_context parameter")((call_context < fork_context_last) ? 0 : __kmp_debug_assert
(("__kmp_fork_call: unknown fork_context parameter"), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 1940))
;
1941 }
1942
1943 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d serial exit\n"
, gtid); }
;
1944 KMP_MB();
1945 return FALSE0;
1946 }
1947
1948 // GEH: only modify the executing flag in the case when not serialized
1949 // serialized case is handled in kmpc_serialized_parallel
1950 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1951 "curtask=%p, curtask_max_aclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1952 parent_team->t.t_active_level, master_th,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1953 master_th->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
1954 master_th->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
"curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
;
1955 // TODO: GEH - cannot do this assertion because root thread not set up as
1956 // executing
1957 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
1958 master_th->th.th_current_task->td_flags.executing = 0;
1959
1960#if OMP_40_ENABLED(50 >= 40)
1961 if (!master_th->th.th_teams_microtask || level > teams_level)
1962#endif /* OMP_40_ENABLED */
1963 {
1964 /* Increment our nested depth level */
1965 KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
)
;
1966 }
1967
1968 // See if we need to make a copy of the ICVs.
1969 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
1970 if ((level + 1 < __kmp_nested_nth.used) &&
1971 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1972 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1973 } else {
1974 nthreads_icv = 0; // don't update
1975 }
1976
1977#if OMP_40_ENABLED(50 >= 40)
1978 // Figure out the proc_bind_policy for the new team.
1979 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
1980 kmp_proc_bind_t proc_bind_icv =
1981 proc_bind_default; // proc_bind_default means don't update
1982 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1983 proc_bind = proc_bind_false;
1984 } else {
1985 if (proc_bind == proc_bind_default) {
1986 // No proc_bind clause specified; use current proc-bind-var for this
1987 // parallel region
1988 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
1989 }
1990 /* else: The proc_bind policy was specified explicitly on parallel clause.
1991 This overrides proc-bind-var for this parallel region, but does not
1992 change proc-bind-var. */
1993 // Figure the value of proc-bind-var for the child threads.
1994 if ((level + 1 < __kmp_nested_proc_bind.used) &&
1995 (__kmp_nested_proc_bind.bind_types[level + 1] !=
1996 master_th->th.th_current_task->td_icvs.proc_bind)) {
1997 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
1998 }
1999 }
2000
2001 // Reset for next parallel region
2002 master_th->th.th_set_proc_bind = proc_bind_default;
2003#endif /* OMP_40_ENABLED */
2004
2005 if ((nthreads_icv > 0)
2006#if OMP_40_ENABLED(50 >= 40)
2007 || (proc_bind_icv != proc_bind_default)
2008#endif /* OMP_40_ENABLED */
2009 ) {
2010 kmp_internal_control_t new_icvs;
2011 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2012 new_icvs.next = NULL__null;
2013 if (nthreads_icv > 0) {
2014 new_icvs.nproc = nthreads_icv;
2015 }
2016
2017#if OMP_40_ENABLED(50 >= 40)
2018 if (proc_bind_icv != proc_bind_default) {
2019 new_icvs.proc_bind = proc_bind_icv;
2020 }
2021#endif /* OMP_40_ENABLED */
2022
2023 /* allocate a new parallel team */
2024 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); }
;
2025 team = __kmp_allocate_team(root, nthreads, nthreads,
2026#if OMPT_SUPPORT1
2027 ompt_parallel_data,
2028#endif
2029#if OMP_40_ENABLED(50 >= 40)
2030 proc_bind,
2031#endif
2032 &new_icvs, argc USE_NESTED_HOT_ARG(master_th), master_th);
2033 } else {
2034 /* allocate a new parallel team */
2035 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); }
;
2036 team = __kmp_allocate_team(root, nthreads, nthreads,
2037#if OMPT_SUPPORT1
2038 ompt_parallel_data,
2039#endif
2040#if OMP_40_ENABLED(50 >= 40)
2041 proc_bind,
2042#endif
2043 &master_th->th.th_current_task->td_icvs,
2044 argc USE_NESTED_HOT_ARG(master_th), master_th);
2045 }
2046 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); }
2047 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); }
;
2048
2049 /* setup the new team */
2050 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid)if ((team->t.t_master_tid) != (master_tid)) (team->t.t_master_tid
) = (master_tid)
;
2051 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons)if ((team->t.t_master_this_cons) != (master_this_cons)) (team
->t.t_master_this_cons) = (master_this_cons)
;
2052 KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
)
;
2053 KMP_CHECK_UPDATE(team->t.t_parent, parent_team)if ((team->t.t_parent) != (parent_team)) (team->t.t_parent
) = (parent_team)
;
2054 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask)if ((team->t.t_pkfn) != (microtask)) (((team->t.t_pkfn)
)) = (((microtask)))
;
2055#if OMPT_SUPPORT1
2056 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)))
2057 return_address)if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)))
;
2058#endif
2059 KMP_CHECK_UPDATE(team->t.t_invoke, invoker)if ((team->t.t_invoke) != (invoker)) (team->t.t_invoke)
= (invoker)
; // TODO move to root, maybe
2060// TODO: parent_team->t.t_level == INT_MAX ???
2061#if OMP_40_ENABLED(50 >= 40)
2062 if (!master_th->th.th_teams_microtask || level > teams_level) {
2063#endif /* OMP_40_ENABLED */
2064 int new_level = parent_team->t.t_level + 1;
2065 KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
= (new_level)
;
2066 new_level = parent_team->t.t_active_level + 1;
2067 KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level)
;
2068#if OMP_40_ENABLED(50 >= 40)
2069 } else {
2070 // AC: Do not increase parallel level at start of the teams construct
2071 int new_level = parent_team->t.t_level;
2072 KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
= (new_level)
;
2073 new_level = parent_team->t.t_active_level;
2074 KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level)
;
2075 }
2076#endif /* OMP_40_ENABLED */
2077 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.sched)
;
2078 // set master's schedule as new run-time schedule
2079 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched)if ((team->t.t_sched.sched) != (new_sched.sched)) (team->
t.t_sched.sched) = (new_sched.sched)
;
2080
2081#if OMP_40_ENABLED(50 >= 40)
2082 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq)if ((team->t.t_cancel_request) != (cancel_noreq)) (team->
t.t_cancel_request) = (cancel_noreq)
;
2083#endif
2084
2085 // Update the floating point rounding in the team if required.
2086 propagateFPControl(team);
2087
2088 if (__kmp_tasking_mode != tskm_immediate_exec) {
2089 // Set master's task team to team's task team. Unless this is hot team, it
2090 // should be NULL.
2091 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==((master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state]) ? 0 : __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2092))
2092 parent_team->t.t_task_team[master_th->th.th_task_state])((master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state]) ? 0 : __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2092))
;
2093 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", ((((master_th) != __null
) ? 0 : __kmp_debug_assert("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2095)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team, team->t.t_task_team[master_th
->th.th_task_state], team); }
2094 "%p, new task_team %p / team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", ((((master_th) != __null
) ? 0 : __kmp_debug_assert("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2095)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team, team->t.t_task_team[master_th
->th.th_task_state], team); }
2095 __kmp_gtid_from_thread(master_th),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", ((((master_th) != __null
) ? 0 : __kmp_debug_assert("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2095)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team, team->t.t_task_team[master_th
->th.th_task_state], team); }
2096 master_th->th.th_task_team, parent_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", ((((master_th) != __null
) ? 0 : __kmp_debug_assert("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2095)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team, team->t.t_task_team[master_th
->th.th_task_state], team); }
2097 team->t.t_task_team[master_th->th.th_task_state], team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
"%p, new task_team %p / team %p\n", ((((master_th) != __null
) ? 0 : __kmp_debug_assert("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2095)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team, team->t.t_task_team[master_th
->th.th_task_state], team); }
;
2098
2099 if (active_level || master_th->th.th_task_team) {
2100 // Take a memo of master's task_state
2101 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)((master_th->th.th_task_state_memo_stack) ? 0 : __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2101))
;
2102 if (master_th->th.th_task_state_top >=
2103 master_th->th.th_task_state_stack_sz) { // increase size
2104 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2105 kmp_uint8 *old_stack, *new_stack;
2106 kmp_uint32 i;
2107 new_stack = (kmp_uint8 *)__kmp_allocate(new_size)___kmp_allocate((new_size), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2107)
;
2108 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2109 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2110 }
2111 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2112 ++i) { // zero-init rest of stack
2113 new_stack[i] = 0;
2114 }
2115 old_stack = master_th->th.th_task_state_memo_stack;
2116 master_th->th.th_task_state_memo_stack = new_stack;
2117 master_th->th.th_task_state_stack_sz = new_size;
2118 __kmp_free(old_stack)___kmp_free((old_stack), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2118)
;
2119 }
2120 // Store master's task_state on stack
2121 master_th->th
2122 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2123 master_th->th.th_task_state;
2124 master_th->th.th_task_state_top++;
2125#if KMP_NESTED_HOT_TEAMS1
2126 if (team == master_th->th.th_hot_teams[active_level].hot_team) {
2127 // Restore master's nested state if nested hot team
2128 master_th->th.th_task_state =
2129 master_th->th
2130 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2131 } else {
2132#endif
2133 master_th->th.th_task_state = 0;
2134#if KMP_NESTED_HOT_TEAMS1
2135 }
2136#endif
2137 }
2138#if !KMP_NESTED_HOT_TEAMS1
2139 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||(((master_th->th.th_task_team == __null) || (team == root->
r.r_hot_team)) ? 0 : __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2140))
2140 (team == root->r.r_hot_team))(((master_th->th.th_task_team == __null) || (team == root->
r.r_hot_team)) ? 0 : __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2140))
;
2141#endif
2142 }
2143
2144 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2145 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2146 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2147 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
2148 team->t.t_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
;
2149 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||((team != root->r.r_hot_team || (team->t.t_master_tid ==
0 && (team->t.t_parent == root->r.r_root_team ||
team->t.t_parent->t.t_serialized))) ? 0 : __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2152))
2150 (team->t.t_master_tid == 0 &&((team != root->r.r_hot_team || (team->t.t_master_tid ==
0 && (team->t.t_parent == root->r.r_root_team ||
team->t.t_parent->t.t_serialized))) ? 0 : __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2152))
2151 (team->t.t_parent == root->r.r_root_team ||((team != root->r.r_hot_team || (team->t.t_master_tid ==
0 && (team->t.t_parent == root->r.r_root_team ||
team->t.t_parent->t.t_serialized))) ? 0 : __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2152))
2152 team->t.t_parent->t.t_serialized)))((team != root->r.r_hot_team || (team->t.t_master_tid ==
0 && (team->t.t_parent == root->r.r_root_team ||
team->t.t_parent->t.t_serialized))) ? 0 : __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2152))
;
2153 KMP_MB();
2154
2155 /* now, setup the arguments */
2156 argv = (void **)team->t.t_argv;
2157#if OMP_40_ENABLED(50 >= 40)
2158 if (ap) {
2159#endif /* OMP_40_ENABLED */
2160 for (i = argc - 1; i >= 0; --i) {
2161// TODO: revert workaround for Intel(R) 64 tracker #96
2162#if (KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640) && KMP_OS_LINUX1
2163 void *new_argv = va_arg(*ap, void *)__builtin_va_arg(*ap, void *);
2164#else
2165 void *new_argv = va_arg(ap, void *)__builtin_va_arg(ap, void *);
2166#endif
2167 KMP_CHECK_UPDATE(*argv, new_argv)if ((*argv) != (new_argv)) (*argv) = (new_argv);
2168 argv++;
2169 }
2170#if OMP_40_ENABLED(50 >= 40)
2171 } else {
2172 for (i = 0; i < argc; ++i) {
2173 // Get args from parent team for teams construct
2174 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i])if ((argv[i]) != (team->t.t_parent->t.t_argv[i])) (argv
[i]) = (team->t.t_parent->t.t_argv[i])
;
2175 }
2176 }
2177#endif /* OMP_40_ENABLED */
2178
2179 /* now actually fork the threads */
2180 KMP_CHECK_UPDATE(team->t.t_master_active, master_active)if ((team->t.t_master_active) != (master_active)) (team->
t.t_master_active) = (master_active)
;
2181 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
2182 root->r.r_active = TRUE(!0);
2183
2184 __kmp_fork_team_threads(root, team, master_th, gtid);
2185 __kmp_setup_icv_copy(team, nthreads,
2186 &master_th->th.th_current_task->td_icvs, loc);
2187
2188#if OMPT_SUPPORT1
2189 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
2190#endif
2191
2192 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2193
2194#if USE_ITT_BUILD1
2195 if (team->t.t_active_level == 1 // only report frames at level 1
2196#if OMP_40_ENABLED(50 >= 40)
2197 && !master_th->th.th_teams_microtask // not in teams construct
2198#endif /* OMP_40_ENABLED */
2199 ) {
2200#if USE_ITT_NOTIFY1
2201 if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2202 (__kmp_forkjoin_frames_mode == 3 ||
2203 __kmp_forkjoin_frames_mode == 1)) {
2204 kmp_uint64 tmp_time = 0;
2205 if (__itt_get_timestamp_ptr__kmp_itt_get_timestamp_ptr__3_0)
2206 tmp_time = __itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0();
2207 // Internal fork - report frame begin
2208 master_th->th.th_frame_time = tmp_time;
2209 if (__kmp_forkjoin_frames_mode == 3)
2210 team->t.t_region_time = tmp_time;
2211 } else
2212// only one notification scheme (either "submit" or "forking/joined", not both)
2213#endif /* USE_ITT_NOTIFY */
2214 if ((__itt_frame_begin_v3_ptr__kmp_itt_frame_begin_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2215 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2216 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer.
2217 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2218 }
2219 }
2220#endif /* USE_ITT_BUILD */
2221
2222 /* now go on and do the work */
2223 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team)((team == __kmp_threads[gtid]->th.th_team) ? 0 : __kmp_debug_assert
("team == __kmp_threads[gtid]->th.th_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2223))
;
2224 KMP_MB();
2225 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
2226 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
2227 root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
;
2228
2229#if USE_ITT_BUILD1
2230 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
2231 team->t.t_stack_id =
2232 __kmp_itt_stack_caller_create(); // create new stack stitching id
2233 // before entering fork barrier
2234 }
2235#endif /* USE_ITT_BUILD */
2236
2237#if OMP_40_ENABLED(50 >= 40)
2238 // AC: skip __kmp_internal_fork at teams construct, let only master
2239 // threads execute
2240 if (ap)
2241#endif /* OMP_40_ENABLED */
2242 {
2243 __kmp_internal_fork(loc, gtid, team);
2244 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
2245 "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
2246 root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
"master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
;
2247 }
2248
2249 if (call_context == fork_context_gnu) {
2250 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
2251 return TRUE(!0);
2252 }
2253
2254 /* Invoke microtask for MASTER thread */
2255 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
2256 team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
;
2257 } // END of timer KMP_fork_call block
2258
2259 {
2260 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
2261 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
2262 if (!team->t.t_invoke(gtid)) {
2263 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread")((0) ? 0 : __kmp_debug_assert(("cannot invoke microtask for MASTER thread"
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2263))
;
2264 }
2265 }
2266 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
2267 team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
;
2268 KMP_MB(); /* Flush all pending memory write invalidates. */
2269
2270 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); }
;
2271
2272#if OMPT_SUPPORT1
2273 if (ompt_enabled.enabled) {
2274 master_th->th.ompt_thread_info.state = omp_state_overhead;
2275 }
2276#endif
2277
2278 return TRUE(!0);
2279}
2280
2281#if OMPT_SUPPORT1
2282static inline void __kmp_join_restore_state(kmp_info_t *thread,
2283 kmp_team_t *team) {
2284 // restore state outside the region
2285 thread->th.ompt_thread_info.state =
2286 ((team->t.t_serialized) ? omp_state_work_serial
2287 : omp_state_work_parallel);
2288}
2289
2290static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread,
2291 kmp_team_t *team, ompt_data_t *parallel_data,
2292 fork_context_e fork_context, void *codeptr) {
2293 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2294 if (ompt_enabled.ompt_callback_parallel_end) {
2295 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
2296 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context)((fork_context == fork_context_gnu) ? ompt_invoker_program : ompt_invoker_runtime
)
,
2297 codeptr);
2298 }
2299
2300 task_info->frame.enter_frame = NULL__null;
2301 __kmp_join_restore_state(thread, team);
2302}
2303#endif
2304
2305void __kmp_join_call(ident_t *loc, int gtid
2306#if OMPT_SUPPORT1
2307 ,
2308 enum fork_context_e fork_context
2309#endif
2310#if OMP_40_ENABLED(50 >= 40)
2311 ,
2312 int exit_teams
2313#endif /* OMP_40_ENABLED */
2314 ) {
2315 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call)((void)0);
2316 kmp_team_t *team;
2317 kmp_team_t *parent_team;
2318 kmp_info_t *master_th;
2319 kmp_root_t *root;
2320 int master_active;
2321 int i;
2322
2323 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: enter T#%d\n"
, gtid); }
;
2324
2325 /* setup current data */
2326 master_th = __kmp_threads[gtid];
2327 root = master_th->th.th_root;
2328 team = master_th->th.th_team;
2329 parent_team = team->t.t_parent;
2330
2331 master_th->th.th_ident = loc;
2332
2333#if OMPT_SUPPORT1
2334 if (ompt_enabled.enabled) {
2335 master_th->th.ompt_thread_info.state = omp_state_overhead;
2336 }
2337#endif
2338
2339#if KMP_DEBUG1
2340 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2341 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", ((((master_th) != __null) ? 0 : __kmp_debug_assert
("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2343)), (master_th)->th.th_info.ds.ds_gtid), team, team->
t.t_task_team[master_th->th.th_task_state], master_th->
th.th_task_team); }
2342 "th_task_team = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", ((((master_th) != __null) ? 0 : __kmp_debug_assert
("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2343)), (master_th)->th.th_info.ds.ds_gtid), team, team->
t.t_task_team[master_th->th.th_task_state], master_th->
th.th_task_team); }
2343 __kmp_gtid_from_thread(master_th), team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", ((((master_th) != __null) ? 0 : __kmp_debug_assert
("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2343)), (master_th)->th.th_info.ds.ds_gtid), team, team->
t.t_task_team[master_th->th.th_task_state], master_th->
th.th_task_team); }
2344 team->t.t_task_team[master_th->th.th_task_state],if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", ((((master_th) != __null) ? 0 : __kmp_debug_assert
("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2343)), (master_th)->th.th_info.ds.ds_gtid), team, team->
t.t_task_team[master_th->th.th_task_state], master_th->
th.th_task_team); }
2345 master_th->th.th_task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
"th_task_team = %p\n", ((((master_th) != __null) ? 0 : __kmp_debug_assert
("(master_th) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2343)), (master_th)->th.th_info.ds.ds_gtid), team, team->
t.t_task_team[master_th->th.th_task_state], master_th->
th.th_task_team); }
;
2346 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==((master_th->th.th_task_team == team->t.t_task_team[master_th
->th.th_task_state]) ? 0 : __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2347))
2347 team->t.t_task_team[master_th->th.th_task_state])((master_th->th.th_task_team == team->t.t_task_team[master_th
->th.th_task_state]) ? 0 : __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2347))
;
2348 }
2349#endif
2350
2351 if (team->t.t_serialized) {
2352#if OMP_40_ENABLED(50 >= 40)
2353 if (master_th->th.th_teams_microtask) {
2354 // We are in teams construct
2355 int level = team->t.t_level;
2356 int tlevel = master_th->th.th_teams_level;
2357 if (level == tlevel) {
2358 // AC: we haven't incremented it earlier at start of teams construct,
2359 // so do it here - at the end of teams construct
2360 team->t.t_level++;
2361 } else if (level == tlevel + 1) {
2362 // AC: we are exiting parallel inside teams, need to increment
2363 // serialization in order to restore it in the next call to
2364 // __kmpc_end_serialized_parallel
2365 team->t.t_serialized++;
2366 }
2367 }
2368#endif /* OMP_40_ENABLED */
2369 __kmpc_end_serialized_parallel(loc, gtid);
2370
2371#if OMPT_SUPPORT1
2372 if (ompt_enabled.enabled) {
2373 __kmp_join_restore_state(master_th, parent_team);
2374 }
2375#endif
2376
2377 return;
2378 }
2379
2380 master_active = team->t.t_master_active;
2381
2382#if OMP_40_ENABLED(50 >= 40)
2383 if (!exit_teams)
2384#endif /* OMP_40_ENABLED */
2385 {
2386 // AC: No barrier for internal teams at exit from teams construct.
2387 // But there is barrier for external team (league).
2388 __kmp_internal_join(loc, gtid, team);
2389 }
2390#if OMP_40_ENABLED(50 >= 40)
2391 else {
2392 master_th->th.th_task_state =
2393 0; // AC: no tasking in teams (out of any parallel)
2394 }
2395#endif /* OMP_40_ENABLED */
2396
2397 KMP_MB();
2398
2399#if OMPT_SUPPORT1
2400 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2401 void *codeptr = team->t.ompt_team_info.master_return_address;
2402#endif
2403
2404#if USE_ITT_BUILD1
2405 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
2406 __kmp_itt_stack_caller_destroy(
2407 (__itt_caller)team->t
2408 .t_stack_id); // destroy the stack stitching id after join barrier
2409 }
2410
2411 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer.
2412 if (team->t.t_active_level == 1
2413#if OMP_40_ENABLED(50 >= 40)
2414 && !master_th->th.th_teams_microtask /* not in teams construct */
2415#endif /* OMP_40_ENABLED */
2416 ) {
2417 master_th->th.th_ident = loc;
2418 // only one notification scheme (either "submit" or "forking/joined", not
2419 // both)
2420 if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2421 __kmp_forkjoin_frames_mode == 3)
2422 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2423 master_th->th.th_frame_time, 0, loc,
2424 master_th->th.th_team_nproc, 1);
2425 else if ((__itt_frame_end_v3_ptr__kmp_itt_frame_end_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
2426 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2427 __kmp_itt_region_joined(gtid);
2428 } // active_level == 1
2429#endif /* USE_ITT_BUILD */
2430
2431#if OMP_40_ENABLED(50 >= 40)
2432 if (master_th->th.th_teams_microtask && !exit_teams &&
2433 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2434 team->t.t_level == master_th->th.th_teams_level + 1) {
2435 // AC: We need to leave the team structure intact at the end of parallel
2436 // inside the teams construct, so that at the next parallel same (hot) team
2437 // works, only adjust nesting levels
2438
2439 /* Decrement our nested depth level */
2440 team->t.t_level--;
2441 team->t.t_active_level--;
2442 KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
)
;
2443
2444 /* Restore number of threads in the team if needed */
2445 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2446 int old_num = master_th->th.th_team_nproc;
2447 int new_num = master_th->th.th_teams_size.nth;
2448 kmp_info_t **other_threads = team->t.t_threads;
2449 team->t.t_nproc = new_num;
2450 for (i = 0; i < old_num; ++i) {
2451 other_threads[i]->th.th_team_nproc = new_num;
2452 }
2453 // Adjust states of non-used threads of the team
2454 for (i = old_num; i < new_num; ++i) {
2455 // Re-initialize thread's barrier data.
2456 int b;
2457 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2458 for (b = 0; b < bs_last_barrier; ++b) {
2459 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2460 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)((balign[b].bb.wait_flag != 2) ? 0 : __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2460))
;
2461#if USE_DEBUGGER0
2462 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2463#endif
2464 }
2465 if (__kmp_tasking_mode != tskm_immediate_exec) {
2466 // Synchronize thread's task state
2467 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2468 }
2469 }
2470 }
2471
2472#if OMPT_SUPPORT1
2473 if (ompt_enabled.enabled) {
2474 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2475 codeptr);
2476 }
2477#endif
2478
2479 return;
2480 }
2481#endif /* OMP_40_ENABLED */
2482
2483 /* do cleanup and restore the parent team */
2484 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2485 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2486
2487 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2488
2489 /* jc: The following lock has instructions with REL and ACQ semantics,
2490 separating the parallel user code called in this parallel region
2491 from the serial user code called after this function returns. */
2492 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2493
2494#if OMP_40_ENABLED(50 >= 40)
2495 if (!master_th->th.th_teams_microtask ||
2496 team->t.t_level > master_th->th.th_teams_level)
2497#endif /* OMP_40_ENABLED */
2498 {
2499 /* Decrement our nested depth level */
2500 KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
)
;
2501 }
2502 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0)((root->r.r_in_parallel >= 0) ? 0 : __kmp_debug_assert(
"root->r.r_in_parallel >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2502))
;
2503
2504#if OMPT_SUPPORT1
2505 if (ompt_enabled.enabled) {
2506 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2507 if (ompt_enabled.ompt_callback_implicit_task) {
2508 int ompt_team_size = team->t.t_nproc;
2509 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
2510 ompt_scope_end, NULL__null, &(task_info->task_data), ompt_team_size,
2511 OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num);
2512 }
2513
2514 task_info->frame.exit_frame = NULL__null;
2515 task_info->task_data = ompt_data_none;
2516 }
2517#endif
2518
2519 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); }
2520 master_th, team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); }
;
2521 __kmp_pop_current_task_from_thread(master_th);
2522
2523#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
2524 // Restore master thread's partition.
2525 master_th->th.th_first_place = team->t.t_first_place;
2526 master_th->th.th_last_place = team->t.t_last_place;
2527#endif /* OMP_40_ENABLED */
2528
2529 updateHWFPControl(team);
2530
2531 if (root->r.r_active != master_active)
2532 root->r.r_active = master_active;
2533
2534 __kmp_free_team(root, team USE_NESTED_HOT_ARG(, master_th
2535 master_th), master_th); // this will free worker threads
2536
2537 /* this race was fun to find. make sure the following is in the critical
2538 region otherwise assertions may fail occasionally since the old team may be
2539 reallocated and the hierarchy appears inconsistent. it is actually safe to
2540 run and won't cause any bugs, but will cause those assertion failures. it's
2541 only one deref&assign so might as well put this in the critical region */
2542 master_th->th.th_team = parent_team;
2543 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2544 master_th->th.th_team_master = parent_team->t.t_threads[0];
2545 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2546
2547 /* restore serialized team, if need be */
2548 if (parent_team->t.t_serialized &&
2549 parent_team != master_th->th.th_serial_team &&
2550 parent_team != root->r.r_root_team) {
2551 __kmp_free_team(root,
2552 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL), __null);
2553 master_th->th.th_serial_team = parent_team;
2554 }
2555
2556 if (__kmp_tasking_mode != tskm_immediate_exec) {
2557 if (master_th->th.th_task_state_top >
2558 0) { // Restore task state from memo stack
2559 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)((master_th->th.th_task_state_memo_stack) ? 0 : __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2559))
;
2560 // Remember master's state if we re-use this nested hot team
2561 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2562 master_th->th.th_task_state;
2563 --master_th->th.th_task_state_top; // pop
2564 // Now restore state at this level
2565 master_th->th.th_task_state =
2566 master_th->th
2567 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2568 }
2569 // Copy the task team from the parent team to the master thread
2570 master_th->th.th_task_team =
2571 parent_team->t.t_task_team[master_th->th.th_task_state];
2572 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, ((((master_th) != __null) ? 0 : __kmp_debug_assert("(master_th) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2574)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team); }
2573 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, ((((master_th) != __null) ? 0 : __kmp_debug_assert("(master_th) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2574)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team); }
2574 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, ((((master_th) != __null) ? 0 : __kmp_debug_assert("(master_th) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2574)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team); }
2575 parent_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n"
, ((((master_th) != __null) ? 0 : __kmp_debug_assert("(master_th) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2574)), (master_th)->th.th_info.ds.ds_gtid), master_th->
th.th_task_team, parent_team); }
;
2576 }
2577
2578 // TODO: GEH - cannot do this assertion because root thread not set up as
2579 // executing
2580 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
2581 master_th->th.th_current_task->td_flags.executing = 1;
2582
2583 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2584
2585#if OMPT_SUPPORT1
2586 if (ompt_enabled.enabled) {
2587 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2588 codeptr);
2589 }
2590#endif
2591
2592 KMP_MB();
2593 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: exit T#%d\n"
, gtid); }
;
2594}
2595
2596/* Check whether we should push an internal control record onto the
2597 serial team stack. If so, do it. */
2598void __kmp_save_internal_controls(kmp_info_t *thread) {
2599
2600 if (thread->th.th_team != thread->th.th_serial_team) {
2601 return;
2602 }
2603 if (thread->th.th_team->t.t_serialized > 1) {
2604 int push = 0;
2605
2606 if (thread->th.th_team->t.t_control_stack_top == NULL__null) {
2607 push = 1;
2608 } else {
2609 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2610 thread->th.th_team->t.t_serialized) {
2611 push = 1;
2612 }
2613 }
2614 if (push) { /* push a record on the serial team's stack */
2615 kmp_internal_control_t *control =
2616 (kmp_internal_control_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_internal_control_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2617)
2617 sizeof(kmp_internal_control_t))___kmp_allocate((sizeof(kmp_internal_control_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2617)
;
2618
2619 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2620
2621 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2622
2623 control->next = thread->th.th_team->t.t_control_stack_top;
2624 thread->th.th_team->t.t_control_stack_top = control;
2625 }
2626 }
2627}
2628
2629/* Changes set_nproc */
2630void __kmp_set_num_threads(int new_nth, int gtid) {
2631 kmp_info_t *thread;
2632 kmp_root_t *root;
2633
2634 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_num_threads: new __kmp_nth = %d\n"
, new_nth); }
;
2635 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2635))
;
2636
2637 if (new_nth < 1)
2638 new_nth = 1;
2639 else if (new_nth > __kmp_max_nth)
2640 new_nth = __kmp_max_nth;
2641
2642 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth)((void)0);
2643 thread = __kmp_threads[gtid];
2644
2645 __kmp_save_internal_controls(thread);
2646
2647 set__nproc(thread, new_nth)(((thread)->th.th_current_task->td_icvs.nproc) = (new_nth
))
;
2648
2649 // If this omp_set_num_threads() call will cause the hot team size to be
2650 // reduced (in the absence of a num_threads clause), then reduce it now,
2651 // rather than waiting for the next parallel region.
2652 root = thread->th.th_root;
2653 if (__kmp_init_parallel && (!root->r.r_active) &&
2654 (root->r.r_hot_team->t.t_nproc > new_nth)
2655#if KMP_NESTED_HOT_TEAMS1
2656 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2657#endif
2658 ) {
2659 kmp_team_t *hot_team = root->r.r_hot_team;
2660 int f;
2661
2662 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2663
2664 // Release the extra threads we don't need any more.
2665 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2666 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)((hot_team->t.t_threads[f] != __null) ? 0 : __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2666))
;
2667 if (__kmp_tasking_mode != tskm_immediate_exec) {
2668 // When decreasing team size, threads no longer in the team should unref
2669 // task team.
2670 hot_team->t.t_threads[f]->th.th_task_team = NULL__null;
2671 }
2672 __kmp_free_thread(hot_team->t.t_threads[f]);
2673 hot_team->t.t_threads[f] = NULL__null;
2674 }
2675 hot_team->t.t_nproc = new_nth;
2676#if KMP_NESTED_HOT_TEAMS1
2677 if (thread->th.th_hot_teams) {
2678 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team)((hot_team == thread->th.th_hot_teams[0].hot_team) ? 0 : __kmp_debug_assert
("hot_team == thread->th.th_hot_teams[0].hot_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2678))
;
2679 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2680 }
2681#endif
2682
2683 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2684
2685 // Update the t_nproc field in the threads that are still active.
2686 for (f = 0; f < new_nth; f++) {
2687 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)((hot_team->t.t_threads[f] != __null) ? 0 : __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2687))
;
2688 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2689 }
2690 // Special flag in case omp_set_num_threads() call
2691 hot_team->t.t_size_changed = -1;
2692 }
2693}
2694
2695/* Changes max_active_levels */
2696void __kmp_set_max_active_levels(int gtid, int max_active_levels) {
2697 kmp_info_t *thread;
2698
2699 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
2700 "%d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
2701 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
"%d = (%d)\n", gtid, max_active_levels); }
;
2702 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2702))
;
2703
2704 // validate max_active_levels
2705 if (max_active_levels < 0) {
2706 KMP_WARNING(ActiveLevelsNegative, max_active_levels)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsNegative
, max_active_levels), __kmp_msg_null)
;
2707 // We ignore this call if the user has specified a negative value.
2708 // The current setting won't be changed. The last valid setting will be
2709 // used. A warning will be issued (if warnings are allowed as controlled by
2710 // the KMP_WARNINGS env var).
2711 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2712 "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2713 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
;
2714 return;
2715 }
2716 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647) {
2717 // it's OK, the max_active_levels is within the valid range: [ 0;
2718 // KMP_MAX_ACTIVE_LEVELS_LIMIT ]
2719 // We allow a zero value. (implementation defined behavior)
2720 } else {
2721 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null)
2722 KMP_MAX_ACTIVE_LEVELS_LIMIT)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null)
;
2723 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647;
2724 // Current upper limit is MAX_INT. (implementation defined behavior)
2725 // If the input exceeds the upper limit, we correct the input to be the
2726 // upper limit. (implementation defined behavior)
2727 // Actually, the flow should never get here until we use MAX_INT limit.
2728 }
2729 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2730 "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
2731 gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
"max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
;
2732
2733 thread = __kmp_threads[gtid];
2734
2735 __kmp_save_internal_controls(thread);
2736
2737 set__max_active_levels(thread, max_active_levels)(((thread)->th.th_current_task->td_icvs.max_active_levels
) = (max_active_levels))
;
2738}
2739
2740/* Gets max_active_levels */
2741int __kmp_get_max_active_levels(int gtid) {
2742 kmp_info_t *thread;
2743
2744 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d\n"
, gtid); }
;
2745 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2745))
;
2746
2747 thread = __kmp_threads[gtid];
2748 KMP_DEBUG_ASSERT(thread->th.th_current_task)((thread->th.th_current_task) ? 0 : __kmp_debug_assert("thread->th.th_current_task"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2748))
;
2749 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2750 "curtask_maxaclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2751 gtid, thread->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
2752 thread->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
"curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
;
2753 return thread->th.th_current_task->td_icvs.max_active_levels;
2754}
2755
2756/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
2757void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
2758 kmp_info_t *thread;
2759 // kmp_team_t *team;
2760
2761 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); }
2762 gtid, (int)kind, chunk))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); }
;
2763 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2763))
;
2764
2765 // Check if the kind parameter is valid, correct if needed.
2766 // Valid parameters should fit in one of two intervals - standard or extended:
2767 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
2768 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103
2769 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2770 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2771 // TODO: Hint needs attention in case we change the default schedule.
2772 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind)__kmp_msg_format(kmp_i18n_msg_ScheduleKindOutOfRange, kind),
2773 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk")__kmp_msg_format(kmp_i18n_hnt_DefaultScheduleKindUsed, "static, no chunk"
)
,
2774 __kmp_msg_null);
2775 kind = kmp_sched_default;
2776 chunk = 0; // ignore chunk value in case of bad kind
2777 }
2778
2779 thread = __kmp_threads[gtid];
2780
2781 __kmp_save_internal_controls(thread);
2782
2783 if (kind < kmp_sched_upper_std) {
2784 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK1) {
2785 // differ static chunked vs. unchunked: chunk should be invalid to
2786 // indicate unchunked schedule (which is the default)
2787 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
2788 } else {
2789 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2790 __kmp_sch_map[kind - kmp_sched_lower - 1];
2791 }
2792 } else {
2793 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2794 // kmp_sched_lower - 2 ];
2795 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2796 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2797 kmp_sched_lower - 2];
2798 }
2799 if (kind == kmp_sched_auto || chunk < 1) {
2800 // ignore parameter chunk for schedule auto
2801 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK1;
2802 } else {
2803 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2804 }
2805}
2806
2807/* Gets def_sched_var ICV values */
2808void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
2809 kmp_info_t *thread;
2810 enum sched_type th_type;
2811
2812 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_schedule: thread %d\n"
, gtid); }
;
2813 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2813))
;
2814
2815 thread = __kmp_threads[gtid];
2816
2817 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2818
2819 switch (th_type) {
2820 case kmp_sch_static:
2821 case kmp_sch_static_greedy:
2822 case kmp_sch_static_balanced:
2823 *kind = kmp_sched_static;
2824 *chunk = 0; // chunk was not set, try to show this fact via zero value
2825 return;
2826 case kmp_sch_static_chunked:
2827 *kind = kmp_sched_static;
2828 break;
2829 case kmp_sch_dynamic_chunked:
2830 *kind = kmp_sched_dynamic;
2831 break;
2832 case kmp_sch_guided_chunked:
2833 case kmp_sch_guided_iterative_chunked:
2834 case kmp_sch_guided_analytical_chunked:
2835 *kind = kmp_sched_guided;
2836 break;
2837 case kmp_sch_auto:
2838 *kind = kmp_sched_auto;
2839 break;
2840 case kmp_sch_trapezoidal:
2841 *kind = kmp_sched_trapezoidal;
2842 break;
2843#if KMP_STATIC_STEAL_ENABLED1
2844 case kmp_sch_static_steal:
2845 *kind = kmp_sched_static_steal;
2846 break;
2847#endif
2848 default:
2849 KMP_FATAL(UnknownSchedulingType, th_type)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownSchedulingType
, th_type), __kmp_msg_null)
;
2850 }
2851
2852 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2853}
2854
2855int __kmp_get_ancestor_thread_num(int gtid, int level) {
2856
2857 int ii, dd;
2858 kmp_team_t *team;
2859 kmp_info_t *thr;
2860
2861 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_ancestor_thread_num: thread %d %d\n"
, gtid, level); }
;
2862 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2862))
;
2863
2864 // validate level
2865 if (level == 0)
2866 return 0;
2867 if (level < 0)
2868 return -1;
2869 thr = __kmp_threads[gtid];
2870 team = thr->th.th_team;
2871 ii = team->t.t_level;
2872 if (level > ii)
2873 return -1;
2874
2875#if OMP_40_ENABLED(50 >= 40)
2876 if (thr->th.th_teams_microtask) {
2877 // AC: we are in teams region where multiple nested teams have same level
2878 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2879 if (level <=
2880 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2881 KMP_DEBUG_ASSERT(ii >= tlevel)((ii >= tlevel) ? 0 : __kmp_debug_assert("ii >= tlevel"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2881))
;
2882 // AC: As we need to pass by the teams league, we need to artificially
2883 // increase ii
2884 if (ii == tlevel) {
2885 ii += 2; // three teams have same level
2886 } else {
2887 ii++; // two teams have same level
2888 }
2889 }
2890 }
2891#endif
2892
2893 if (ii == level)
2894 return __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2894)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
2895
2896 dd = team->t.t_serialized;
2897 level++;
2898 while (ii > level) {
2899 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2900 }
2901 if ((team->t.t_serialized) && (!dd)) {
2902 team = team->t.t_parent;
2903 continue;
2904 }
2905 if (ii > level) {
2906 team = team->t.t_parent;
2907 dd = team->t.t_serialized;
2908 ii--;
2909 }
2910 }
2911
2912 return (dd > 1) ? (0) : (team->t.t_master_tid);
2913}
2914
2915int __kmp_get_team_size(int gtid, int level) {
2916
2917 int ii, dd;
2918 kmp_team_t *team;
2919 kmp_info_t *thr;
2920
2921 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_team_size: thread %d %d\n"
, gtid, level); }
;
2922 KMP_DEBUG_ASSERT(__kmp_init_serial)((__kmp_init_serial) ? 0 : __kmp_debug_assert("__kmp_init_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2922))
;
2923
2924 // validate level
2925 if (level == 0)
2926 return 1;
2927 if (level < 0)
2928 return -1;
2929 thr = __kmp_threads[gtid];
2930 team = thr->th.th_team;
2931 ii = team->t.t_level;
2932 if (level > ii)
2933 return -1;
2934
2935#if OMP_40_ENABLED(50 >= 40)
2936 if (thr->th.th_teams_microtask) {
2937 // AC: we are in teams region where multiple nested teams have same level
2938 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2939 if (level <=
2940 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2941 KMP_DEBUG_ASSERT(ii >= tlevel)((ii >= tlevel) ? 0 : __kmp_debug_assert("ii >= tlevel"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 2941))
;
2942 // AC: As we need to pass by the teams league, we need to artificially
2943 // increase ii
2944 if (ii == tlevel) {
2945 ii += 2; // three teams have same level
2946 } else {
2947 ii++; // two teams have same level
2948 }
2949 }
2950 }
2951#endif
2952
2953 while (ii > level) {
2954 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2955 }
2956 if (team->t.t_serialized && (!dd)) {
2957 team = team->t.t_parent;
2958 continue;
2959 }
2960 if (ii > level) {
2961 team = team->t.t_parent;
2962 ii--;
2963 }
2964 }
2965
2966 return team->t.t_nproc;
2967}
2968
2969kmp_r_sched_t __kmp_get_schedule_global() {
2970 // This routine created because pairs (__kmp_sched, __kmp_chunk) and
2971 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults
2972 // independently. So one can get the updated schedule here.
2973
2974 kmp_r_sched_t r_sched;
2975
2976 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
2977 // __kmp_guided. __kmp_sched should keep original value, so that user can set
2978 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in
2979 // different roots (even in OMP 2.5)
2980 if (__kmp_sched == kmp_sch_static) {
2981 // replace STATIC with more detailed schedule (balanced or greedy)
2982 r_sched.r_sched_type = __kmp_static;
2983 } else if (__kmp_sched == kmp_sch_guided_chunked) {
2984 // replace GUIDED with more detailed schedule (iterative or analytical)
2985 r_sched.r_sched_type = __kmp_guided;
2986 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
2987 r_sched.r_sched_type = __kmp_sched;
2988 }
2989
2990 if (__kmp_chunk < KMP_DEFAULT_CHUNK1) {
2991 // __kmp_chunk may be wrong here (if it was not ever set)
2992 r_sched.chunk = KMP_DEFAULT_CHUNK1;
2993 } else {
2994 r_sched.chunk = __kmp_chunk;
2995 }
2996
2997 return r_sched;
2998}
2999
3000/* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
3001 at least argc number of *t_argv entries for the requested team. */
3002static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) {
3003
3004 KMP_DEBUG_ASSERT(team)((team) ? 0 : __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3004))
;
3005 if (!realloc || argc > team->t.t_max_argc) {
3006
3007 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
3008 "current entries=%d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
3009 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
"current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
;
3010 /* if previously allocated heap space for args, free them */
3011 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3012 __kmp_free((void *)team->t.t_argv)___kmp_free(((void *)team->t.t_argv), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3012)
;
3013
3014 if (argc <= KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
) {
3015 /* use unused space in the cache line for arguments */
3016 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
;
3017 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3018 "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3019 team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
;
3020 team->t.t_argv = &team->t.t_inline_argv[0];
3021 if (__kmp_storage_map) {
3022 __kmp_print_storage_map_gtid(
3023 -1, &team->t.t_inline_argv[0],
3024 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
],
3025 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
64)) / (sizeof(void *)))
), "team_%d.t_inline_argv",
3026 team->t.t_id);
3027 }
3028 } else {
3029 /* allocate space for arguments in the heap */
3030 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES100 >> 1))
3031 ? KMP_MIN_MALLOC_ARGV_ENTRIES100
3032 : 2 * argc;
3033 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3034 "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
3035 team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
"argv entries\n", team->t.t_id, team->t.t_max_argc); }
;
3036 team->t.t_argv =
3037 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc)___kmp_page_allocate((sizeof(void *) * team->t.t_max_argc)
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3037)
;
3038 if (__kmp_storage_map) {
3039 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3040 &team->t.t_argv[team->t.t_max_argc],
3041 sizeof(void *) * team->t.t_max_argc,
3042 "team_%d.t_argv", team->t.t_id);
3043 }
3044 }
3045 }
3046}
3047
3048static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) {
3049 int i;
3050 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3051 team->t.t_threads =
3052 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth)___kmp_allocate((sizeof(kmp_info_t *) * max_nth), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3052)
;
3053 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3054)
3054 sizeof(dispatch_shared_info_t) * num_disp_buff)___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3054)
;
3055 team->t.t_dispatch =
3056 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth)___kmp_allocate((sizeof(kmp_disp_t) * max_nth), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3056)
;
3057 team->t.t_implicit_task_taskdata =
3058 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth)___kmp_allocate((sizeof(kmp_taskdata_t) * max_nth), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3058)
;
3059 team->t.t_max_nproc = max_nth;
3060
3061 /* setup dispatch buffers */
3062 for (i = 0; i < num_disp_buff; ++i) {
3063 team->t.t_disp_buffer[i].buffer_index = i;
3064#if OMP_45_ENABLED(50 >= 45)
3065 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3066#endif
3067 }
3068}
3069
3070static void __kmp_free_team_arrays(kmp_team_t *team) {
3071 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */
3072 int i;
3073 for (i = 0; i < team->t.t_max_nproc; ++i) {
3074 if (team->t.t_dispatch[i].th_disp_buffer != NULL__null) {
3075 __kmp_free(team->t.t_dispatch[i].th_disp_buffer)___kmp_free((team->t.t_dispatch[i].th_disp_buffer), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3075)
;
3076 team->t.t_dispatch[i].th_disp_buffer = NULL__null;
3077 }
3078 }
3079#if KMP_USE_HIER_SCHED0
3080 __kmp_dispatch_free_hierarchies(team);
3081#endif
3082 __kmp_free(team->t.t_threads)___kmp_free((team->t.t_threads), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3082)
;
3083 __kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3083)
;
3084 __kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3084)
;
3085 __kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3085)
;
3086 team->t.t_threads = NULL__null;
3087 team->t.t_disp_buffer = NULL__null;
3088 team->t.t_dispatch = NULL__null;
3089 team->t.t_implicit_task_taskdata = 0;
3090}
3091
3092static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
3093 kmp_info_t **oldThreads = team->t.t_threads;
3094
3095 __kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3095)
;
3096 __kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3096)
;
3097 __kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3097)
;
3098 __kmp_allocate_team_arrays(team, max_nth);
3099
3100 KMP_MEMCPYmemcpy(team->t.t_threads, oldThreads,
3101 team->t.t_nproc * sizeof(kmp_info_t *));
3102
3103 __kmp_free(oldThreads)___kmp_free((oldThreads), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3103)
;
3104}
3105
3106static kmp_internal_control_t __kmp_get_global_icvs(void) {
3107
3108 kmp_r_sched_t r_sched =
3109 __kmp_get_schedule_global(); // get current state of scheduling globals
3110
3111#if OMP_40_ENABLED(50 >= 40)
3112 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0)((__kmp_nested_proc_bind.used > 0) ? 0 : __kmp_debug_assert
("__kmp_nested_proc_bind.used > 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3112))
;
3113#endif /* OMP_40_ENABLED */
3114
3115 kmp_internal_control_t g_icvs = {
3116 0, // int serial_nesting_level; //corresponds to value of th_team_serialized
3117 (kmp_int8)__kmp_dflt_nested, // int nested; //internal control
3118 // for nested parallelism (per thread)
3119 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
3120 // adjustment of threads (per thread)
3121 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
3122 // whether blocktime is explicitly set
3123 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
3124#if KMP_USE_MONITOR
3125 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
3126// intervals
3127#endif
3128 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for
3129 // next parallel region (per thread)
3130 // (use a max ub on value if __kmp_parallel_initialize not called yet)
3131 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control
3132 // for max_active_levels
3133 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule
3134// {sched,chunk} pair
3135#if OMP_40_ENABLED(50 >= 40)
3136 __kmp_nested_proc_bind.bind_types[0],
3137 __kmp_default_device,
3138#endif /* OMP_40_ENABLED */
3139 NULL__null // struct kmp_internal_control *next;
3140 };
3141
3142 return g_icvs;
3143}
3144
3145static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) {
3146
3147 kmp_internal_control_t gx_icvs;
3148 gx_icvs.serial_nesting_level =
3149 0; // probably =team->t.t_serial like in save_inter_controls
3150 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3151 gx_icvs.next = NULL__null;
3152
3153 return gx_icvs;
3154}
3155
3156static void __kmp_initialize_root(kmp_root_t *root) {
3157 int f;
3158 kmp_team_t *root_team;
3159 kmp_team_t *hot_team;
3160 int hot_team_max_nth;
3161 kmp_r_sched_t r_sched =
3162 __kmp_get_schedule_global(); // get current state of scheduling globals
3163 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3164 KMP_DEBUG_ASSERT(root)((root) ? 0 : __kmp_debug_assert("root", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3164))
;
3165 KMP_ASSERT(!root->r.r_begin)((!root->r.r_begin) ? 0 : __kmp_debug_assert("!root->r.r_begin"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3165))
;
3166
3167 /* setup the root state structure */
3168 __kmp_init_lock(&root->r.r_begin_lock);
3169 root->r.r_begin = FALSE0;
3170 root->r.r_active = FALSE0;
3171 root->r.r_in_parallel = 0;
3172 root->r.r_blocktime = __kmp_dflt_blocktime;
3173 root->r.r_nested = __kmp_dflt_nested;
3174 root->r.r_cg_nthreads = 1;
3175
3176 /* setup the root team for this task */
3177 /* allocate the root team structure */
3178 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before root_team\n"
); }
;
3179
3180 root_team =
3181 __kmp_allocate_team(root,
3182 1, // new_nproc
3183 1, // max_nproc
3184#if OMPT_SUPPORT1
3185 ompt_data_none, // root parallel id
3186#endif
3187#if OMP_40_ENABLED(50 >= 40)
3188 __kmp_nested_proc_bind.bind_types[0],
3189#endif
3190 &r_icvs,
3191 0 // argc
3192 USE_NESTED_HOT_ARG(NULL), __null // master thread is unknown
3193 );
3194#if USE_DEBUGGER0
3195 // Non-NULL value should be assigned to make the debugger display the root
3196 // team.
3197 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0))((root_team->t.t_pkfn)) = (((microtask_t)(~0)));
3198#endif
3199
3200 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after root_team = %p\n"
, root_team); }
;
3201
3202 root->r.r_root_team = root_team;
3203 root_team->t.t_control_stack_top = NULL__null;
3204
3205 /* initialize root team */
3206 root_team->t.t_threads[0] = NULL__null;
3207 root_team->t.t_nproc = 1;
3208 root_team->t.t_serialized = 1;
3209 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3210 root_team->t.t_sched.sched = r_sched.sched;
3211 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3212 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3213 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
3214 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
;
3215
3216 /* setup the hot team for this task */
3217 /* allocate the hot team structure */
3218 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before hot_team\n"
); }
;
3219
3220 hot_team =
3221 __kmp_allocate_team(root,
3222 1, // new_nproc
3223 __kmp_dflt_team_nth_ub * 2, // max_nproc
3224#if OMPT_SUPPORT1
3225 ompt_data_none, // root parallel id
3226#endif
3227#if OMP_40_ENABLED(50 >= 40)
3228 __kmp_nested_proc_bind.bind_types[0],
3229#endif
3230 &r_icvs,
3231 0 // argc
3232 USE_NESTED_HOT_ARG(NULL), __null // master thread is unknown
3233 );
3234 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after hot_team = %p\n"
, hot_team); }
;
3235
3236 root->r.r_hot_team = hot_team;
3237 root_team->t.t_control_stack_top = NULL__null;
3238
3239 /* first-time initialization */
3240 hot_team->t.t_parent = root_team;
3241
3242 /* initialize hot team */
3243 hot_team_max_nth = hot_team->t.t_max_nproc;
3244 for (f = 0; f < hot_team_max_nth; ++f) {
3245 hot_team->t.t_threads[f] = NULL__null;
3246 }
3247 hot_team->t.t_nproc = 1;
3248 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3249 hot_team->t.t_sched.sched = r_sched.sched;
3250 hot_team->t.t_size_changed = 0;
3251}
3252
3253#ifdef KMP_DEBUG1
3254
3255typedef struct kmp_team_list_item {
3256 kmp_team_p const *entry;
3257 struct kmp_team_list_item *next;
3258} kmp_team_list_item_t;
3259typedef kmp_team_list_item_t *kmp_team_list_t;
3260
3261static void __kmp_print_structure_team_accum( // Add team to list of teams.
3262 kmp_team_list_t list, // List of teams.
3263 kmp_team_p const *team // Team to add.
3264 ) {
3265
3266 // List must terminate with item where both entry and next are NULL.
3267 // Team is added to the list only once.
3268 // List is sorted in ascending order by team id.
3269 // Team id is *not* a key.
3270
3271 kmp_team_list_t l;
3272
3273 KMP_DEBUG_ASSERT(list != NULL)((list != __null) ? 0 : __kmp_debug_assert("list != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3273))
;
3274 if (team == NULL__null) {
3275 return;
3276 }
3277
3278 __kmp_print_structure_team_accum(list, team->t.t_parent);
3279 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3280
3281 // Search list for the team.
3282 l = list;
3283 while (l->next != NULL__null && l->entry != team) {
3284 l = l->next;
3285 }
3286 if (l->next != NULL__null) {
3287 return; // Team has been added before, exit.
3288 }
3289
3290 // Team is not found. Search list again for insertion point.
3291 l = list;
3292 while (l->next != NULL__null && l->entry->t.t_id <= team->t.t_id) {
3293 l = l->next;
3294 }
3295
3296 // Insert team.
3297 {
3298 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(malloc(sizeof(kmp_team_list_item_t))
3299 sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
3300 *item = *l;
3301 l->entry = team;
3302 l->next = item;
3303 }
3304}
3305
3306static void __kmp_print_structure_team(char const *title, kmp_team_p const *team
3307
3308 ) {
3309 __kmp_printf("%s", title);
3310 if (team != NULL__null) {
3311 __kmp_printf("%2x %p\n", team->t.t_id, team);
3312 } else {
3313 __kmp_printf(" - (nil)\n");
3314 }
3315}
3316
3317static void __kmp_print_structure_thread(char const *title,
3318 kmp_info_p const *thread) {
3319 __kmp_printf("%s", title);
3320 if (thread != NULL__null) {
3321 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3322 } else {
3323 __kmp_printf(" - (nil)\n");
3324 }
3325}
3326
3327void __kmp_print_structure(void) {
3328
3329 kmp_team_list_t list;
3330
3331 // Initialize list of teams.
3332 list =
3333 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
3334 list->entry = NULL__null;
3335 list->next = NULL__null;
3336
3337 __kmp_printf("\n------------------------------\nGlobal Thread "
3338 "Table\n------------------------------\n");
3339 {
3340 int gtid;
3341 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3342 __kmp_printf("%2d", gtid);
3343 if (__kmp_threads != NULL__null) {
3344 __kmp_printf(" %p", __kmp_threads[gtid]);
3345 }
3346 if (__kmp_root != NULL__null) {
3347 __kmp_printf(" %p", __kmp_root[gtid]);
3348 }
3349 __kmp_printf("\n");
3350 }
3351 }
3352
3353 // Print out __kmp_threads array.
3354 __kmp_printf("\n------------------------------\nThreads\n--------------------"
3355 "----------\n");
3356 if (__kmp_threads != NULL__null) {
3357 int gtid;
3358 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3359 kmp_info_t const *thread = __kmp_threads[gtid];
3360 if (thread != NULL__null) {
3361 __kmp_printf("GTID %2d %p:\n", gtid, thread);
3362 __kmp_printf(" Our Root: %p\n", thread->th.th_root);
3363 __kmp_print_structure_team(" Our Team: ", thread->th.th_team);
3364 __kmp_print_structure_team(" Serial Team: ",
3365 thread->th.th_serial_team);
3366 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc);
3367 __kmp_print_structure_thread(" Master: ",
3368 thread->th.th_team_master);
3369 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized);
3370 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc);
3371#if OMP_40_ENABLED(50 >= 40)
3372 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3373#endif
3374 __kmp_print_structure_thread(" Next in pool: ",
3375 thread->th.th_next_pool);
3376 __kmp_printf("\n");
3377 __kmp_print_structure_team_accum(list, thread->th.th_team);
3378 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3379 }
3380 }
3381 } else {
3382 __kmp_printf("Threads array is not allocated.\n");
3383 }
3384
3385 // Print out __kmp_root array.
3386 __kmp_printf("\n------------------------------\nUbers\n----------------------"
3387 "--------\n");
3388 if (__kmp_root != NULL__null) {
3389 int gtid;
3390 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3391 kmp_root_t const *root = __kmp_root[gtid];
3392 if (root != NULL__null) {
3393 __kmp_printf("GTID %2d %p:\n", gtid, root);
3394 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team);
3395 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team);
3396 __kmp_print_structure_thread(" Uber Thread: ",
3397 root->r.r_uber_thread);
3398 __kmp_printf(" Active?: %2d\n", root->r.r_active);
3399 __kmp_printf(" Nested?: %2d\n", root->r.r_nested);
3400 __kmp_printf(" In Parallel: %2d\n",
3401 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)(&root->r.r_in_parallel)->load(std::memory_order_relaxed
)
);
3402 __kmp_printf("\n");
3403 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3404 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3405 }
3406 }
3407 } else {
3408 __kmp_printf("Ubers array is not allocated.\n");
3409 }
3410
3411 __kmp_printf("\n------------------------------\nTeams\n----------------------"
3412 "--------\n");
3413 while (list->next != NULL__null) {
3414 kmp_team_p const *team = list->entry;
3415 int i;
3416 __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
3417 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent);
3418 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid);
3419 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc);
3420 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized);
3421 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc);
3422 for (i = 0; i < team->t.t_nproc; ++i) {
3423 __kmp_printf(" Thread %2d: ", i);
3424 __kmp_print_structure_thread("", team->t.t_threads[i]);
3425 }
3426 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool);
3427 __kmp_printf("\n");
3428 list = list->next;
3429 }
3430
3431 // Print out __kmp_thread_pool and __kmp_team_pool.
3432 __kmp_printf("\n------------------------------\nPools\n----------------------"
3433 "--------\n");
3434 __kmp_print_structure_thread("Thread pool: ",
3435 CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool));
3436 __kmp_print_structure_team("Team pool: ",
3437 CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool));
3438 __kmp_printf("\n");
3439
3440 // Free team list.
3441 while (list != NULL__null) {
3442 kmp_team_list_item_t *item = list;
3443 list = list->next;
3444 KMP_INTERNAL_FREE(item)free(item);
3445 }
3446}
3447
3448#endif
3449
3450//---------------------------------------------------------------------------
3451// Stuff for per-thread fast random number generator
3452// Table of primes
3453static const unsigned __kmp_primes[] = {
3454 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3455 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3456 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3457 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3458 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3459 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3460 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3461 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3462 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3463 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3464 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3465
3466//---------------------------------------------------------------------------
3467// __kmp_get_random: Get a random number using a linear congruential method.
3468unsigned short __kmp_get_random(kmp_info_t *thread) {
3469 unsigned x = thread->th.th_x;
3470 unsigned short r = x >> 16;
3471
3472 thread->th.th_x = x * thread->th.th_a + 1;
3473
3474 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); }
3475 thread->th.th_info.ds.ds_tid, r))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); }
;
3476
3477 return r;
3478}
3479//--------------------------------------------------------
3480// __kmp_init_random: Initialize a random number generator
3481void __kmp_init_random(kmp_info_t *thread) {
3482 unsigned seed = thread->th.th_info.ds.ds_tid;
3483
3484 thread->th.th_a =
3485 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))];
3486 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3487 KA_TRACE(30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); }
3488 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); }
;
3489}
3490
3491#if KMP_OS_WINDOWS0
3492/* reclaim array entries for root threads that are already dead, returns number
3493 * reclaimed */
3494static int __kmp_reclaim_dead_roots(void) {
3495 int i, r = 0;
3496
3497 for (i = 0; i < __kmp_threads_capacity; ++i) {
3498 if (KMP_UBER_GTID(i)((((i) >= (-6)) ? 0 : __kmp_debug_assert("(i) >= (-6)",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3498)), (((i) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(i) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3498)), (i) >= 0 && __kmp_root[(i)] && __kmp_threads
[(i)] && (__kmp_threads[(i)] == __kmp_root[(i)]->r
.r_uber_thread))
&&
3499 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])((void *)(__kmp_threads[i]))) &&
3500 !__kmp_root[i]
3501 ->r.r_active) { // AC: reclaim only roots died in non-active state
3502 r += __kmp_unregister_root_other_thread(i);
3503 }
3504 }
3505 return r;
3506}
3507#endif
3508
3509/* This function attempts to create free entries in __kmp_threads and
3510 __kmp_root, and returns the number of free entries generated.
3511
3512 For Windows* OS static library, the first mechanism used is to reclaim array
3513 entries for root threads that are already dead.
3514
3515 On all platforms, expansion is attempted on the arrays __kmp_threads_ and
3516 __kmp_root, with appropriate update to __kmp_threads_capacity. Array
3517 capacity is increased by doubling with clipping to __kmp_tp_capacity, if
3518 threadprivate cache array has been created. Synchronization with
3519 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.
3520
3521 After any dead root reclamation, if the clipping value allows array expansion
3522 to result in the generation of a total of nNeed free slots, the function does
3523 that expansion. If not, nothing is done beyond the possible initial root
3524 thread reclamation.
3525
3526 If any argument is negative, the behavior is undefined. */
3527static int __kmp_expand_threads(int nNeed) {
3528 int added = 0;
3529 int minimumRequiredCapacity;
3530 int newCapacity;
3531 kmp_info_t **newThreads;
3532 kmp_root_t **newRoot;
3533
3534// All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so
3535// resizing __kmp_threads does not need additional protection if foreign
3536// threads are present
3537
3538#if KMP_OS_WINDOWS0 && !defined KMP_DYNAMIC_LIB1
3539 /* only for Windows static library */
3540 /* reclaim array entries for root threads that are already dead */
3541 added = __kmp_reclaim_dead_roots();
3542
3543 if (nNeed) {
3544 nNeed -= added;
3545 if (nNeed < 0)
3546 nNeed = 0;
3547 }
3548#endif
3549 if (nNeed <= 0)
3550 return added;
3551
3552 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
3553 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
3554 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
3555 // > __kmp_max_nth in one of two ways:
3556 //
3557 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0]
3558 // may not be resused by another thread, so we may need to increase
3559 // __kmp_threads_capacity to __kmp_max_nth + 1.
3560 //
3561 // 2) New foreign root(s) are encountered. We always register new foreign
3562 // roots. This may cause a smaller # of threads to be allocated at
3563 // subsequent parallel regions, but the worker threads hang around (and
3564 // eventually go to sleep) and need slots in the __kmp_threads[] array.
3565 //
3566 // Anyway, that is the reason for moving the check to see if
3567 // __kmp_max_nth was exceeded into __kmp_reserve_threads()
3568 // instead of having it performed here. -BB
3569
3570 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity)((__kmp_sys_max_nth >= __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("__kmp_sys_max_nth >= __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3570))
;
3571
3572 /* compute expansion headroom to check if we can expand */
3573 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3574 /* possible expansion too small -- give up */
3575 return added;
3576 }
3577 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3578
3579 newCapacity = __kmp_threads_capacity;
3580 do {
3581 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3582 : __kmp_sys_max_nth;
3583 } while (newCapacity < minimumRequiredCapacity);
3584 newThreads = (kmp_info_t **)__kmp_allocate(___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3585)
3585 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE)___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3585)
;
3586 newRoot =
3587 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity);
3588 KMP_MEMCPYmemcpy(newThreads, __kmp_threads,
3589 __kmp_threads_capacity * sizeof(kmp_info_t *));
3590 KMP_MEMCPYmemcpy(newRoot, __kmp_root,
3591 __kmp_threads_capacity * sizeof(kmp_root_t *));
3592
3593 kmp_info_t **temp_threads = __kmp_threads;
3594 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads;
3595 *(kmp_root_t * *volatile *)&__kmp_root = newRoot;
3596 __kmp_free(temp_threads)___kmp_free((temp_threads), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3596)
;
3597 added += newCapacity - __kmp_threads_capacity;
3598 *(volatile int *)&__kmp_threads_capacity = newCapacity;
3599
3600 if (newCapacity > __kmp_tp_capacity) {
3601 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3602 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3603 __kmp_threadprivate_resize_cache(newCapacity);
3604 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size
3605 *(volatile int *)&__kmp_tp_capacity = newCapacity;
3606 }
3607 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3608 }
3609
3610 return added;
3611}
3612
3613/* Register the current thread as a root thread and obtain our gtid. We must
3614 have the __kmp_initz_lock held at this point. Argument TRUE only if are the
3615 thread that calls from __kmp_do_serial_initialize() */
3616int __kmp_register_root(int initial_thread) {
3617 kmp_info_t *root_thread;
3618 kmp_root_t *root;
3619 int gtid;
3620 int capacity;
3621 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3622 KA_TRACE(20, ("__kmp_register_root: entered\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: entered\n"
); }
;
3623 KMP_MB();
3624
3625 /* 2007-03-02:
3626 If initial thread did not invoke OpenMP RTL yet, and this thread is not an
3627 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not
3628 work as expected -- it may return false (that means there is at least one
3629 empty slot in __kmp_threads array), but it is possible the only free slot
3630 is #0, which is reserved for initial thread and so cannot be used for this
3631 one. Following code workarounds this bug.
3632
3633 However, right solution seems to be not reserving slot #0 for initial
3634 thread because:
3635 (1) there is no magic in slot #0,
3636 (2) we cannot detect initial thread reliably (the first thread which does
3637 serial initialization may be not a real initial thread).
3638 */
3639 capacity = __kmp_threads_capacity;
3640 if (!initial_thread && TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
3641 --capacity;
3642 }
3643
3644 /* see if there are too many threads */
3645 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
3646 if (__kmp_tp_cached) {
3647 __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread),
3648 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
)
,
3649 KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
3650 } else {
3651 __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads),
3652 __kmp_msg_null);
3653 }
3654 }
3655
3656 /* find an available thread slot */
3657 /* Don't reassign the zero slot since we need that to only be used by initial
3658 thread */
3659 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid])((void *)(__kmp_threads[gtid])) != NULL__null;
3660 gtid++)
3661 ;
3662 KA_TRACE(1,if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); }
3663 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); }
;
3664 KMP_ASSERT(gtid < __kmp_threads_capacity)((gtid < __kmp_threads_capacity) ? 0 : __kmp_debug_assert(
"gtid < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3664))
;
3665
3666 /* update global accounting */
3667 __kmp_all_nth++;
3668 TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);
3669
3670 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
3671 // numbers of procs, and method #2 (keyed API call) for higher numbers.
3672 if (__kmp_adjust_gtid_mode) {
3673 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3674 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
3675 TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
3676 }
3677 } else {
3678 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
3679 TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
3680 }
3681 }
3682 }
3683
3684#ifdef KMP_ADJUST_BLOCKTIME1
3685 /* Adjust blocktime to zero if necessary */
3686 /* Middle initialization might not have occurred yet */
3687 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3688 if (__kmp_nth > __kmp_avail_proc) {
3689 __kmp_zero_bt = TRUE(!0);
3690 }
3691 }
3692#endif /* KMP_ADJUST_BLOCKTIME */
3693
3694 /* setup this new hierarchy */
3695 if (!(root = __kmp_root[gtid])) {
3696 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t))___kmp_allocate((sizeof(kmp_root_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3696)
;
3697 KMP_DEBUG_ASSERT(!root->r.r_root_team)((!root->r.r_root_team) ? 0 : __kmp_debug_assert("!root->r.r_root_team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3697))
;
3698 }
3699
3700#if KMP_STATS_ENABLED0
3701 // Initialize stats as soon as possible (right after gtid assignment).
3702 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3703 KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life)((void)0);
3704 KMP_SET_THREAD_STATE(SERIAL_REGION)((void)0);
3705 KMP_INIT_PARTITIONED_TIMERS(OMP_serial)((void)0);
3706#endif
3707 __kmp_initialize_root(root);
3708
3709 /* setup new root thread structure */
3710 if (root->r.r_uber_thread) {
3711 root_thread = root->r.r_uber_thread;
3712 } else {
3713 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3713)
;
3714 if (__kmp_storage_map) {
3715 __kmp_print_thread_storage_map(root_thread, gtid);
3716 }
3717 root_thread->th.th_info.ds.ds_gtid = gtid;
3718#if OMPT_SUPPORT1
3719 root_thread->th.ompt_thread_info.thread_data.ptr = NULL__null;
3720#endif
3721 root_thread->th.th_root = root;
3722 if (__kmp_env_consistency_check) {
3723 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3724 }
3725#if USE_FAST_MEMORY3
3726 __kmp_initialize_fast_memory(root_thread);
3727#endif /* USE_FAST_MEMORY */
3728
3729#if KMP_USE_BGET1
3730 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL)((root_thread->th.th_local.bget_data == __null) ? 0 : __kmp_debug_assert
("root_thread->th.th_local.bget_data == __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3730))
;
3731 __kmp_initialize_bget(root_thread);
3732#endif
3733 __kmp_init_random(root_thread); // Initialize random number generator
3734 }
3735
3736 /* setup the serial team held in reserve by the root thread */
3737 if (!root_thread->th.th_serial_team) {
3738 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3739 KF_TRACE(10, ("__kmp_register_root: before serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: before serial_team\n"
); }
;
3740 root_thread->th.th_serial_team =
3741 __kmp_allocate_team(root, 1, 1,
3742#if OMPT_SUPPORT1
3743 ompt_data_none, // root parallel id
3744#endif
3745#if OMP_40_ENABLED(50 >= 40)
3746 proc_bind_default,
3747#endif
3748 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL), __null);
3749 }
3750 KMP_ASSERT(root_thread->th.th_serial_team)((root_thread->th.th_serial_team) ? 0 : __kmp_debug_assert
("root_thread->th.th_serial_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3750))
;
3751 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); }
3752 root_thread->th.th_serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); }
;
3753
3754 /* drop root_thread into place */
3755 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread)((__kmp_threads[gtid])) = ((root_thread));
3756
3757 root->r.r_root_team->t.t_threads[0] = root_thread;
3758 root->r.r_hot_team->t.t_threads[0] = root_thread;
3759 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3760 // AC: the team created in reserve, not for execution (it is unused for now).
3761 root_thread->th.th_serial_team->t.t_serialized = 0;
3762 root->r.r_uber_thread = root_thread;
3763
3764 /* initialize the thread, get it ready to go */
3765 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3766 TCW_4(__kmp_init_gtid, TRUE)(__kmp_init_gtid) = ((!0));
3767
3768 /* prepare the master thread for get_gtid() */
3769 __kmp_gtid_set_specific(gtid);
3770
3771#if USE_ITT_BUILD1
3772 __kmp_itt_thread_name(gtid);
3773#endif /* USE_ITT_BUILD */
3774
3775#ifdef KMP_TDATA_GTID1
3776 __kmp_gtid = gtid;
3777#endif
3778 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3779 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid)((__kmp_gtid_get_specific() == gtid) ? 0 : __kmp_debug_assert
("__kmp_gtid_get_specific() == gtid", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3779))
;
3780
3781 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, ((((0) >= 0 && (root->r.r_hot_team
) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (root->r.r_hot_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3783)), root->r.r_hot_team->t.t_threads[(0)]->th.th_info
.ds.ds_gtid), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3782 "plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, ((((0) >= 0 && (root->r.r_hot_team
) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (root->r.r_hot_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3783)), root->r.r_hot_team->t.t_threads[(0)]->th.th_info
.ds.ds_gtid), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3783 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, ((((0) >= 0 && (root->r.r_hot_team
) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (root->r.r_hot_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3783)), root->r.r_hot_team->t.t_threads[(0)]->th.th_info
.ds.ds_gtid), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3784 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, ((((0) >= 0 && (root->r.r_hot_team
) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (root->r.r_hot_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3783)), root->r.r_hot_team->t.t_threads[(0)]->th.th_info
.ds.ds_gtid), root->r.r_hot_team->t.t_id, 0, 0, 0); }
3785 KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
"plain=%u\n", gtid, ((((0) >= 0 && (root->r.r_hot_team
) != __null) ? 0 : __kmp_debug_assert("(0) >= 0 && (root->r.r_hot_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3783)), root->r.r_hot_team->t.t_threads[(0)]->th.th_info
.ds.ds_gtid), root->r.r_hot_team->t.t_id, 0, 0, 0); }
;
3786 { // Initialize barrier data.
3787 int b;
3788 for (b = 0; b < bs_last_barrier; ++b) {
3789 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE0;
3790#if USE_DEBUGGER0
3791 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3792#endif
3793 }
3794 }
3795 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==((root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived
== 0) ? 0 : __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3796))
3796 KMP_INIT_BARRIER_STATE)((root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived
== 0) ? 0 : __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3796))
;
3797
3798#if KMP_AFFINITY_SUPPORTED1
3799#if OMP_40_ENABLED(50 >= 40)
3800 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
3801 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
3802 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
3803 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
3804#endif
3805
3806 if (TCR_4(__kmp_init_middle)(__kmp_init_middle)) {
3807 __kmp_affinity_set_init_mask(gtid, TRUE(!0));
3808 }
3809#endif /* KMP_AFFINITY_SUPPORTED */
3810
3811 __kmp_root_counter++;
3812
3813#if OMPT_SUPPORT1
3814 if (!initial_thread && ompt_enabled.enabled) {
3815
3816 ompt_thread_t *root_thread = ompt_get_thread();
3817
3818 ompt_set_thread_state(root_thread, omp_state_overhead);
3819
3820 if (ompt_enabled.ompt_callback_thread_begin) {
3821 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)ompt_callback_thread_begin_callback(
3822 ompt_thread_initial, __ompt_get_thread_data_internal());
3823 }
3824 ompt_data_t *task_data;
3825 __ompt_get_task_info_internal(0, NULL__null, &task_data, NULL__null, NULL__null, NULL__null);
3826 if (ompt_enabled.ompt_callback_task_create) {
3827 ompt_callbacks.ompt_callback(ompt_callback_task_create)ompt_callback_task_create_callback(
3828 NULL__null, NULL__null, task_data, ompt_task_initial, 0, NULL__null);
3829 // initial task has nothing to return to
3830 }
3831
3832 ompt_set_thread_state(root_thread, omp_state_work_serial);
3833 }
3834#endif
3835
3836 KMP_MB();
3837 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3838
3839 return gtid;
3840}
3841
3842#if KMP_NESTED_HOT_TEAMS1
3843static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
3844 const int max_level) {
3845 int i, n, nth;
3846 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3847 if (!hot_teams || !hot_teams[level].hot_team) {
3848 return 0;
3849 }
3850 KMP_DEBUG_ASSERT(level < max_level)((level < max_level) ? 0 : __kmp_debug_assert("level < max_level"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3850))
;
3851 kmp_team_t *team = hot_teams[level].hot_team;
3852 nth = hot_teams[level].hot_team_nth;
3853 n = nth - 1; // master is not freed
3854 if (level < max_level - 1) {
3855 for (i = 0; i < nth; ++i) {
3856 kmp_info_t *th = team->t.t_threads[i];
3857 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3858 if (i > 0 && th->th.th_hot_teams) {
3859 __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3859)
;
3860 th->th.th_hot_teams = NULL__null;
3861 }
3862 }
3863 }
3864 __kmp_free_team(root, team, NULL__null);
3865 return n;
3866}
3867#endif
3868
3869// Resets a root thread and clear its root and hot teams.
3870// Returns the number of __kmp_threads entries directly and indirectly freed.
3871static int __kmp_reset_root(int gtid, kmp_root_t *root) {
3872 kmp_team_t *root_team = root->r.r_root_team;
3873 kmp_team_t *hot_team = root->r.r_hot_team;
3874 int n = hot_team->t.t_nproc;
3875 int i;
3876
3877 KMP_DEBUG_ASSERT(!root->r.r_active)((!root->r.r_active) ? 0 : __kmp_debug_assert("!root->r.r_active"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3877))
;
3878
3879 root->r.r_root_team = NULL__null;
3880 root->r.r_hot_team = NULL__null;
3881 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team
3882 // before call to __kmp_free_team().
3883 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL), __null);
3884#if KMP_NESTED_HOT_TEAMS1
3885 if (__kmp_hot_teams_max_level >
3886 0) { // need to free nested hot teams and their threads if any
3887 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3888 kmp_info_t *th = hot_team->t.t_threads[i];
3889 if (__kmp_hot_teams_max_level > 1) {
3890 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3891 }
3892 if (th->th.th_hot_teams) {
3893 __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3893)
;
3894 th->th.th_hot_teams = NULL__null;
3895 }
3896 }
3897 }
3898#endif
3899 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL), __null);
3900
3901 // Before we can reap the thread, we need to make certain that all other
3902 // threads in the teams that had this root as ancestor have stopped trying to
3903 // steal tasks.
3904 if (__kmp_tasking_mode != tskm_immediate_exec) {
3905 __kmp_wait_to_unref_task_teams();
3906 }
3907
3908#if KMP_OS_WINDOWS0
3909 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
3910 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3911 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPECif (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3912 "\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3913 (LPVOID) & (root->r.r_uber_thread->th),if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
3914 root->r.r_uber_thread->th.th_info.ds.ds_thread))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
"lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
root->r.r_uber_thread->th.th_info.ds.ds_thread); }
;
3915 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3916#endif /* KMP_OS_WINDOWS */
3917
3918#if OMPT_SUPPORT1
3919 if (ompt_enabled.ompt_callback_thread_end) {
3920 ompt_callbacks.ompt_callback(ompt_callback_thread_end)ompt_callback_thread_end_callback(
3921 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
3922 }
3923#endif
3924
3925 TCW_4(__kmp_nth,(__kmp_nth) = (__kmp_nth - 1)
3926 __kmp_nth - 1)(__kmp_nth) = (__kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
3927 root->r.r_cg_nthreads--;
3928
3929 __kmp_reap_thread(root->r.r_uber_thread, 1);
3930
3931 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead
3932 // of freeing.
3933 root->r.r_uber_thread = NULL__null;
3934 /* mark root as no longer in use */
3935 root->r.r_begin = FALSE0;
3936
3937 return n;
3938}
3939
3940void __kmp_unregister_root_current_thread(int gtid) {
3941 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: enter T#%d\n"
, gtid); }
;
3942 /* this lock should be ok, since unregister_root_current_thread is never
3943 called during an abort, only during a normal close. furthermore, if you
3944 have the forkjoin lock, you should never try to get the initz lock */
3945 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3946 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
3947 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
3948 "exiting T#%d\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
3949 gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
"exiting T#%d\n", gtid); }
;
3950 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3951 return;
3952 }
3953 kmp_root_t *root = __kmp_root[gtid];
3954
3955 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])((__kmp_threads && __kmp_threads[gtid]) ? 0 : __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3955))
;
3956 KMP_ASSERT(KMP_UBER_GTID(gtid))((((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3956)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3956)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))) ? 0 : __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3956))
;
3957 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)((root == __kmp_threads[gtid]->th.th_root) ? 0 : __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3957))
;
3958 KMP_ASSERT(root->r.r_active == FALSE)((root->r.r_active == 0) ? 0 : __kmp_debug_assert("root->r.r_active == FALSE"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 3958))
;
3959
3960 KMP_MB();
3961
3962#if OMP_45_ENABLED(50 >= 45)
3963 kmp_info_t *thread = __kmp_threads[gtid];
3964 kmp_team_t *team = thread->th.th_team;
3965 kmp_task_team_t *task_team = thread->th.th_task_team;
3966
3967 // we need to wait for the proxy tasks before finishing the thread
3968 if (task_team != NULL__null && task_team->tt.tt_found_proxy_tasks) {
3969#if OMPT_SUPPORT1
3970 // the runtime is shutting down so we won't report any events
3971 thread->th.ompt_thread_info.state = omp_state_undefined;
3972#endif
3973 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL), __null);
3974 }
3975#endif
3976
3977 __kmp_reset_root(gtid, root);
3978
3979 /* free up this thread slot */
3980 __kmp_gtid_set_specific(KMP_GTID_DNE(-2));
3981#ifdef KMP_TDATA_GTID1
3982 __kmp_gtid = KMP_GTID_DNE(-2);
3983#endif
3984
3985 KMP_MB();
3986 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); }
3987 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); }
;
3988
3989 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3990}
3991
3992#if KMP_OS_WINDOWS0
3993/* __kmp_forkjoin_lock must be already held
3994 Unregisters a root thread that is not the current thread. Returns the number
3995 of __kmp_threads entries freed as a result. */
3996static int __kmp_unregister_root_other_thread(int gtid) {
3997 kmp_root_t *root = __kmp_root[gtid];
3998 int r;
3999
4000 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: enter T#%d\n"
, gtid); }
;
4001 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])((__kmp_threads && __kmp_threads[gtid]) ? 0 : __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4001))
;
4002 KMP_ASSERT(KMP_UBER_GTID(gtid))((((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4002)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4002)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))) ? 0 : __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4002))
;
4003 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)((root == __kmp_threads[gtid]->th.th_root) ? 0 : __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4003))
;
4004 KMP_ASSERT(root->r.r_active == FALSE)((root->r.r_active == 0) ? 0 : __kmp_debug_assert("root->r.r_active == FALSE"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4004))
;
4005
4006 r = __kmp_reset_root(gtid, root);
4007 KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); }
4008 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); }
;
4009 return r;
4010}
4011#endif
4012
4013#if KMP_DEBUG1
4014void __kmp_task_info() {
4015
4016 kmp_int32 gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
4017 kmp_int32 tid = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4017)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
4018 kmp_info_t *this_thr = __kmp_threads[gtid];
4019 kmp_team_t *steam = this_thr->th.th_serial_team;
4020 kmp_team_t *team = this_thr->th.th_team;
4021
4022 __kmp_printf("__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p curtask=%p "
4023 "ptask=%p\n",
4024 gtid, tid, this_thr, team, this_thr->th.th_current_task,
4025 team->t.t_implicit_task_taskdata[tid].td_parent);
4026}
4027#endif // KMP_DEBUG
4028
4029/* TODO optimize with one big memclr, take out what isn't needed, split
4030 responsibility to workers as much as possible, and delay initialization of
4031 features as much as possible */
4032static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4033 int tid, int gtid) {
4034 /* this_thr->th.th_info.ds.ds_gtid is setup in
4035 kmp_allocate_thread/create_worker.
4036 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
4037 kmp_info_t *master = team->t.t_threads[0];
4038 KMP_DEBUG_ASSERT(this_thr != NULL)((this_thr != __null) ? 0 : __kmp_debug_assert("this_thr != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4038))
;
4039 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team)((this_thr->th.th_serial_team) ? 0 : __kmp_debug_assert("this_thr->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4039))
;
4040 KMP_DEBUG_ASSERT(team)((team) ? 0 : __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4040))
;
4041 KMP_DEBUG_ASSERT(team->t.t_threads)((team->t.t_threads) ? 0 : __kmp_debug_assert("team->t.t_threads"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4041))
;
4042 KMP_DEBUG_ASSERT(team->t.t_dispatch)((team->t.t_dispatch) ? 0 : __kmp_debug_assert("team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4042))
;
4043 KMP_DEBUG_ASSERT(master)((master) ? 0 : __kmp_debug_assert("master", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4043))
;
4044 KMP_DEBUG_ASSERT(master->th.th_root)((master->th.th_root) ? 0 : __kmp_debug_assert("master->th.th_root"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4044))
;
4045
4046 KMP_MB();
4047
4048 TCW_SYNC_PTR(this_thr->th.th_team, team)((this_thr->th.th_team)) = ((team));
4049
4050 this_thr->th.th_info.ds.ds_tid = tid;
4051 this_thr->th.th_set_nproc = 0;
4052 if (__kmp_tasking_mode != tskm_immediate_exec)
4053 // When tasking is possible, threads are not safe to reap until they are
4054 // done tasking; this will be set when tasking code is exited in wait
4055 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP0;
4056 else // no tasking --> always safe to reap
4057 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP1;
4058#if OMP_40_ENABLED(50 >= 40)
4059 this_thr->th.th_set_proc_bind = proc_bind_default;
4060#if KMP_AFFINITY_SUPPORTED1
4061 this_thr->th.th_new_place = this_thr->th.th_current_place;
4062#endif
4063#endif
4064 this_thr->th.th_root = master->th.th_root;
4065
4066 /* setup the thread's cache of the team structure */
4067 this_thr->th.th_team_nproc = team->t.t_nproc;
4068 this_thr->th.th_team_master = master;
4069 this_thr->th.th_team_serialized = team->t.t_serialized;
4070 TCW_PTR(this_thr->th.th_sleep_loc, NULL)((this_thr->th.th_sleep_loc)) = ((__null));
4071
4072 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata)((team->t.t_implicit_task_taskdata) ? 0 : __kmp_debug_assert
("team->t.t_implicit_task_taskdata", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4072))
;
4073
4074 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
4075 tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
;
4076
4077 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4078 team, tid, TRUE(!0));
4079
4080 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
4081 tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
;
4082 // TODO: Initialize ICVs from parent; GEH - isn't that already done in
4083 // __kmp_initialize_team()?
4084
4085 /* TODO no worksharing in speculative threads */
4086 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4087
4088 this_thr->th.th_local.this_construct = 0;
4089
4090 if (!this_thr->th.th_pri_common) {
4091 this_thr->th.th_pri_common =
4092 (struct common_table *)__kmp_allocate(sizeof(struct common_table))___kmp_allocate((sizeof(struct common_table)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4092)
;
4093 if (__kmp_storage_map) {
4094 __kmp_print_storage_map_gtid(
4095 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4096 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
4097 }
4098 this_thr->th.th_pri_head = NULL__null;
4099 }
4100
4101 /* Initialize dynamic dispatch */
4102 {
4103 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4104 // Use team max_nproc since this will never change for the team.
4105 size_t disp_size =
4106 sizeof(dispatch_private_info_t) *
4107 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4108 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); }
4109 team->t.t_max_nproc))if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); }
;
4110 KMP_ASSERT(dispatch)((dispatch) ? 0 : __kmp_debug_assert("dispatch", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4110))
;
4111 KMP_DEBUG_ASSERT(team->t.t_dispatch)((team->t.t_dispatch) ? 0 : __kmp_debug_assert("team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4111))
;
4112 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid])((dispatch == &team->t.t_dispatch[tid]) ? 0 : __kmp_debug_assert
("dispatch == &team->t.t_dispatch[tid]", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4112))
;
4113
4114 dispatch->th_disp_index = 0;
4115#if OMP_45_ENABLED(50 >= 45)
4116 dispatch->th_doacross_buf_idx = 0;
4117#endif
4118 if (!dispatch->th_disp_buffer) {
4119 dispatch->th_disp_buffer =
4120 (dispatch_private_info_t *)__kmp_allocate(disp_size)___kmp_allocate((disp_size), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4120)
;
4121
4122 if (__kmp_storage_map) {
4123 __kmp_print_storage_map_gtid(
4124 gtid, &dispatch->th_disp_buffer[0],
4125 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4126 ? 1
4127 : __kmp_dispatch_num_buffers],
4128 disp_size, "th_%d.th_dispatch.th_disp_buffer "
4129 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4130 gtid, team->t.t_id, gtid);
4131 }
4132 } else {
4133 memset(&dispatch->th_disp_buffer[0], '\0', disp_size);
4134 }
4135
4136 dispatch->th_dispatch_pr_current = 0;
4137 dispatch->th_dispatch_sh_current = 0;
4138
4139 dispatch->th_deo_fcn = 0; /* ORDERED */
4140 dispatch->th_dxo_fcn = 0; /* END ORDERED */
4141 }
4142
4143 this_thr->th.th_next_pool = NULL__null;
4144
4145 if (!this_thr->th.th_task_state_memo_stack) {
4146 size_t i;
4147 this_thr->th.th_task_state_memo_stack =
4148 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8))___kmp_allocate((4 * sizeof(kmp_uint8)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4148)
;
4149 this_thr->th.th_task_state_top = 0;
4150 this_thr->th.th_task_state_stack_sz = 4;
4151 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4152 ++i) // zero init the stack
4153 this_thr->th.th_task_state_memo_stack[i] = 0;
4154 }
4155
4156 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here)((!this_thr->th.th_spin_here) ? 0 : __kmp_debug_assert("!this_thr->th.th_spin_here"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4156))
;
4157 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0)((this_thr->th.th_next_waiting == 0) ? 0 : __kmp_debug_assert
("this_thr->th.th_next_waiting == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4157))
;
4158
4159 KMP_MB();
4160}
4161
4162/* allocate a new thread for the requesting team. this is only called from
4163 within a forkjoin critical section. we will first try to get an available
4164 thread from the thread pool. if none is available, we will fork a new one
4165 assuming we are able to create a new one. this should be assured, as the
4166 caller should check on this first. */
4167kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4168 int new_tid) {
4169 kmp_team_t *serial_team;
4170 kmp_info_t *new_thr;
4171 int new_gtid;
4172
4173 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d\n"
, __kmp_get_global_thread_id()); }
;
4174 KMP_DEBUG_ASSERT(root && team)((root && team) ? 0 : __kmp_debug_assert("root && team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4174))
;
4175#if !KMP_NESTED_HOT_TEAMS1
4176 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()))((((((((__kmp_get_global_thread_id())) >= 0) ? 0 : __kmp_debug_assert
("((__kmp_get_global_thread_id())) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4176)), __kmp_threads[((__kmp_get_global_thread_id()))]->
th.th_info.ds.ds_tid) == 0)) ? 0 : __kmp_debug_assert("((((((__kmp_get_global_thread_id())) >= 0) ? 0 : __kmp_debug_assert(\"((__kmp_get_global_thread_id())) >= 0\", \"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp\", 4176)), __kmp_threads[((__kmp_get_global_thread_id()))]->th.th_info.ds.ds_tid) == 0)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4176))
;
4177#endif
4178 KMP_MB();
4179
4180 /* first, try to get one from the thread pool */
4181 if (__kmp_thread_pool) {
4182
4183 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool);
4184 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;
4185 if (new_thr == __kmp_thread_pool_insert_pt) {
4186 __kmp_thread_pool_insert_pt = NULL__null;
4187 }
4188 TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
4189 // Don't touch th_active_in_pool or th_active.
4190 // The worker thread adjusts those flags as it sleeps/awakens.
4191 __kmp_thread_pool_nth--;
4192
4193 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); }
4194 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); }
;
4195 KMP_ASSERT(!new_thr->th.th_team)((!new_thr->th.th_team) ? 0 : __kmp_debug_assert("!new_thr->th.th_team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4195))
;
4196 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity)((__kmp_nth < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("__kmp_nth < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4196))
;
4197 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0)((__kmp_thread_pool_nth >= 0) ? 0 : __kmp_debug_assert("__kmp_thread_pool_nth >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4197))
;
4198
4199 /* setup the thread structure */
4200 __kmp_initialize_info(new_thr, team, new_tid,
4201 new_thr->th.th_info.ds.ds_gtid);
4202 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team)((new_thr->th.th_serial_team) ? 0 : __kmp_debug_assert("new_thr->th.th_serial_team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4202))
;
4203
4204 TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);
4205 root->r.r_cg_nthreads++;
4206
4207 new_thr->th.th_task_state = 0;
4208 new_thr->th.th_task_state_top = 0;
4209 new_thr->th.th_task_state_stack_sz = 4;
4210
4211#ifdef KMP_ADJUST_BLOCKTIME1
4212 /* Adjust blocktime back to zero if necessary */
4213 /* Middle initialization might not have occurred yet */
4214 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4215 if (__kmp_nth > __kmp_avail_proc) {
4216 __kmp_zero_bt = TRUE(!0);
4217 }
4218 }
4219#endif /* KMP_ADJUST_BLOCKTIME */
4220
4221#if KMP_DEBUG1
4222 // If thread entered pool via __kmp_free_thread, wait_flag should !=
4223 // KMP_BARRIER_PARENT_FLAG.
4224 int b;
4225 kmp_balign_t *balign = new_thr->th.th_bar;
4226 for (b = 0; b < bs_last_barrier; ++b)
4227 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)((balign[b].bb.wait_flag != 2) ? 0 : __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4227))
;
4228#endif
4229
4230 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); }
4231 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); }
;
4232
4233 KMP_MB();
4234 return new_thr;
4235 }
4236
4237 /* no, well fork a new one */
4238 KMP_ASSERT(__kmp_nth == __kmp_all_nth)((__kmp_nth == __kmp_all_nth) ? 0 : __kmp_debug_assert("__kmp_nth == __kmp_all_nth"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4238))
;
4239 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity)((__kmp_all_nth < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("__kmp_all_nth < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4239))
;
4240
4241#if KMP_USE_MONITOR
4242 // If this is the first worker thread the RTL is creating, then also
4243 // launch the monitor thread. We try to do this as early as possible.
4244 if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
4245 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4246 if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
4247 KF_TRACE(10, ("before __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("before __kmp_create_monitor\n"
); }
;
4248 TCW_4(__kmp_init_monitor, 1)(__kmp_init_monitor) = (1);
4249 __kmp_create_monitor(&__kmp_monitor);
4250 KF_TRACE(10, ("after __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after __kmp_create_monitor\n"
); }
;
4251#if KMP_OS_WINDOWS0
4252 // AC: wait until monitor has started. This is a fix for CQ232808.
4253 // The reason is that if the library is loaded/unloaded in a loop with
4254 // small (parallel) work in between, then there is high probability that
4255 // monitor thread started after the library shutdown. At shutdown it is
4256 // too late to cope with the problem, because when the master is in
4257 // DllMain (process detach) the monitor has no chances to start (it is
4258 // blocked), and master has no means to inform the monitor that the
4259 // library has gone, because all the memory which the monitor can access
4260 // is going to be released/reset.
4261 while (TCR_4(__kmp_init_monitor)(__kmp_init_monitor) < 2) {
4262 KMP_YIELD(TRUE){ __kmp_x86_pause(); __kmp_yield(((!0))); };
4263 }
4264 KF_TRACE(10, ("after monitor thread has started\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after monitor thread has started\n"
); }
;
4265#endif
4266 }
4267 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4268 }
4269#endif
4270
4271 KMP_MB();
4272 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid])((void *)(__kmp_threads[new_gtid])) != NULL__null; ++new_gtid) {
4273 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity)((new_gtid < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("new_gtid < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4273))
;
4274 }
4275
4276 /* allocate space for it. */
4277 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4277)
;
4278
4279 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr)((__kmp_threads[new_gtid])) = ((new_thr));
4280
4281 if (__kmp_storage_map) {
4282 __kmp_print_thread_storage_map(new_thr, new_gtid);
4283 }
4284
4285 // add the reserve serialized team, initialized from the team's master thread
4286 {
4287 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4288 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before th_serial/serial_team\n"
); }
;
4289 new_thr->th.th_serial_team = serial_team =
4290 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4291#if OMPT_SUPPORT1
4292 ompt_data_none, // root parallel id
4293#endif
4294#if OMP_40_ENABLED(50 >= 40)
4295 proc_bind_default,
4296#endif
4297 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL), __null);
4298 }
4299 KMP_ASSERT(serial_team)((serial_team) ? 0 : __kmp_debug_assert("serial_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4299))
;
4300 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for
4301 // execution (it is unused for now).
4302 serial_team->t.t_threads[0] = new_thr;
4303 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
4304 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
4305 new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
;
4306
4307 /* setup the thread structures */
4308 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4309
4310#if USE_FAST_MEMORY3
4311 __kmp_initialize_fast_memory(new_thr);
4312#endif /* USE_FAST_MEMORY */
4313
4314#if KMP_USE_BGET1
4315 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL)((new_thr->th.th_local.bget_data == __null) ? 0 : __kmp_debug_assert
("new_thr->th.th_local.bget_data == __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4315))
;
4316 __kmp_initialize_bget(new_thr);
4317#endif
4318
4319 __kmp_init_random(new_thr); // Initialize random number generator
4320
4321 /* Initialize these only once when thread is grabbed for a team allocation */
4322 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
4323 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
4324 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
;
4325
4326 int b;
4327 kmp_balign_t *balign = new_thr->th.th_bar;
4328 for (b = 0; b < bs_last_barrier; ++b) {
4329 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE0;
4330 balign[b].bb.team = NULL__null;
4331 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING0;
4332 balign[b].bb.use_oncore_barrier = 0;
4333 }
4334
4335 new_thr->th.th_spin_here = FALSE0;
4336 new_thr->th.th_next_waiting = 0;
4337#if KMP_OS_UNIX1
4338 new_thr->th.th_blocking = false;
4339#endif
4340
4341#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
4342 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
4343 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
4344 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
4345 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
4346#endif
4347
4348 TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
4349 new_thr->th.th_active_in_pool = FALSE0;
4350 TCW_4(new_thr->th.th_active, TRUE)(new_thr->th.th_active) = ((!0));
4351
4352 /* adjust the global counters */
4353 __kmp_all_nth++;
4354 __kmp_nth++;
4355
4356 root->r.r_cg_nthreads++;
4357
4358 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
4359 // numbers of procs, and method #2 (keyed API call) for higher numbers.
4360 if (__kmp_adjust_gtid_mode) {
4361 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4362 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
4363 TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
4364 }
4365 } else {
4366 if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
4367 TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
4368 }
4369 }
4370 }
4371
4372#ifdef KMP_ADJUST_BLOCKTIME1
4373 /* Adjust blocktime back to zero if necessary */
4374 /* Middle initialization might not have occurred yet */
4375 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4376 if (__kmp_nth > __kmp_avail_proc) {
4377 __kmp_zero_bt = TRUE(!0);
4378 }
4379 }
4380#endif /* KMP_ADJUST_BLOCKTIME */
4381
4382 /* actually fork it and create the new worker thread */
4383 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); }
4384 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); }
;
4385 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4386 KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); }
4387 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); }
;
4388
4389 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); }
4390 new_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); }
;
4391 KMP_MB();
4392 return new_thr;
4393}
4394
4395/* Reinitialize team for reuse.
4396 The hot team code calls this case at every fork barrier, so EPCC barrier
4397 test are extremely sensitive to changes in it, esp. writes to the team
4398 struct, which cause a cache invalidation in all threads.
4399 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4400static void __kmp_reinitialize_team(kmp_team_t *team,
4401 kmp_internal_control_t *new_icvs,
4402 ident_t *loc) {
4403 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
4404 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
;
4405 KMP_DEBUG_ASSERT(team && new_icvs)((team && new_icvs) ? 0 : __kmp_debug_assert("team && new_icvs"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4405))
;
4406 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc)(((!(__kmp_init_parallel)) || new_icvs->nproc) ? 0 : __kmp_debug_assert
("(!(__kmp_init_parallel)) || new_icvs->nproc", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4406))
;
4407 KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
)
;
4408
4409 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID())if ((team->t.t_id) != ((~0))) (team->t.t_id) = ((~0));
4410 // Copy ICVs to the master thread's implicit taskdata
4411 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE0);
4412 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4413
4414 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
4415 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
;
4416}
4417
4418/* Initialize the team data structure.
4419 This assumes the t_threads and t_max_nproc are already set.
4420 Also, we don't touch the arguments */
4421static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
4422 kmp_internal_control_t *new_icvs,
4423 ident_t *loc) {
4424 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: enter: team=%p\n"
, team); }
;
4425
4426 /* verify */
4427 KMP_DEBUG_ASSERT(team)((team) ? 0 : __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4427))
;
4428 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc)((new_nproc <= team->t.t_max_nproc) ? 0 : __kmp_debug_assert
("new_nproc <= team->t.t_max_nproc", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4428))
;
4429 KMP_DEBUG_ASSERT(team->t.t_threads)((team->t.t_threads) ? 0 : __kmp_debug_assert("team->t.t_threads"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4429))
;
4430 KMP_MB();
4431
4432 team->t.t_master_tid = 0; /* not needed */
4433 /* team->t.t_master_bar; not needed */
4434 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4435 team->t.t_nproc = new_nproc;
4436
4437 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */
4438 team->t.t_next_pool = NULL__null;
4439 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess
4440 * up hot team */
4441
4442 TCW_SYNC_PTR(team->t.t_pkfn, NULL)((team->t.t_pkfn)) = ((__null)); /* not needed */
4443 team->t.t_invoke = NULL__null; /* not needed */
4444
4445 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4446 team->t.t_sched.sched = new_icvs->sched.sched;
4447
4448#if KMP_ARCH_X860 || KMP_ARCH_X86_641
4449 team->t.t_fp_control_saved = FALSE0; /* not needed */
4450 team->t.t_x87_fpu_control_word = 0; /* not needed */
4451 team->t.t_mxcsr = 0; /* not needed */
4452#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4453
4454 team->t.t_construct = 0;
4455
4456 team->t.t_ordered.dt.t_value = 0;
4457 team->t.t_master_active = FALSE0;
4458
4459 memset(&team->t.t_taskq, '\0', sizeof(kmp_taskq_t));
4460
4461#ifdef KMP_DEBUG1
4462 team->t.t_copypriv_data = NULL__null; /* not necessary, but nice for debugging */
4463#endif
4464#if KMP_OS_WINDOWS0
4465 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
4466#endif
4467
4468 team->t.t_control_stack_top = NULL__null;
4469
4470 __kmp_reinitialize_team(team, new_icvs, loc);
4471
4472 KMP_MB();
4473 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: exit: team=%p\n"
, team); }
;
4474}
4475
4476#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
4477/* Sets full mask for thread and returns old mask, no changes to structures. */
4478static void
4479__kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4480 if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
4481 int status;
4482 if (old_mask != NULL__null) {
4483 status = __kmp_get_system_affinity(old_mask, TRUE)(old_mask)->get_system_affinity((!0));
4484 int error = errno(*__errno_location ());
4485 if (status != 0) {
4486 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError)__kmp_msg_format(kmp_i18n_msg_ChangeThreadAffMaskError), KMP_ERR(error)__kmp_msg_error_code(error),
4487 __kmp_msg_null);
4488 }
4489 }
4490 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE)(__kmp_affin_fullMask)->set_system_affinity((!0));
4491 }
4492}
4493#endif
4494
4495#if OMP_40_ENABLED(50 >= 40) && KMP_AFFINITY_SUPPORTED1
4496
4497// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
4498// It calculats the worker + master thread's partition based upon the parent
4499// thread's partition, and binds each worker to a thread in their partition.
4500// The master thread's partition should already include its current binding.
4501static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
4502 // Copy the master thread's place partion to the team struct
4503 kmp_info_t *master_th = team->t.t_threads[0];
4504 KMP_DEBUG_ASSERT(master_th != NULL)((master_th != __null) ? 0 : __kmp_debug_assert("master_th != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4504))
;
4505 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4506 int first_place = master_th->th.th_first_place;
4507 int last_place = master_th->th.th_last_place;
4508 int masters_place = master_th->th.th_current_place;
4509 team->t.t_first_place = first_place;
4510 team->t.t_last_place = last_place;
4511
4512 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, ((((team
->t.t_threads[0]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[0]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4514)), (team->t.t_threads[0])->th.th_info.ds.ds_gtid
), team->t.t_id, masters_place, first_place, last_place); }
4513 "bound to place %d partition = [%d,%d]\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, ((((team
->t.t_threads[0]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[0]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4514)), (team->t.t_threads[0])->th.th_info.ds.ds_gtid
), team->t.t_id, masters_place, first_place, last_place); }
4514 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, ((((team
->t.t_threads[0]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[0]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4514)), (team->t.t_threads[0])->th.th_info.ds.ds_gtid
), team->t.t_id, masters_place, first_place, last_place); }
4515 team->t.t_id, masters_place, first_place, last_place))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
"bound to place %d partition = [%d,%d]\n", proc_bind, ((((team
->t.t_threads[0]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[0]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4514)), (team->t.t_threads[0])->th.th_info.ds.ds_gtid
), team->t.t_id, masters_place, first_place, last_place); }
;
4516
4517 switch (proc_bind) {
4518
4519 case proc_bind_default:
4520 // serial teams might have the proc_bind policy set to proc_bind_default. It
4521 // doesn't matter, as we don't rebind master thread for any proc_bind policy
4522 KMP_DEBUG_ASSERT(team->t.t_nproc == 1)((team->t.t_nproc == 1) ? 0 : __kmp_debug_assert("team->t.t_nproc == 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4522))
;
4523 break;
4524
4525 case proc_bind_master: {
4526 int f;
4527 int n_th = team->t.t_nproc;
4528 for (f = 1; f < n_th; f++) {
4529 kmp_info_t *th = team->t.t_threads[f];
4530 KMP_DEBUG_ASSERT(th != NULL)((th != __null) ? 0 : __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4530))
;
4531 th->th.th_first_place = first_place;
4532 th->th.th_last_place = last_place;
4533 th->th.th_new_place = masters_place;
4534
4535 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4537)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, masters_place, first_place, last_place
); }
4536 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4537)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, masters_place, first_place, last_place
); }
4537 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4537)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, masters_place, first_place, last_place
); }
4538 f, masters_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4537)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, masters_place, first_place, last_place
); }
;
4539 }
4540 } break;
4541
4542 case proc_bind_close: {
4543 int f;
4544 int n_th = team->t.t_nproc;
4545 int n_places;
4546 if (first_place <= last_place) {
4547 n_places = last_place - first_place + 1;
4548 } else {
4549 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4550 }
4551 if (n_th <= n_places) {
4552 int place = masters_place;
4553 for (f = 1; f < n_th; f++) {
4554 kmp_info_t *th = team->t.t_threads[f];
4555 KMP_DEBUG_ASSERT(th != NULL)((th != __null) ? 0 : __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4555))
;
4556
4557 if (place == last_place) {
4558 place = first_place;
4559 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4560 place = 0;
4561 } else {
4562 place++;
4563 }
4564 th->th.th_first_place = first_place;
4565 th->th.th_last_place = last_place;
4566 th->th.th_new_place = place;
4567
4568 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4570)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, place, first_place, last_place); }
4569 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4570)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, place, first_place, last_place); }
4570 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4570)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, place, first_place, last_place); }
4571 team->t.t_id, f, place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4570)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, place, first_place, last_place); }
;
4572 }
4573 } else {
4574 int S, rem, gap, s_count;
4575 S = n_th / n_places;
4576 s_count = 0;
4577 rem = n_th - (S * n_places);
4578 gap = rem > 0 ? n_places / rem : n_places;
4579 int place = masters_place;
4580 int gap_ct = gap;
4581 for (f = 0; f < n_th; f++) {
4582 kmp_info_t *th = team->t.t_threads[f];
4583 KMP_DEBUG_ASSERT(th != NULL)((th != __null) ? 0 : __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4583))
;
4584
4585 th->th.th_first_place = first_place;
4586 th->th.th_last_place = last_place;
4587 th->th.th_new_place = place;
4588 s_count++;
4589
4590 if ((s_count == S) && rem && (gap_ct == gap)) {
4591 // do nothing, add an extra thread to place on next iteration
4592 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4593 // we added an extra thread to this place; move to next place
4594 if (place == last_place) {
4595 place = first_place;
4596 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4597 place = 0;
4598 } else {
4599 place++;
4600 }
4601 s_count = 0;
4602 gap_ct = 1;
4603 rem--;
4604 } else if (s_count == S) { // place full; don't add extra
4605 if (place == last_place) {
4606 place = first_place;
4607 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4608 place = 0;
4609 } else {
4610 place++;
4611 }
4612 gap_ct++;
4613 s_count = 0;
4614 }
4615
4616 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4619)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, first_place, last_place
); }
4617 ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4619)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, first_place, last_place
); }
4618 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4619)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, first_place, last_place
); }
4619 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4619)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, first_place, last_place
); }
4620 th->th.th_new_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4619)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, first_place, last_place
); }
;
4621 }
4622 KMP_DEBUG_ASSERT(place == masters_place)((place == masters_place) ? 0 : __kmp_debug_assert("place == masters_place"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4622))
;
4623 }
4624 } break;
4625
4626 case proc_bind_spread: {
4627 int f;
4628 int n_th = team->t.t_nproc;
4629 int n_places;
4630 int thidx;
4631 if (first_place <= last_place) {
4632 n_places = last_place - first_place + 1;
4633 } else {
4634 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4635 }
4636 if (n_th <= n_places) {
4637 int place = -1;
4638
4639 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4640 int S = n_places / n_th;
4641 int s_count, rem, gap, gap_ct;
4642
4643 place = masters_place;
4644 rem = n_places - n_th * S;
4645 gap = rem ? n_th / rem : 1;
4646 gap_ct = gap;
4647 thidx = n_th;
4648 if (update_master_only == 1)
4649 thidx = 1;
4650 for (f = 0; f < thidx; f++) {
4651 kmp_info_t *th = team->t.t_threads[f];
4652 KMP_DEBUG_ASSERT(th != NULL)((th != __null) ? 0 : __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4652))
;
4653
4654 th->th.th_first_place = place;
4655 th->th.th_new_place = place;
4656 s_count = 1;
4657 while (s_count < S) {
4658 if (place == last_place) {
4659 place = first_place;
4660 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4661 place = 0;
4662 } else {
4663 place++;
4664 }
4665 s_count++;
4666 }
4667 if (rem && (gap_ct == gap)) {
4668 if (place == last_place) {
4669 place = first_place;
4670 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4671 place = 0;
4672 } else {
4673 place++;
4674 }
4675 rem--;
4676 gap_ct = 0;
4677 }
4678 th->th.th_last_place = place;
4679 gap_ct++;
4680
4681 if (place == last_place) {
4682 place = first_place;
4683 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4684 place = 0;
4685 } else {
4686 place++;
4687 }
4688
4689 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", ((((team
->t.t_threads[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4692)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, __kmp_affinity_num_masks); }
4690 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", ((((team
->t.t_threads[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4692)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, __kmp_affinity_num_masks); }
4691 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", ((((team
->t.t_threads[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4692)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, __kmp_affinity_num_masks); }
4692 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", ((((team
->t.t_threads[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4692)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, __kmp_affinity_num_masks); }
4693 f, th->th.th_new_place, th->th.th_first_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", ((((team
->t.t_threads[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4692)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, __kmp_affinity_num_masks); }
4694 th->th.th_last_place, __kmp_affinity_num_masks))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], __kmp_affinity_num_masks: %u\n", ((((team
->t.t_threads[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4692)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, __kmp_affinity_num_masks); }
;
4695 }
4696 } else {
4697 /* Having uniform space of available computation places I can create
4698 T partitions of round(P/T) size and put threads into the first
4699 place of each partition. */
4700 double current = static_cast<double>(masters_place);
4701 double spacing =
4702 (static_cast<double>(n_places + 1) / static_cast<double>(n_th));
4703 int first, last;
4704 kmp_info_t *th;
4705
4706 thidx = n_th + 1;
4707 if (update_master_only == 1)
4708 thidx = 1;
4709 for (f = 0; f < thidx; f++) {
4710 first = static_cast<int>(current);
4711 last = static_cast<int>(current + spacing) - 1;
4712 KMP_DEBUG_ASSERT(last >= first)((last >= first) ? 0 : __kmp_debug_assert("last >= first"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4712))
;
4713 if (first >= n_places) {
4714 if (masters_place) {
4715 first -= n_places;
4716 last -= n_places;
4717 if (first == (masters_place + 1)) {
4718 KMP_DEBUG_ASSERT(f == n_th)((f == n_th) ? 0 : __kmp_debug_assert("f == n_th", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4718))
;
4719 first--;
4720 }
4721 if (last == masters_place) {
4722 KMP_DEBUG_ASSERT(f == (n_th - 1))((f == (n_th - 1)) ? 0 : __kmp_debug_assert("f == (n_th - 1)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4722))
;
4723 last--;
4724 }
4725 } else {
4726 KMP_DEBUG_ASSERT(f == n_th)((f == n_th) ? 0 : __kmp_debug_assert("f == n_th", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4726))
;
4727 first = 0;
4728 last = 0;
4729 }
4730 }
4731 if (last >= n_places) {
4732 last = (n_places - 1);
4733 }
4734 place = first;
4735 current += spacing;
4736 if (f < n_th) {
4737 KMP_DEBUG_ASSERT(0 <= first)((0 <= first) ? 0 : __kmp_debug_assert("0 <= first", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4737))
;
4738 KMP_DEBUG_ASSERT(n_places > first)((n_places > first) ? 0 : __kmp_debug_assert("n_places > first"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4738))
;
4739 KMP_DEBUG_ASSERT(0 <= last)((0 <= last) ? 0 : __kmp_debug_assert("0 <= last", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4739))
;
4740 KMP_DEBUG_ASSERT(n_places > last)((n_places > last) ? 0 : __kmp_debug_assert("n_places > last"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4740))
;
4741 KMP_DEBUG_ASSERT(last_place >= first_place)((last_place >= first_place) ? 0 : __kmp_debug_assert("last_place >= first_place"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4741))
;
4742 th = team->t.t_threads[f];
4743 KMP_DEBUG_ASSERT(th)((th) ? 0 : __kmp_debug_assert("th", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4743))
;
4744 th->th.th_first_place = first;
4745 th->th.th_new_place = place;
4746 th->th.th_last_place = last;
4747
4748 KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", ((((team->t.t_threads
[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4751)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, spacing); }
4749 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", ((((team->t.t_threads
[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4751)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, spacing); }
4750 "partition = [%d,%d], spacing = %.4f\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", ((((team->t.t_threads
[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4751)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, spacing); }
4751 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", ((((team->t.t_threads
[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4751)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, spacing); }
4752 team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", ((((team->t.t_threads
[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4751)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, spacing); }
4753 th->th.th_first_place, th->th.th_last_place, spacing))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d], spacing = %.4f\n", ((((team->t.t_threads
[f]) != __null) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4751)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place, spacing); }
;
4754 }
4755 }
4756 }
4757 KMP_DEBUG_ASSERT(update_master_only || place == masters_place)((update_master_only || place == masters_place) ? 0 : __kmp_debug_assert
("update_master_only || place == masters_place", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4757))
;
4758 } else {
4759 int S, rem, gap, s_count;
4760 S = n_th / n_places;
4761 s_count = 0;
4762 rem = n_th - (S * n_places);
4763 gap = rem > 0 ? n_places / rem : n_places;
4764 int place = masters_place;
4765 int gap_ct = gap;
4766 thidx = n_th;
4767 if (update_master_only == 1)
4768 thidx = 1;
4769 for (f = 0; f < thidx; f++) {
4770 kmp_info_t *th = team->t.t_threads[f];
4771 KMP_DEBUG_ASSERT(th != NULL)((th != __null) ? 0 : __kmp_debug_assert("th != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4771))
;
4772
4773 th->th.th_first_place = place;
4774 th->th.th_last_place = place;
4775 th->th.th_new_place = place;
4776 s_count++;
4777
4778 if ((s_count == S) && rem && (gap_ct == gap)) {
4779 // do nothing, add an extra thread to place on next iteration
4780 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4781 // we added an extra thread to this place; move on to next place
4782 if (place == last_place) {
4783 place = first_place;
4784 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4785 place = 0;
4786 } else {
4787 place++;
4788 }
4789 s_count = 0;
4790 gap_ct = 1;
4791 rem--;
4792 } else if (s_count == S) { // place is full; don't add extra thread
4793 if (place == last_place) {
4794 place = first_place;
4795 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4796 place = 0;
4797 } else {
4798 place++;
4799 }
4800 gap_ct++;
4801 s_count = 0;
4802 }
4803
4804 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4806)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4805 "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4806)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4806 __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4806)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4807 team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4806)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
4808 th->th.th_first_place, th->th.th_last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
"partition = [%d,%d]\n", ((((team->t.t_threads[f]) != __null
) ? 0 : __kmp_debug_assert("(team->t.t_threads[f]) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4806)), (team->t.t_threads[f])->th.th_info.ds.ds_gtid
), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
;
4809 }
4810 KMP_DEBUG_ASSERT(update_master_only || place == masters_place)((update_master_only || place == masters_place) ? 0 : __kmp_debug_assert
("update_master_only || place == masters_place", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4810))
;
4811 }
4812 } break;
4813
4814 default:
4815 break;
4816 }
4817
4818 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: exit T#%d\n"
, team->t.t_id); }
;
4819}
4820
4821#endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */
4822
4823/* allocate a new team data structure to use. take one off of the free pool if
4824 available */
4825kmp_team_t *
4826__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
4827#if OMPT_SUPPORT1
4828 ompt_data_t ompt_parallel_data,
4829#endif
4830#if OMP_40_ENABLED(50 >= 40)
4831 kmp_proc_bind_t new_proc_bind,
4832#endif
4833 kmp_internal_control_t *new_icvs,
4834 int argc USE_NESTED_HOT_ARG(kmp_info_t *master), kmp_info_t *master) {
4835 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team)((void)0);
4836 int f;
4837 kmp_team_t *team;
4838 int use_hot_team = !root->r.r_active;
4839 int level = 0;
4840
4841 KA_TRACE(20, ("__kmp_allocate_team: called\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: called\n"
); }
;
4842 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0)((new_nproc >= 1 && argc >= 0) ? 0 : __kmp_debug_assert
("new_nproc >= 1 && argc >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4842))
;
4843 KMP_DEBUG_ASSERT(max_nproc >= new_nproc)((max_nproc >= new_nproc) ? 0 : __kmp_debug_assert("max_nproc >= new_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4843))
;
4844 KMP_MB();
4845
4846#if KMP_NESTED_HOT_TEAMS1
4847 kmp_hot_team_ptr_t *hot_teams;
4848 if (master) {
4849 team = master->th.th_team;
4850 level = team->t.t_active_level;
4851 if (master->th.th_teams_microtask) { // in teams construct?
4852 if (master->th.th_teams_size.nteams > 1 &&
4853 ( // #teams > 1
4854 team->t.t_pkfn ==
4855 (microtask_t)__kmp_teams_master || // inner fork of the teams
4856 master->th.th_teams_level <
4857 team->t.t_level)) { // or nested parallel inside the teams
4858 ++level; // not increment if #teams==1, or for outer fork of the teams;
4859 // increment otherwise
4860 }
4861 }
4862 hot_teams = master->th.th_hot_teams;
4863 if (level < __kmp_hot_teams_max_level && hot_teams &&
4864 hot_teams[level]
4865 .hot_team) { // hot team has already been allocated for given level
4866 use_hot_team = 1;
4867 } else {
4868 use_hot_team = 0;
4869 }
4870 }
4871#endif
4872 // Optimization to use a "hot" team
4873 if (use_hot_team && new_nproc > 1) {
4874 KMP_DEBUG_ASSERT(new_nproc == max_nproc)((new_nproc == max_nproc) ? 0 : __kmp_debug_assert("new_nproc == max_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4874))
;
4875#if KMP_NESTED_HOT_TEAMS1
4876 team = hot_teams[level].hot_team;
4877#else
4878 team = root->r.r_hot_team;
4879#endif
4880#if KMP_DEBUG1
4881 if (__kmp_tasking_mode != tskm_immediate_exec) {
4882 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
4883 "task_team[1] = %p before reinit\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
4884 team->t.t_task_team[0], team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
;
4885 }
4886#endif
4887
4888 // Has the number of threads changed?
4889 /* Let's assume the most common case is that the number of threads is
4890 unchanged, and put that case first. */
4891 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
4892 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team\n"
); }
;
4893 // This case can mean that omp_set_num_threads() was called and the hot
4894 // team size was already reduced, so we check the special flag
4895 if (team->t.t_size_changed == -1) {
4896 team->t.t_size_changed = 1;
4897 } else {
4898 KMP_CHECK_UPDATE(team->t.t_size_changed, 0)if ((team->t.t_size_changed) != (0)) (team->t.t_size_changed
) = (0)
;
4899 }
4900
4901 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4902 kmp_r_sched_t new_sched = new_icvs->sched;
4903 // set master's schedule as new run-time schedule
4904 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched)if ((team->t.t_sched.sched) != (new_sched.sched)) (team->
t.t_sched.sched) = (new_sched.sched)
;
4905
4906 __kmp_reinitialize_team(team, new_icvs,
4907 root->r.r_uber_thread->th.th_ident);
4908
4909 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
4910 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
;
4911 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
4912
4913#if OMP_40_ENABLED(50 >= 40)
4914#if KMP_AFFINITY_SUPPORTED1
4915 if ((team->t.t_size_changed == 0) &&
4916 (team->t.t_proc_bind == new_proc_bind)) {
4917 if (new_proc_bind == proc_bind_spread) {
4918 __kmp_partition_places(
4919 team, 1); // add flag to update only master for spread
4920 }
4921 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: "if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
4922 "proc_bind = %d, partition = [%d,%d]\n",if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
4923 team->t.t_id, new_proc_bind, team->t.t_first_place,if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
4924 team->t.t_last_place))if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
"proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
;
4925 } else {
4926 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
4927 __kmp_partition_places(team);
4928 }
4929#else
4930 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
4931#endif /* KMP_AFFINITY_SUPPORTED */
4932#endif /* OMP_40_ENABLED */
4933 } else if (team->t.t_nproc > new_nproc) {
4934 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
4935 ("__kmp_allocate_team: decreasing hot team thread count to %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
4936 new_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
;
4937
4938 team->t.t_size_changed = 1;
4939#if KMP_NESTED_HOT_TEAMS1
4940 if (__kmp_hot_teams_mode == 0) {
4941 // AC: saved number of threads should correspond to team's value in this
4942 // mode, can be bigger in mode 1, when hot team has threads in reserve
4943 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc)((hot_teams[level].hot_team_nth == team->t.t_nproc) ? 0 : __kmp_debug_assert
("hot_teams[level].hot_team_nth == team->t.t_nproc", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4943))
;
4944 hot_teams[level].hot_team_nth = new_nproc;
4945#endif // KMP_NESTED_HOT_TEAMS
4946 /* release the extra threads we don't need any more */
4947 for (f = new_nproc; f < team->t.t_nproc; f++) {
4948 KMP_DEBUG_ASSERT(team->t.t_threads[f])((team->t.t_threads[f]) ? 0 : __kmp_debug_assert("team->t.t_threads[f]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4948))
;
4949 if (__kmp_tasking_mode != tskm_immediate_exec) {
4950 // When decreasing team size, threads no longer in the team should
4951 // unref task team.
4952 team->t.t_threads[f]->th.th_task_team = NULL__null;
4953 }
4954 __kmp_free_thread(team->t.t_threads[f]);
4955 team->t.t_threads[f] = NULL__null;
4956 }
4957#if KMP_NESTED_HOT_TEAMS1
4958 } // (__kmp_hot_teams_mode == 0)
4959 else {
4960 // When keeping extra threads in team, switch threads to wait on own
4961 // b_go flag
4962 for (f = new_nproc; f < team->t.t_nproc; ++f) {
4963 KMP_DEBUG_ASSERT(team->t.t_threads[f])((team->t.t_threads[f]) ? 0 : __kmp_debug_assert("team->t.t_threads[f]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4963))
;
4964 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
4965 for (int b = 0; b < bs_last_barrier; ++b) {
4966 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG2) {
4967 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG3;
4968 }
4969 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0)if ((balign[b].bb.leaf_kids) != (0)) (balign[b].bb.leaf_kids)
= (0)
;
4970 }
4971 }
4972 }
4973#endif // KMP_NESTED_HOT_TEAMS
4974 team->t.t_nproc = new_nproc;
4975 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4976 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched)if ((team->t.t_sched.sched) != (new_icvs->sched.sched))
(team->t.t_sched.sched) = (new_icvs->sched.sched)
;
4977 __kmp_reinitialize_team(team, new_icvs,
4978 root->r.r_uber_thread->th.th_ident);
4979
4980 /* update the remaining threads */
4981 for (f = 0; f < new_nproc; ++f) {
4982 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
4983 }
4984 // restore the current task state of the master thread: should be the
4985 // implicit task
4986 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
4987 team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
;
4988
4989 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
4990
4991#ifdef KMP_DEBUG1
4992 for (f = 0; f < team->t.t_nproc; f++) {
4993 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4995))
4994 team->t.t_threads[f]->th.th_team_nproc ==((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4995))
4995 team->t.t_nproc)((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 4995))
;
4996 }
4997#endif
4998
4999#if OMP_40_ENABLED(50 >= 40)
5000 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
5001#if KMP_AFFINITY_SUPPORTED1
5002 __kmp_partition_places(team);
5003#endif
5004#endif
5005 } else { // team->t.t_nproc < new_nproc
5006#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
5007 kmp_affin_mask_t *old_mask;
5008 if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
5009 KMP_CPU_ALLOC(old_mask)(old_mask = __kmp_affinity_dispatch->allocate_mask());
5010 }
5011#endif
5012
5013 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
5014 ("__kmp_allocate_team: increasing hot team thread count to %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
5015 new_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
;
5016
5017 team->t.t_size_changed = 1;
5018
5019#if KMP_NESTED_HOT_TEAMS1
5020 int avail_threads = hot_teams[level].hot_team_nth;
5021 if (new_nproc < avail_threads)
5022 avail_threads = new_nproc;
5023 kmp_info_t **other_threads = team->t.t_threads;
5024 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5025 // Adjust barrier data of reserved threads (if any) of the team
5026 // Other data will be set in __kmp_initialize_info() below.
5027 int b;
5028 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5029 for (b = 0; b < bs_last_barrier; ++b) {
5030 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5031 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)((balign[b].bb.wait_flag != 2) ? 0 : __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5031))
;
5032#if USE_DEBUGGER0
5033 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5034#endif
5035 }
5036 }
5037 if (hot_teams[level].hot_team_nth >= new_nproc) {
5038 // we have all needed threads in reserve, no need to allocate any
5039 // this only possible in mode 1, cannot have reserved threads in mode 0
5040 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1)((__kmp_hot_teams_mode == 1) ? 0 : __kmp_debug_assert("__kmp_hot_teams_mode == 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5040))
;
5041 team->t.t_nproc = new_nproc; // just get reserved threads involved
5042 } else {
5043 // we may have some threads in reserve, but not enough
5044 team->t.t_nproc =
5045 hot_teams[level]
5046 .hot_team_nth; // get reserved threads involved if any
5047 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size
5048#endif // KMP_NESTED_HOT_TEAMS
5049 if (team->t.t_max_nproc < new_nproc) {
5050 /* reallocate larger arrays */
5051 __kmp_reallocate_team_arrays(team, new_nproc);
5052 __kmp_reinitialize_team(team, new_icvs, NULL__null);
5053 }
5054
5055#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
5056 /* Temporarily set full mask for master thread before creation of
5057 workers. The reason is that workers inherit the affinity from master,
5058 so if a lot of workers are created on the single core quickly, they
5059 don't get a chance to set their own affinity for a long time. */
5060 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5061#endif
5062
5063 /* allocate new threads for the hot team */
5064 for (f = team->t.t_nproc; f < new_nproc; f++) {
5065 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5066 KMP_DEBUG_ASSERT(new_worker)((new_worker) ? 0 : __kmp_debug_assert("new_worker", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5066))
;
5067 team->t.t_threads[f] = new_worker;
5068
5069 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, ((((f) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(f) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5072)), team->t.t_threads[(f)]->th.th_info.ds.ds_gtid
), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5070 ("__kmp_allocate_team: team %d init T#%d arrived: "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, ((((f) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(f) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5072)), team->t.t_threads[(f)]->th.th_info.ds.ds_gtid
), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5071 "join=%llu, plain=%llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, ((((f) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(f) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5072)), team->t.t_threads[(f)]->th.th_info.ds.ds_gtid
), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5072 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, ((((f) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(f) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5072)), team->t.t_threads[(f)]->th.th_info.ds.ds_gtid
), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5073 team->t.t_bar[bs_forkjoin_barrier].b_arrived,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, ((((f) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(f) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5072)), team->t.t_threads[(f)]->th.th_info.ds.ds_gtid
), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
5074 team->t.t_bar[bs_plain_barrier].b_arrived))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
"join=%llu, plain=%llu\n", team->t.t_id, ((((f) >= 0 &&
(team) != __null) ? 0 : __kmp_debug_assert("(f) >= 0 && (team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5072)), team->t.t_threads[(f)]->th.th_info.ds.ds_gtid
), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier].
b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
;
5075
5076 { // Initialize barrier data for new threads.
5077 int b;
5078 kmp_balign_t *balign = new_worker->th.th_bar;
5079 for (b = 0; b < bs_last_barrier; ++b) {
5080 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5081 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=((balign[b].bb.wait_flag != 2) ? 0 : __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5082))
5082 KMP_BARRIER_PARENT_FLAG)((balign[b].bb.wait_flag != 2) ? 0 : __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5082))
;
5083#if USE_DEBUGGER0
5084 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5085#endif
5086 }
5087 }
5088 }
5089
5090#if KMP_OS_LINUX1 && KMP_AFFINITY_SUPPORTED1
5091 if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
5092 /* Restore initial master thread's affinity mask */
5093 __kmp_set_system_affinity(old_mask, TRUE)(old_mask)->set_system_affinity((!0));
5094 KMP_CPU_FREE(old_mask)__kmp_affinity_dispatch->deallocate_mask(old_mask);
5095 }
5096#endif
5097#if KMP_NESTED_HOT_TEAMS1
5098 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
5099#endif // KMP_NESTED_HOT_TEAMS
5100 /* make sure everyone is syncronized */
5101 int old_nproc = team->t.t_nproc; // save old value and use to update only
5102 // new threads below
5103 __kmp_initialize_team(team, new_nproc, new_icvs,
5104 root->r.r_uber_thread->th.th_ident);
5105
5106 /* reinitialize the threads */
5107 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc)((team->t.t_nproc == new_nproc) ? 0 : __kmp_debug_assert("team->t.t_nproc == new_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5107))
;
5108 for (f = 0; f < team->t.t_nproc; ++f)
5109 __kmp_initialize_info(team->t.t_threads[f], team, f,
5110 __kmp_gtid_from_tid(f, team)((((f) >= 0 && (team) != __null) ? 0 : __kmp_debug_assert
("(f) >= 0 && (team) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5110)), team->t.t_threads[(f)]->th.th_info.ds.ds_gtid
)
);
5111 if (level) { // set th_task_state for new threads in nested hot team
5112 // __kmp_initialize_info() no longer zeroes th_task_state, so we should
5113 // only need to set the th_task_state for the new threads. th_task_state
5114 // for master thread will not be accurate until after this in
5115 // __kmp_fork_call(), so we look to the master's memo_stack to get the
5116 // correct value.
5117 for (f = old_nproc; f < team->t.t_nproc; ++f)
5118 team->t.t_threads[f]->th.th_task_state =
5119 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5120 } else { // set th_task_state for new threads in non-nested hot team
5121 int old_state =
5122 team->t.t_threads[0]->th.th_task_state; // copy master's state
5123 for (f = old_nproc; f < team->t.t_nproc; ++f)
5124 team->t.t_threads[f]->th.th_task_state = old_state;
5125 }
5126
5127#ifdef KMP_DEBUG1
5128 for (f = 0; f < team->t.t_nproc; ++f) {
5129 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5131))
5130 team->t.t_threads[f]->th.th_team_nproc ==((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5131))
5131 team->t.t_nproc)((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5131))
;
5132 }
5133#endif
5134
5135#if OMP_40_ENABLED(50 >= 40)
5136 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind)
;
5137#if KMP_AFFINITY_SUPPORTED1
5138 __kmp_partition_places(team);
5139#endif
5140#endif
5141 } // Check changes in number of threads
5142
5143#if OMP_40_ENABLED(50 >= 40)
5144 kmp_info_t *master = team->t.t_threads[0];
5145 if (master->th.th_teams_microtask) {
5146 for (f = 1; f < new_nproc; ++f) {
5147 // propagate teams construct specific info to workers
5148 kmp_info_t *thr = team->t.t_threads[f];
5149 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5150 thr->th.th_teams_level = master->th.th_teams_level;
5151 thr->th.th_teams_size = master->th.th_teams_size;
5152 }
5153 }
5154#endif /* OMP_40_ENABLED */
5155#if KMP_NESTED_HOT_TEAMS1
5156 if (level) {
5157 // Sync barrier state for nested hot teams, not needed for outermost hot
5158 // team.
5159 for (f = 1; f < new_nproc; ++f) {
5160 kmp_info_t *thr = team->t.t_threads[f];
5161 int b;
5162 kmp_balign_t *balign = thr->th.th_bar;
5163 for (b = 0; b < bs_last_barrier; ++b) {
5164 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5165 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)((balign[b].bb.wait_flag != 2) ? 0 : __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5165))
;
5166#if USE_DEBUGGER0
5167 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5168#endif
5169 }
5170 }
5171 }
5172#endif // KMP_NESTED_HOT_TEAMS
5173
5174 /* reallocate space for arguments if necessary */
5175 __kmp_alloc_argv_entries(argc, team, TRUE(!0));
5176 KMP_CHECK_UPDATE(team->t.t_argc, argc)if ((team->t.t_argc) != (argc)) (team->t.t_argc) = (argc
)
;
5177 // The hot team re-uses the previous task team,
5178 // if untouched during the previous release->gather phase.
5179
5180 KF_TRACE(10, (" hot_team = %p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf (" hot_team = %p\n"
, team); }
;
5181
5182#if KMP_DEBUG1
5183 if (__kmp_tasking_mode != tskm_immediate_exec) {
5184 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
5185 "task_team[1] = %p after reinit\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
5186 team->t.t_task_team[0], team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
"task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
;
5187 }
5188#endif
5189
5190#if OMPT_SUPPORT1
5191 __ompt_team_assign_id(team, ompt_parallel_data);
5192#endif
5193
5194 KMP_MB();
5195
5196 return team;
5197 }
5198
5199 /* next, let's try to take one from the team pool */
5200 KMP_MB();
5201 for (team = CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool); (team);) {
5202 /* TODO: consider resizing undersized teams instead of reaping them, now
5203 that we have a resizing mechanism */
5204 if (team->t.t_max_nproc >= max_nproc) {
5205 /* take this team from the team pool */
5206 __kmp_team_pool = team->t.t_next_pool;
5207
5208 /* setup the team for fresh use */
5209 __kmp_initialize_team(team, new_nproc, new_icvs, NULL__null);
5210
5211 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
"task_team[1] %p to NULL\n", &team->t.t_task_team[0],
&team->t.t_task_team[1]); }
5212 "task_team[1] %p to NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
"task_team[1] %p to NULL\n", &team->t.t_task_team[0],
&team->t.t_task_team[1]); }
5213 &team->t.t_task_team[0], &team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
"task_team[1] %p to NULL\n", &team->t.t_task_team[0],
&team->t.t_task_team[1]); }
;
5214 team->t.t_task_team[0] = NULL__null;
5215 team->t.t_task_team[1] = NULL__null;
5216
5217 /* reallocate space for arguments if necessary */
5218 __kmp_alloc_argv_entries(argc, team, TRUE(!0));
5219 KMP_CHECK_UPDATE(team->t.t_argc, argc)if ((team->t.t_argc) != (argc)) (team->t.t_argc) = (argc
)
;
5220
5221 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
5222 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
5223 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
;
5224 { // Initialize barrier data.
5225 int b;
5226 for (b = 0; b < bs_last_barrier; ++b) {
5227 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE0;
5228#if USE_DEBUGGER0
5229 team->t.t_bar[b].b_master_arrived = 0;
5230 team->t.t_bar[b].b_team_arrived = 0;
5231#endif
5232 }
5233 }
5234
5235#if OMP_40_ENABLED(50 >= 40)
5236 team->t.t_proc_bind = new_proc_bind;
5237#endif
5238
5239 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: using team from pool %d.\n"
, team->t.t_id); }
5240 team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: using team from pool %d.\n"
, team->t.t_id); }
;
5241
5242#if OMPT_SUPPORT1
5243 __ompt_team_assign_id(team, ompt_parallel_data);
5244#endif
5245
5246 KMP_MB();
5247
5248 return team;
5249 }
5250
5251 /* reap team if it is too small, then loop back and check the next one */
5252 // not sure if this is wise, but, will be redone during the hot-teams
5253 // rewrite.
5254 /* TODO: Use technique to find the right size hot-team, don't reap them */
5255 team = __kmp_reap_team(team);
5256 __kmp_team_pool = team;
5257 }
5258
5259 /* nothing available in the pool, no matter, make a new team! */
5260 KMP_MB();
5261 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t))___kmp_allocate((sizeof(kmp_team_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5261)
;
5262
5263 /* and set it up */
5264 team->t.t_max_nproc = max_nproc;
5265 /* NOTE well, for some reason allocating one big buffer and dividing it up
5266 seems to really hurt performance a lot on the P4, so, let's not use this */
5267 __kmp_allocate_team_arrays(team, max_nproc);
5268
5269 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: making a new team\n"
); }
;
5270 __kmp_initialize_team(team, new_nproc, new_icvs, NULL__null);
5271
5272 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
"%p to NULL\n", &team->t.t_task_team[0], &team->
t.t_task_team[1]); }
5273 "%p to NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
"%p to NULL\n", &team->t.t_task_team[0], &team->
t.t_task_team[1]); }
5274 &team->t.t_task_team[0], &team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
"%p to NULL\n", &team->t.t_task_team[0], &team->
t.t_task_team[1]); }
;
5275 team->t.t_task_team[0] = NULL__null; // to be removed, as __kmp_allocate zeroes
5276 // memory, no need to duplicate
5277 team->t.t_task_team[1] = NULL__null; // to be removed, as __kmp_allocate zeroes
5278 // memory, no need to duplicate
5279
5280 if (__kmp_storage_map) {
5281 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc);
5282 }
5283
5284 /* allocate space for arguments */
5285 __kmp_alloc_argv_entries(argc, team, FALSE0);
5286 team->t.t_argc = argc;
5287
5288 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
5289 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
5290 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
;
5291 { // Initialize barrier data.
5292 int b;
5293 for (b = 0; b < bs_last_barrier; ++b) {
5294 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE0;
5295#if USE_DEBUGGER0
5296 team->t.t_bar[b].b_master_arrived = 0;
5297 team->t.t_bar[b].b_team_arrived = 0;
5298#endif
5299 }
5300 }
5301
5302#if OMP_40_ENABLED(50 >= 40)
5303 team->t.t_proc_bind = new_proc_bind;
5304#endif
5305
5306#if OMPT_SUPPORT1
5307 __ompt_team_assign_id(team, ompt_parallel_data);
5308 team->t.ompt_serialized_team_info = NULL__null;
5309#endif
5310
5311 KMP_MB();
5312
5313 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: done creating a new team %d.\n"
, team->t.t_id); }
5314 team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: done creating a new team %d.\n"
, team->t.t_id); }
;
5315
5316 return team;
5317}
5318
5319/* TODO implement hot-teams at all levels */
5320/* TODO implement lazy thread release on demand (disband request) */
5321
5322/* free the team. return it to the team pool. release all the threads
5323 * associated with it */
5324void __kmp_free_team(kmp_root_t *root,
5325 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master), kmp_info_t *master) {
5326 int f;
5327 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d freeing team %d\n"
, __kmp_get_global_thread_id(), team->t.t_id); }
5328 team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d freeing team %d\n"
, __kmp_get_global_thread_id(), team->t.t_id); }
;
5329
5330 /* verify state */
5331 KMP_DEBUG_ASSERT(root)((root) ? 0 : __kmp_debug_assert("root", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5331))
;
5332 KMP_DEBUG_ASSERT(team)((team) ? 0 : __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5332))
;
5333 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc)((team->t.t_nproc <= team->t.t_max_nproc) ? 0 : __kmp_debug_assert
("team->t.t_nproc <= team->t.t_max_nproc", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5333))
;
5334 KMP_DEBUG_ASSERT(team->t.t_threads)((team->t.t_threads) ? 0 : __kmp_debug_assert("team->t.t_threads"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5334))
;
5335
5336 int use_hot_team = team == root->r.r_hot_team;
5337#if KMP_NESTED_HOT_TEAMS1
5338 int level;
5339 kmp_hot_team_ptr_t *hot_teams;
5340 if (master) {
5341 level = team->t.t_active_level - 1;
5342 if (master->th.th_teams_microtask) { // in teams construct?
5343 if (master->th.th_teams_size.nteams > 1) {
5344 ++level; // level was not increased in teams construct for
5345 // team_of_masters
5346 }
5347 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5348 master->th.th_teams_level == team->t.t_level) {
5349 ++level; // level was not increased in teams construct for
5350 // team_of_workers before the parallel
5351 } // team->t.t_level will be increased inside parallel
5352 }
5353 hot_teams = master->th.th_hot_teams;
5354 if (level < __kmp_hot_teams_max_level) {
5355 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team)((team == hot_teams[level].hot_team) ? 0 : __kmp_debug_assert
("team == hot_teams[level].hot_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5355))
;
5356 use_hot_team = 1;
5357 }
5358 }
5359#endif // KMP_NESTED_HOT_TEAMS
5360
5361 /* team is done working */
5362 TCW_SYNC_PTR(team->t.t_pkfn,((team->t.t_pkfn)) = ((__null))
5363 NULL)((team->t.t_pkfn)) = ((__null)); // Important for Debugging Support Library.
5364#if KMP_OS_WINDOWS0
5365 team->t.t_copyin_counter = 0; // init counter for possible reuse
5366#endif
5367 // Do not reset pointer to parent team to NULL for hot teams.
5368
5369 /* if we are non-hot team, release our threads */
5370 if (!use_hot_team) {
5371 if (__kmp_tasking_mode != tskm_immediate_exec) {
5372 // Wait for threads to reach reapable state
5373 for (f = 1; f < team->t.t_nproc; ++f) {
5374 KMP_DEBUG_ASSERT(team->t.t_threads[f])((team->t.t_threads[f]) ? 0 : __kmp_debug_assert("team->t.t_threads[f]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5374))
;
5375 kmp_info_t *th = team->t.t_threads[f];
5376 volatile kmp_uint32 *state = &th->th.th_reap_state;
5377 while (*state != KMP_SAFE_TO_REAP1) {
5378#if KMP_OS_WINDOWS0
5379 // On Windows a thread can be killed at any time, check this
5380 DWORD ecode;
5381 if (!__kmp_is_thread_alive(th, &ecode)) {
5382 *state = KMP_SAFE_TO_REAP1; // reset the flag for dead thread
5383 break;
5384 }
5385#endif
5386 // first check if thread is sleeping
5387 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5388 if (fl.is_sleeping())
5389 fl.resume(__kmp_gtid_from_thread(th)((((th) != __null) ? 0 : __kmp_debug_assert("(th) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5389)), (th)->th.th_info.ds.ds_gtid)
);
5390 KMP_CPU_PAUSE()__kmp_x86_pause();
5391 }
5392 }
5393
5394 // Delete task teams
5395 int tt_idx;
5396 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5397 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5398 if (task_team != NULL__null) {
5399 for (f = 0; f < team->t.t_nproc;
5400 ++f) { // Have all threads unref task teams
5401 team->t.t_threads[f]->th.th_task_team = NULL__null;
5402 }
5403 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); }
5404 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); }
5405 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); }
5406 __kmp_get_gtid(), task_team, team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); }
;
5407#if KMP_NESTED_HOT_TEAMS1
5408 __kmp_free_task_team(master, task_team);
5409#endif
5410 team->t.t_task_team[tt_idx] = NULL__null;
5411 }
5412 }
5413 }
5414
5415 // Reset pointer to parent team only for non-hot teams.
5416 team->t.t_parent = NULL__null;
5417 team->t.t_level = 0;
5418 team->t.t_active_level = 0;
5419
5420 /* free the worker threads */
5421 for (f = 1; f < team->t.t_nproc; ++f) {
5422 KMP_DEBUG_ASSERT(team->t.t_threads[f])((team->t.t_threads[f]) ? 0 : __kmp_debug_assert("team->t.t_threads[f]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5422))
;
5423 __kmp_free_thread(team->t.t_threads[f]);
5424 team->t.t_threads[f] = NULL__null;
5425 }
5426
5427 /* put the team back in the team pool */
5428 /* TODO limit size of team pool, call reap_team if pool too large */
5429 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool);
5430 __kmp_team_pool = (volatile kmp_team_t *)team;
5431 }
5432
5433 KMP_MB();
5434}
5435
5436/* reap the team. destroy it, reclaim all its resources and free its memory */
5437kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5438 kmp_team_t *next_pool = team->t.t_next_pool;
5439
5440 KMP_DEBUG_ASSERT(team)((team) ? 0 : __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5440))
;
5441 KMP_DEBUG_ASSERT(team->t.t_dispatch)((team->t.t_dispatch) ? 0 : __kmp_debug_assert("team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5441))
;
5442 KMP_DEBUG_ASSERT(team->t.t_disp_buffer)((team->t.t_disp_buffer) ? 0 : __kmp_debug_assert("team->t.t_disp_buffer"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5442))
;
5443 KMP_DEBUG_ASSERT(team->t.t_threads)((team->t.t_threads) ? 0 : __kmp_debug_assert("team->t.t_threads"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5443))
;
5444 KMP_DEBUG_ASSERT(team->t.t_argv)((team->t.t_argv) ? 0 : __kmp_debug_assert("team->t.t_argv"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5444))
;
5445
5446 /* TODO clean the threads that are a part of this? */
5447
5448 /* free stuff */
5449 __kmp_free_team_arrays(team);
5450 if (team->t.t_argv != &team->t.t_inline_argv[0])
5451 __kmp_free((void *)team->t.t_argv)___kmp_free(((void *)team->t.t_argv), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5451)
;
5452 __kmp_free(team)___kmp_free((team), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5452)
;
5453
5454 KMP_MB();
5455 return next_pool;
5456}
5457
5458// Free the thread. Don't reap it, just place it on the pool of available
5459// threads.
5460//
5461// Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid
5462// binding for the affinity mechanism to be useful.
5463//
5464// Now, we always keep the free list (__kmp_thread_pool) sorted by gtid.
5465// However, we want to avoid a potential performance problem by always
5466// scanning through the list to find the correct point at which to insert
5467// the thread (potential N**2 behavior). To do this we keep track of the
5468// last place a thread struct was inserted (__kmp_thread_pool_insert_pt).
5469// With single-level parallelism, threads will always be added to the tail
5470// of the list, kept track of by __kmp_thread_pool_insert_pt. With nested
5471// parallelism, all bets are off and we may need to scan through the entire
5472// free list.
5473//
5474// This change also has a potentially large performance benefit, for some
5475// applications. Previously, as threads were freed from the hot team, they
5476// would be placed back on the free list in inverse order. If the hot team
5477// grew back to it's original size, then the freed thread would be placed
5478// back on the hot team in reverse order. This could cause bad cache
5479// locality problems on programs where the size of the hot team regularly
5480// grew and shrunk.
5481//
5482// Now, for single-level parallelism, the OMP tid is alway == gtid.
5483void __kmp_free_thread(kmp_info_t *this_th) {
5484 int gtid;
5485 kmp_info_t **scan;
5486 kmp_root_t *root = this_th->th.th_root;
5487
5488 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n"
, __kmp_get_global_thread_id(), this_th->th.th_info.ds.ds_gtid
); }
5489 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n"
, __kmp_get_global_thread_id(), this_th->th.th_info.ds.ds_gtid
); }
;
5490
5491 KMP_DEBUG_ASSERT(this_th)((this_th) ? 0 : __kmp_debug_assert("this_th", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5491))
;
5492
5493 // When moving thread to pool, switch thread to wait on own b_go flag, and
5494 // uninitialized (NULL team).
5495 int b;
5496 kmp_balign_t *balign = this_th->th.th_bar;
5497 for (b = 0; b < bs_last_barrier; ++b) {
5498 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG2)
5499 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG3;
5500 balign[b].bb.team = NULL__null;
5501 balign[b].bb.leaf_kids = 0;
5502 }
5503 this_th->th.th_task_state = 0;
5504 this_th->th.th_reap_state = KMP_SAFE_TO_REAP1;
5505
5506 /* put thread back on the free pool */
5507 TCW_PTR(this_th->th.th_team, NULL)((this_th->th.th_team)) = ((__null));
5508 TCW_PTR(this_th->th.th_root, NULL)((this_th->th.th_root)) = ((__null));
5509 TCW_PTR(this_th->th.th_dispatch, NULL)((this_th->th.th_dispatch)) = ((__null)); /* NOT NEEDED */
5510
5511 /* If the implicit task assigned to this thread can be used by other threads
5512 * -> multiple threads can share the data and try to free the task at
5513 * __kmp_reap_thread at exit. This duplicate use of the task data can happen
5514 * with higher probability when hot team is disabled but can occurs even when
5515 * the hot team is enabled */
5516 __kmp_free_implicit_task(this_th);
5517 this_th->th.th_current_task = NULL__null;
5518
5519 // If the __kmp_thread_pool_insert_pt is already past the new insert
5520 // point, then we need to re-scan the entire list.
5521 gtid = this_th->th.th_info.ds.ds_gtid;
5522 if (__kmp_thread_pool_insert_pt != NULL__null) {
5523 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL)((__kmp_thread_pool != __null) ? 0 : __kmp_debug_assert("__kmp_thread_pool != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5523))
;
5524 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5525 __kmp_thread_pool_insert_pt = NULL__null;
5526 }
5527 }
5528
5529 // Scan down the list to find the place to insert the thread.
5530 // scan is the address of a link in the list, possibly the address of
5531 // __kmp_thread_pool itself.
5532 //
5533 // In the absence of nested parallism, the for loop will have 0 iterations.
5534 if (__kmp_thread_pool_insert_pt != NULL__null) {
5535 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5536 } else {
5537 scan = CCAST(kmp_info_t **, &__kmp_thread_pool)const_cast<kmp_info_t **>(&__kmp_thread_pool);
5538 }
5539 for (; (*scan != NULL__null) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5540 scan = &((*scan)->th.th_next_pool))
5541 ;
5542
5543 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt
5544 // to its address.
5545 TCW_PTR(this_th->th.th_next_pool, *scan)((this_th->th.th_next_pool)) = ((*scan));
5546 __kmp_thread_pool_insert_pt = *scan = this_th;
5547 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||(((this_th->th.th_next_pool == __null) || (this_th->th.
th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info
.ds.ds_gtid)) ? 0 : __kmp_debug_assert("(this_th->th.th_next_pool == __null) || (this_th->th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info.ds.ds_gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5549))
5548 (this_th->th.th_info.ds.ds_gtid <(((this_th->th.th_next_pool == __null) || (this_th->th.
th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info
.ds.ds_gtid)) ? 0 : __kmp_debug_assert("(this_th->th.th_next_pool == __null) || (this_th->th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info.ds.ds_gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5549))
5549 this_th->th.th_next_pool->th.th_info.ds.ds_gtid))(((this_th->th.th_next_pool == __null) || (this_th->th.
th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info
.ds.ds_gtid)) ? 0 : __kmp_debug_assert("(this_th->th.th_next_pool == __null) || (this_th->th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info.ds.ds_gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5549))
;
5550 TCW_4(this_th->th.th_in_pool, TRUE)(this_th->th.th_in_pool) = ((!0));
5551 __kmp_thread_pool_nth++;
5552
5553 TCW_4(__kmp_nth, __kmp_nth - 1)(__kmp_nth) = (__kmp_nth - 1);
5554 root->r.r_cg_nthreads--;
5555
5556#ifdef KMP_ADJUST_BLOCKTIME1
5557 /* Adjust blocktime back to user setting or default if necessary */
5558 /* Middle initialization might never have occurred */
5559 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5560 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0)((__kmp_avail_proc > 0) ? 0 : __kmp_debug_assert("__kmp_avail_proc > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5560))
;
5561 if (__kmp_nth <= __kmp_avail_proc) {
5562 __kmp_zero_bt = FALSE0;
5563 }
5564 }
5565#endif /* KMP_ADJUST_BLOCKTIME */
5566
5567 KMP_MB();
5568}
5569
5570/* ------------------------------------------------------------------------ */
5571
5572void *__kmp_launch_thread(kmp_info_t *this_thr) {
5573 int gtid = this_thr->th.th_info.ds.ds_gtid;
5574 /* void *stack_data;*/
5575 kmp_team_t *(*volatile pteam);
5576
5577 KMP_MB();
5578 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_launch_thread: T#%d start\n"
, gtid); }
;
5579
5580 if (__kmp_env_consistency_check) {
5581 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak?
5582 }
5583
5584#if OMPT_SUPPORT1
5585 ompt_data_t *thread_data;
5586 if (ompt_enabled.enabled) {
5587 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
5588 thread_data->ptr = NULL__null;
5589
5590 this_thr->th.ompt_thread_info.state = omp_state_overhead;
5591 this_thr->th.ompt_thread_info.wait_id = 0;
5592 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
5593 if (ompt_enabled.ompt_callback_thread_begin) {
5594 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)ompt_callback_thread_begin_callback(
5595 ompt_thread_worker, thread_data);
5596 }
5597 }
5598#endif
5599
5600#if OMPT_SUPPORT1
5601 if (ompt_enabled.enabled) {
5602 this_thr->th.ompt_thread_info.state = omp_state_idle;
5603 }
5604#endif
5605 /* This is the place where threads wait for work */
5606 while (!TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
5607 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid])((this_thr == __kmp_threads[gtid]) ? 0 : __kmp_debug_assert("this_thr == __kmp_threads[gtid]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5607))
;
5608 KMP_MB();
5609
5610 /* wait for work to do */
5611 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d waiting for work\n"
, gtid); }
;
5612
5613 /* No tid yet since not part of a team */
5614 __kmp_fork_barrier(gtid, KMP_GTID_DNE(-2));
5615
5616#if OMPT_SUPPORT1
5617 if (ompt_enabled.enabled) {
5618 this_thr->th.ompt_thread_info.state = omp_state_overhead;
5619 }
5620#endif
5621
5622 pteam = (kmp_team_t * (*))(&this_thr->th.th_team);
5623
5624 /* have we been allocated? */
5625 if (TCR_SYNC_PTR(*pteam)((void *)(*pteam)) && !TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
5626 /* we were just woken up, so run our new task */
5627 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn)((void *)((*pteam)->t.t_pkfn)) != NULL__null) {
5628 int rc;
5629 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, ((((gtid) >= 0) ? 0 : __kmp_debug_assert
("(gtid) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5631)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), (*
pteam)->t.t_pkfn); }
5630 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, ((((gtid) >= 0) ? 0 : __kmp_debug_assert
("(gtid) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5631)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), (*
pteam)->t.t_pkfn); }
5631 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, ((((gtid) >= 0) ? 0 : __kmp_debug_assert
("(gtid) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5631)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), (*
pteam)->t.t_pkfn); }
5632 (*pteam)->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, ((((gtid) >= 0) ? 0 : __kmp_debug_assert
("(gtid) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5631)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), (*
pteam)->t.t_pkfn); }
;
5633
5634 updateHWFPControl(*pteam);
5635
5636#if OMPT_SUPPORT1
5637 if (ompt_enabled.enabled) {
5638 this_thr->th.ompt_thread_info.state = omp_state_work_parallel;
5639 }
5640#endif
5641
5642 {
5643 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
5644 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
5645 rc = (*pteam)->t.t_invoke(gtid);
5646 }
5647 KMP_ASSERT(rc)((rc) ? 0 : __kmp_debug_assert("rc", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5647))
;
5648
5649 KMP_MB();
5650 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n"
, gtid, (*pteam)->t.t_id, ((((gtid) >= 0) ? 0 : __kmp_debug_assert
("(gtid) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5651)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), (*
pteam)->t.t_pkfn); }
5651 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n"
, gtid, (*pteam)->t.t_id, ((((gtid) >= 0) ? 0 : __kmp_debug_assert
("(gtid) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5651)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), (*
pteam)->t.t_pkfn); }
5652 (*pteam)->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n"
, gtid, (*pteam)->t.t_id, ((((gtid) >= 0) ? 0 : __kmp_debug_assert
("(gtid) >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5651)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), (*
pteam)->t.t_pkfn); }
;
5653 }
5654#if OMPT_SUPPORT1
5655 if (ompt_enabled.enabled) {
5656 /* no frame set while outside task */
5657 __ompt_get_task_info_object(0)->frame.exit_frame = NULL__null;
5658
5659 this_thr->th.ompt_thread_info.state = omp_state_overhead;
5660 }
5661#endif
5662 /* join barrier after parallel region */
5663 __kmp_join_barrier(gtid);
5664 }
5665 }
5666 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done)((void *)((intptr_t)__kmp_global.g.g_done));
5667
5668#if OMPT_SUPPORT1
5669 if (ompt_enabled.ompt_callback_thread_end) {
5670 ompt_callbacks.ompt_callback(ompt_callback_thread_end)ompt_callback_thread_end_callback(thread_data);
5671 }
5672#endif
5673
5674 this_thr->th.th_task_team = NULL__null;
5675 /* run the destructors for the threadprivate data for this thread */
5676 __kmp_common_destroy_gtid(gtid);
5677
5678 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_launch_thread: T#%d done\n"
, gtid); }
;
5679 KMP_MB();
5680 return this_thr;
5681}
5682
5683/* ------------------------------------------------------------------------ */
5684
5685void __kmp_internal_end_dest(void *specific_gtid) {
5686#if KMP_COMPILER_ICC0
5687#pragma warning(push)
5688#pragma warning(disable : 810) // conversion from "void *" to "int" may lose
5689// significant bits
5690#endif
5691 // Make sure no significant bits are lost
5692 int gtid = (kmp_intptr_t)specific_gtid - 1;
5693#if KMP_COMPILER_ICC0
5694#pragma warning(pop)
5695#endif
5696
5697 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_internal_end_dest: T#%d\n"
, gtid); }
;
5698 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage
5699 * this is because 0 is reserved for the nothing-stored case */
5700
5701 /* josh: One reason for setting the gtid specific data even when it is being
5702 destroyed by pthread is to allow gtid lookup through thread specific data
5703 (__kmp_gtid_get_specific). Some of the code, especially stat code,
5704 that gets executed in the call to __kmp_internal_end_thread, actually
5705 gets the gtid through the thread specific data. Setting it here seems
5706 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread
5707 to run smoothly.
5708 todo: get rid of this after we remove the dependence on
5709 __kmp_gtid_get_specific */
5710 if (gtid >= 0 && KMP_UBER_GTID(gtid)((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5710)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5710)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))
)
5711 __kmp_gtid_set_specific(gtid);
5712#ifdef KMP_TDATA_GTID1
5713 __kmp_gtid = gtid;
5714#endif
5715 __kmp_internal_end_thread(gtid);
5716}
5717
5718#if KMP_OS_UNIX1 && KMP_DYNAMIC_LIB1
5719
5720// 2009-09-08 (lev): It looks the destructor does not work. In simple test cases
5721// destructors work perfectly, but in real libomp.so I have no evidence it is
5722// ever called. However, -fini linker option in makefile.mk works fine.
5723
5724__attribute__((destructor)) void __kmp_internal_end_dtor(void) {
5725 __kmp_internal_end_atexit();
5726}
5727
5728void __kmp_internal_end_fini(void) { __kmp_internal_end_atexit(); }
5729
5730#endif
5731
5732/* [Windows] josh: when the atexit handler is called, there may still be more
5733 than one thread alive */
5734void __kmp_internal_end_atexit(void) {
5735 KA_TRACE(30, ("__kmp_internal_end_atexit\n"))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_internal_end_atexit\n"
); }
;
5736 /* [Windows]
5737 josh: ideally, we want to completely shutdown the library in this atexit
5738 handler, but stat code that depends on thread specific data for gtid fails
5739 because that data becomes unavailable at some point during the shutdown, so
5740 we call __kmp_internal_end_thread instead. We should eventually remove the
5741 dependency on __kmp_get_specific_gtid in the stat code and use
5742 __kmp_internal_end_library to cleanly shutdown the library.
5743
5744 // TODO: Can some of this comment about GVS be removed?
5745 I suspect that the offending stat code is executed when the calling thread
5746 tries to clean up a dead root thread's data structures, resulting in GVS
5747 code trying to close the GVS structures for that thread, but since the stat
5748 code uses __kmp_get_specific_gtid to get the gtid with the assumption that
5749 the calling thread is cleaning up itself instead of another thread, it get
5750 confused. This happens because allowing a thread to unregister and cleanup
5751 another thread is a recent modification for addressing an issue.
5752 Based on the current design (20050722), a thread may end up
5753 trying to unregister another thread only if thread death does not trigger
5754 the calling of __kmp_internal_end_thread. For Linux* OS, there is the
5755 thread specific data destructor function to detect thread death. For
5756 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there
5757 is nothing. Thus, the workaround is applicable only for Windows static
5758 stat library. */
5759 __kmp_internal_end_library(-1);
5760#if KMP_OS_WINDOWS0
5761 __kmp_close_console();
5762#endif
5763}
5764
5765static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
5766 // It is assumed __kmp_forkjoin_lock is acquired.
5767
5768 int gtid;
5769
5770 KMP_DEBUG_ASSERT(thread != NULL)((thread != __null) ? 0 : __kmp_debug_assert("thread != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5770))
;
5771
5772 gtid = thread->th.th_info.ds.ds_gtid;
5773
5774 if (!is_root) {
5775
5776 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647)) {
5777 /* Assume the threads are at the fork barrier here */
5778 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n"
, gtid); }
5779 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n"
, gtid); }
5780 gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n"
, gtid); }
;
5781 /* Need release fence here to prevent seg faults for tree forkjoin barrier
5782 * (GEH) */
5783 ANNOTATE_HAPPENS_BEFORE(thread);
5784 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
5785 __kmp_release_64(&flag);
5786 }
5787
5788 // Terminate OS thread.
5789 __kmp_reap_worker(thread);
5790
5791 // The thread was killed asynchronously. If it was actively
5792 // spinning in the thread pool, decrement the global count.
5793 //
5794 // There is a small timing hole here - if the worker thread was just waking
5795 // up after sleeping in the pool, had reset it's th_active_in_pool flag but
5796 // not decremented the global counter __kmp_thread_pool_active_nth yet, then
5797 // the global counter might not get updated.
5798 //
5799 // Currently, this can only happen as the library is unloaded,
5800 // so there are no harmful side effects.
5801 if (thread->th.th_active_in_pool) {
5802 thread->th.th_active_in_pool = FALSE0;
5803 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth)(&__kmp_thread_pool_active_nth)->fetch_sub(1, std::memory_order_acq_rel
)
;
5804 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0)((__kmp_thread_pool_active_nth >= 0) ? 0 : __kmp_debug_assert
("__kmp_thread_pool_active_nth >= 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5804))
;
5805 }
5806
5807 // Decrement # of [worker] threads in the pool.
5808 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth > 0)((__kmp_thread_pool_nth > 0) ? 0 : __kmp_debug_assert("__kmp_thread_pool_nth > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5808))
;
5809 --__kmp_thread_pool_nth;
5810 }
5811
5812 __kmp_free_implicit_task(thread);
5813
5814// Free the fast memory for tasking
5815#if USE_FAST_MEMORY3
5816 __kmp_free_fast_memory(thread);
5817#endif /* USE_FAST_MEMORY */
5818
5819 __kmp_suspend_uninitialize_thread(thread);
5820
5821 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread)((__kmp_threads[gtid] == thread) ? 0 : __kmp_debug_assert("__kmp_threads[gtid] == thread"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5821))
;
5822 TCW_SYNC_PTR(__kmp_threads[gtid], NULL)((__kmp_threads[gtid])) = ((__null));
5823
5824 --__kmp_all_nth;
5825// __kmp_nth was decremented when thread is added to the pool.
5826
5827#ifdef KMP_ADJUST_BLOCKTIME1
5828 /* Adjust blocktime back to user setting or default if necessary */
5829 /* Middle initialization might never have occurred */
5830 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5831 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0)((__kmp_avail_proc > 0) ? 0 : __kmp_debug_assert("__kmp_avail_proc > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5831))
;
5832 if (__kmp_nth <= __kmp_avail_proc) {
5833 __kmp_zero_bt = FALSE0;
5834 }
5835 }
5836#endif /* KMP_ADJUST_BLOCKTIME */
5837
5838 /* free the memory being used */
5839 if (__kmp_env_consistency_check) {
5840 if (thread->th.th_cons) {
5841 __kmp_free_cons_stack(thread->th.th_cons);
5842 thread->th.th_cons = NULL__null;
5843 }
5844 }
5845
5846 if (thread->th.th_pri_common != NULL__null) {
5847 __kmp_free(thread->th.th_pri_common)___kmp_free((thread->th.th_pri_common), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5847)
;
5848 thread->th.th_pri_common = NULL__null;
5849 }
5850
5851 if (thread->th.th_task_state_memo_stack != NULL__null) {
5852 __kmp_free(thread->th.th_task_state_memo_stack)___kmp_free((thread->th.th_task_state_memo_stack), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5852)
;
5853 thread->th.th_task_state_memo_stack = NULL__null;
5854 }
5855
5856#if KMP_USE_BGET1
5857 if (thread->th.th_local.bget_data != NULL__null) {
5858 __kmp_finalize_bget(thread);
5859 }
5860#endif
5861
5862#if KMP_AFFINITY_SUPPORTED1
5863 if (thread->th.th_affin_mask != NULL__null) {
5864 KMP_CPU_FREE(thread->th.th_affin_mask)__kmp_affinity_dispatch->deallocate_mask(thread->th.th_affin_mask
)
;
5865 thread->th.th_affin_mask = NULL__null;
5866 }
5867#endif /* KMP_AFFINITY_SUPPORTED */
5868
5869#if KMP_USE_HIER_SCHED0
5870 if (thread->th.th_hier_bar_data != NULL__null) {
5871 __kmp_free(thread->th.th_hier_bar_data)___kmp_free((thread->th.th_hier_bar_data), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5871)
;
5872 thread->th.th_hier_bar_data = NULL__null;
5873 }
5874#endif
5875
5876 __kmp_reap_team(thread->th.th_serial_team);
5877 thread->th.th_serial_team = NULL__null;
5878 __kmp_free(thread)___kmp_free((thread), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5878)
;
5879
5880 KMP_MB();
5881
5882} // __kmp_reap_thread
5883
5884static void __kmp_internal_end(void) {
5885 int i;
5886
5887 /* First, unregister the library */
5888 __kmp_unregister_library();
5889
5890#if KMP_OS_WINDOWS0
5891 /* In Win static library, we can't tell when a root actually dies, so we
5892 reclaim the data structures for any root threads that have died but not
5893 unregistered themselves, in order to shut down cleanly.
5894 In Win dynamic library we also can't tell when a thread dies. */
5895 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of
5896// dead roots
5897#endif
5898
5899 for (i = 0; i < __kmp_threads_capacity; i++)
5900 if (__kmp_root[i])
5901 if (__kmp_root[i]->r.r_active)
5902 break;
5903 KMP_MB(); /* Flush all pending memory write invalidates. */
5904 TCW_SYNC_4(__kmp_global.g.g_done, TRUE)(__kmp_global.g.g_done) = ((!0));
5905
5906 if (i < __kmp_threads_capacity) {
5907#if KMP_USE_MONITOR
5908 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor??
5909 KMP_MB(); /* Flush all pending memory write invalidates. */
5910
5911 // Need to check that monitor was initialized before reaping it. If we are
5912 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then
5913 // __kmp_monitor will appear to contain valid data, but it is only valid in
5914 // the parent process, not the child.
5915 // New behavior (201008): instead of keying off of the flag
5916 // __kmp_init_parallel, the monitor thread creation is keyed off
5917 // of the new flag __kmp_init_monitor.
5918 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
5919 if (TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
5920 __kmp_reap_monitor(&__kmp_monitor);
5921 TCW_4(__kmp_init_monitor, 0)(__kmp_init_monitor) = (0);
5922 }
5923 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
5924 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end: monitor reaped\n"
); }
;
5925#endif // KMP_USE_MONITOR
5926 } else {
5927/* TODO move this to cleanup code */
5928#ifdef KMP_DEBUG1
5929 /* make sure that everything has properly ended */
5930 for (i = 0; i < __kmp_threads_capacity; i++) {
5931 if (__kmp_root[i]) {
5932 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC:
5933 // there can be uber threads alive here
5934 KMP_ASSERT(!__kmp_root[i]->r.r_active)((!__kmp_root[i]->r.r_active) ? 0 : __kmp_debug_assert("!__kmp_root[i]->r.r_active"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5934))
; // TODO: can they be active?
5935 }
5936 }
5937#endif
5938
5939 KMP_MB();
5940
5941 // Reap the worker threads.
5942 // This is valid for now, but be careful if threads are reaped sooner.
5943 while (__kmp_thread_pool != NULL__null) { // Loop thru all the thread in the pool.
5944 // Get the next thread from the pool.
5945 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool);
5946 __kmp_thread_pool = thread->th.th_next_pool;
5947 // Reap it.
5948 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP)((thread->th.th_reap_state == 1) ? 0 : __kmp_debug_assert(
"thread->th.th_reap_state == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 5948))
;
5949 thread->th.th_next_pool = NULL__null;
5950 thread->th.th_in_pool = FALSE0;
5951 __kmp_reap_thread(thread, 0);
5952 }
5953 __kmp_thread_pool_insert_pt = NULL__null;
5954
5955 // Reap teams.
5956 while (__kmp_team_pool != NULL__null) { // Loop thru all the teams in the pool.
5957 // Get the next team from the pool.
5958 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool);
5959 __kmp_team_pool = team->t.t_next_pool;
5960 // Reap it.
5961 team->t.t_next_pool = NULL__null;
5962 __kmp_reap_team(team);
5963 }
5964
5965 __kmp_reap_task_teams();
5966
5967#if KMP_OS_UNIX1
5968 // Threads that are not reaped should not access any resources since they
5969 // are going to be deallocated soon, so the shutdown sequence should wait
5970 // until all threads either exit the final spin-waiting loop or begin
5971 // sleeping after the given blocktime.
5972 for (i = 0; i < __kmp_threads_capacity; i++) {
5973 kmp_info_t *thr = __kmp_threads[i];
5974 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)(&thr->th.th_blocking)->load(std::memory_order_acquire
)
)
5975 KMP_CPU_PAUSE()__kmp_x86_pause();
5976 }
5977#endif
5978
5979 for (i = 0; i < __kmp_threads_capacity; ++i) {
5980 // TBD: Add some checking...
5981 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL );
5982 }
5983
5984 /* Make sure all threadprivate destructors get run by joining with all
5985 worker threads before resetting this flag */
5986 TCW_SYNC_4(__kmp_init_common, FALSE)(__kmp_init_common) = (0);
5987
5988 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end: all workers reaped\n"
); }
;
5989 KMP_MB();
5990
5991#if KMP_USE_MONITOR
5992 // See note above: One of the possible fixes for CQ138434 / CQ140126
5993 //
5994 // FIXME: push both code fragments down and CSE them?
5995 // push them into __kmp_cleanup() ?
5996 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
5997 if (TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
5998 __kmp_reap_monitor(&__kmp_monitor);
5999 TCW_4(__kmp_init_monitor, 0)(__kmp_init_monitor) = (0);
6000 }
6001 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6002 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end: monitor reaped\n"
); }
;
6003#endif
6004 } /* else !__kmp_global.t_active */
6005 TCW_4(__kmp_init_gtid, FALSE)(__kmp_init_gtid) = (0);
6006 KMP_MB(); /* Flush all pending memory write invalidates. */
6007
6008 __kmp_cleanup();
6009#if OMPT_SUPPORT1
6010 ompt_fini();
6011#endif
6012}
6013
6014void __kmp_internal_end_library(int gtid_req) {
6015 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
6016 /* this shouldn't be a race condition because __kmp_internal_end() is the
6017 only place to clear __kmp_serial_init */
6018 /* we'll check this later too, after we get the lock */
6019 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6020 // redundaant, because the next check will work in any case.
6021 if (__kmp_global.g.g_abort) {
6022 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n"))if (kmp_a_debug >= 11) { __kmp_debug_printf ("__kmp_internal_end_library: abort, exiting\n"
); }
;
6023 /* TODO abort? */
6024 return;
6025 }
6026 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
6027 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: already finished\n"
); }
;
6028 return;
6029 }
6030
6031 KMP_MB(); /* Flush all pending memory write invalidates. */
6032
6033 /* find out who we are and what we should do */
6034 {
6035 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6036 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: enter T#%d (%d)\n"
, gtid, gtid_req); }
6037 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: enter T#%d (%d)\n"
, gtid, gtid_req); }
;
6038 if (gtid == KMP_GTID_SHUTDOWN(-3)) {
6039 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: !__kmp_init_runtime, system "
"already shutdown\n"); }
6040 "already shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: !__kmp_init_runtime, system "
"already shutdown\n"); }
;
6041 return;
6042 } else if (gtid == KMP_GTID_MONITOR(-4)) {
6043 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: monitor thread, gtid not "
"registered, or system shutdown\n"); }
6044 "registered, or system shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: monitor thread, gtid not "
"registered, or system shutdown\n"); }
;
6045 return;
6046 } else if (gtid == KMP_GTID_DNE(-2)) {
6047 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: gtid not registered or system "
"shutdown\n"); }
6048 "shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: gtid not registered or system "
"shutdown\n"); }
;
6049 /* we don't know who we are, but we may still shutdown the library */
6050 } else if (KMP_UBER_GTID(gtid)((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6050)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6050)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))
) {
6051 /* unregister ourselves as an uber thread. gtid is no longer valid */
6052 if (__kmp_root[gtid]->r.r_active) {
6053 __kmp_global.g.g_abort = -1;
6054 TCW_SYNC_4(__kmp_global.g.g_done, TRUE)(__kmp_global.g.g_done) = ((!0));
6055 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: root still active, abort T#%d\n"
, gtid); }
6056 ("__kmp_internal_end_library: root still active, abort T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: root still active, abort T#%d\n"
, gtid); }
6057 gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: root still active, abort T#%d\n"
, gtid); }
;
6058 return;
6059 } else {
6060 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: unregistering sibling T#%d\n"
, gtid); }
6061 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: unregistering sibling T#%d\n"
, gtid); }
6062 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: unregistering sibling T#%d\n"
, gtid); }
;
6063 __kmp_unregister_root_current_thread(gtid);
6064 }
6065 } else {
6066/* worker threads may call this function through the atexit handler, if they
6067 * call exit() */
6068/* For now, skip the usual subsequent processing and just dump the debug buffer.
6069 TODO: do a thorough shutdown instead */
6070#ifdef DUMP_DEBUG_ON_EXIT
6071 if (__kmp_debug_buf)
6072 __kmp_dump_debug_buffer();
6073#endif
6074 return;
6075 }
6076 }
6077 /* synchronize the termination process */
6078 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6079
6080 /* have we already finished */
6081 if (__kmp_global.g.g_abort) {
6082 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: abort, exiting\n"
); }
;
6083 /* TODO abort? */
6084 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6085 return;
6086 }
6087 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
6088 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6089 return;
6090 }
6091
6092 /* We need this lock to enforce mutex between this reading of
6093 __kmp_threads_capacity and the writing by __kmp_register_root.
6094 Alternatively, we can use a counter of roots that is atomically updated by
6095 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6096 __kmp_internal_end_*. */
6097 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6098
6099 /* now we can safely conduct the actual termination */
6100 __kmp_internal_end();
6101
6102 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6103 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6104
6105 KA_TRACE(10, ("__kmp_internal_end_library: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: exit\n"
); }
;
6106
6107#ifdef DUMP_DEBUG_ON_EXIT
6108 if (__kmp_debug_buf)
6109 __kmp_dump_debug_buffer();
6110#endif
6111
6112#if KMP_OS_WINDOWS0
6113 __kmp_close_console();
6114#endif
6115
6116 __kmp_fini_allocator();
6117
6118} // __kmp_internal_end_library
6119
6120void __kmp_internal_end_thread(int gtid_req) {
6121 int i;
6122
6123 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
6124 /* this shouldn't be a race condition because __kmp_internal_end() is the
6125 * only place to clear __kmp_serial_init */
6126 /* we'll check this later too, after we get the lock */
6127 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6128 // redundant, because the next check will work in any case.
6129 if (__kmp_global.g.g_abort) {
6130 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n"))if (kmp_a_debug >= 11) { __kmp_debug_printf ("__kmp_internal_end_thread: abort, exiting\n"
); }
;
6131 /* TODO abort? */
6132 return;
6133 }
6134 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
6135 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: already finished\n"
); }
;
6136 return;
6137 }
6138
6139 KMP_MB(); /* Flush all pending memory write invalidates. */
6140
6141 /* find out who we are and what we should do */
6142 {
6143 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6144 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: enter T#%d (%d)\n"
, gtid, gtid_req); }
6145 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: enter T#%d (%d)\n"
, gtid, gtid_req); }
;
6146 if (gtid == KMP_GTID_SHUTDOWN(-3)) {
6147 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: !__kmp_init_runtime, system "
"already shutdown\n"); }
6148 "already shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: !__kmp_init_runtime, system "
"already shutdown\n"); }
;
6149 return;
6150 } else if (gtid == KMP_GTID_MONITOR(-4)) {
6151 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: monitor thread, gtid not "
"registered, or system shutdown\n"); }
6152 "registered, or system shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: monitor thread, gtid not "
"registered, or system shutdown\n"); }
;
6153 return;
6154 } else if (gtid == KMP_GTID_DNE(-2)) {
6155 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: gtid not registered or system "
"shutdown\n"); }
6156 "shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: gtid not registered or system "
"shutdown\n"); }
;
6157 return;
6158 /* we don't know who we are */
6159 } else if (KMP_UBER_GTID(gtid)((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6159)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6159)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))
) {
6160 /* unregister ourselves as an uber thread. gtid is no longer valid */
6161 if (__kmp_root[gtid]->r.r_active) {
6162 __kmp_global.g.g_abort = -1;
6163 TCW_SYNC_4(__kmp_global.g.g_done, TRUE)(__kmp_global.g.g_done) = ((!0));
6164 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: root still active, abort T#%d\n"
, gtid); }
6165 ("__kmp_internal_end_thread: root still active, abort T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: root still active, abort T#%d\n"
, gtid); }
6166 gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: root still active, abort T#%d\n"
, gtid); }
;
6167 return;
6168 } else {
6169 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: unregistering sibling T#%d\n"
, gtid); }
6170 gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: unregistering sibling T#%d\n"
, gtid); }
;
6171 __kmp_unregister_root_current_thread(gtid);
6172 }
6173 } else {
6174 /* just a worker thread, let's leave */
6175 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread T#%d\n"
, gtid); }
;
6176
6177 if (gtid >= 0) {
6178 __kmp_threads[gtid]->th.th_task_team = NULL__null;
6179 }
6180
6181 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n"
, gtid); }
6182 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n"
, gtid); }
6183 gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n"
, gtid); }
;
6184 return;
6185 }
6186 }
6187#if defined KMP_DYNAMIC_LIB1
6188 // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber
6189 // thread, because we will better shutdown later in the library destructor.
6190 // The reason of this change is performance problem when non-openmp thread in
6191 // a loop forks and joins many openmp threads. We can save a lot of time
6192 // keeping worker threads alive until the program shutdown.
6193 // OM: Removed Linux* OS restriction to fix the crash on OS X* (DPD200239966)
6194 // and Windows(DPD200287443) that occurs when using critical sections from
6195 // foreign threads.
6196 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: exiting T#%d\n"
, gtid_req); }
;
6197 return;
6198#endif
6199 /* synchronize the termination process */
6200 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6201
6202 /* have we already finished */
6203 if (__kmp_global.g.g_abort) {
6204 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: abort, exiting\n"
); }
;
6205 /* TODO abort? */
6206 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6207 return;
6208 }
6209 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
6210 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6211 return;
6212 }
6213
6214 /* We need this lock to enforce mutex between this reading of
6215 __kmp_threads_capacity and the writing by __kmp_register_root.
6216 Alternatively, we can use a counter of roots that is atomically updated by
6217 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6218 __kmp_internal_end_*. */
6219
6220 /* should we finish the run-time? are all siblings done? */
6221 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6222
6223 for (i = 0; i < __kmp_threads_capacity; ++i) {
6224 if (KMP_UBER_GTID(i)((((i) >= (-6)) ? 0 : __kmp_debug_assert("(i) >= (-6)",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6224)), (((i) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(i) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6224)), (i) >= 0 && __kmp_root[(i)] && __kmp_threads
[(i)] && (__kmp_threads[(i)] == __kmp_root[(i)]->r
.r_uber_thread))
) {
6225 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n"
, i); }
6226 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n"
, i); }
6227 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n"
, i); }
;
6228 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6229 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6230 return;
6231 }
6232 }
6233
6234 /* now we can safely conduct the actual termination */
6235
6236 __kmp_internal_end();
6237
6238 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6239 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6240
6241 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: exit T#%d\n"
, gtid_req); }
;
6242
6243#ifdef DUMP_DEBUG_ON_EXIT
6244 if (__kmp_debug_buf)
6245 __kmp_dump_debug_buffer();
6246#endif
6247} // __kmp_internal_end_thread
6248
6249// -----------------------------------------------------------------------------
6250// Library registration stuff.
6251
6252static long __kmp_registration_flag = 0;
6253// Random value used to indicate library initialization.
6254static char *__kmp_registration_str = NULL__null;
6255// Value to be saved in env var __KMP_REGISTERED_LIB_<pid>.
6256
6257static inline char *__kmp_reg_status_name() {
6258 /* On RHEL 3u5 if linked statically, getpid() returns different values in
6259 each thread. If registration and unregistration go in different threads
6260 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env
6261 env var can not be found, because the name will contain different pid. */
6262 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid());
6263} // __kmp_reg_status_get
6264
6265void __kmp_register_library_startup(void) {
6266
6267 char *name = __kmp_reg_status_name(); // Name of the environment variable.
6268 int done = 0;
6269 union {
6270 double dtime;
6271 long ltime;
6272 } time;
6273#if KMP_ARCH_X860 || KMP_ARCH_X86_641
6274 __kmp_initialize_system_tick();
6275#endif
6276 __kmp_read_system_time(&time.dtime);
6277 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6278 __kmp_registration_str =
6279 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag,
6280 __kmp_registration_flag, KMP_LIBRARY_FILE"libomp.so.5");
6281
6282 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name,if (kmp_a_debug >= 50) { __kmp_debug_printf ("__kmp_register_library_startup: %s=\"%s\"\n"
, name, __kmp_registration_str); }
6283 __kmp_registration_str))if (kmp_a_debug >= 50) { __kmp_debug_printf ("__kmp_register_library_startup: %s=\"%s\"\n"
, name, __kmp_registration_str); }
;
6284
6285 while (!done) {
6286
6287 char *value = NULL__null; // Actual value of the environment variable.
6288
6289 // Set environment variable, but do not overwrite if it is exist.
6290 __kmp_env_set(name, __kmp_registration_str, 0);
6291 // Check the variable is written.
6292 value = __kmp_env_get(name);
6293 if (value != NULL__null && strcmp(value, __kmp_registration_str) == 0) {
6294
6295 done = 1; // Ok, environment variable set successfully, exit the loop.
6296
6297 } else {
6298
6299 // Oops. Write failed. Another copy of OpenMP RTL is in memory.
6300 // Check whether it alive or dead.
6301 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead.
6302 char *tail = value;
6303 char *flag_addr_str = NULL__null;
6304 char *flag_val_str = NULL__null;
6305 char const *file_name = NULL__null;
6306 __kmp_str_split(tail, '-', &flag_addr_str, &tail);
6307 __kmp_str_split(tail, '-', &flag_val_str, &tail);
6308 file_name = tail;
6309 if (tail != NULL__null) {
6310 long *flag_addr = 0;
6311 long flag_val = 0;
6312 KMP_SSCANFsscanf(flag_addr_str, "%p", &flag_addr);
6313 KMP_SSCANFsscanf(flag_val_str, "%lx", &flag_val);
6314 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) {
6315 // First, check whether environment-encoded address is mapped into
6316 // addr space.
6317 // If so, dereference it to see if it still has the right value.
6318 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6319 neighbor = 1;
6320 } else {
6321 // If not, then we know the other copy of the library is no longer
6322 // running.
6323 neighbor = 2;
6324 }
6325 }
6326 }
6327 switch (neighbor) {
6328 case 0: // Cannot parse environment variable -- neighbor status unknown.
6329 // Assume it is the incompatible format of future version of the
6330 // library. Assume the other library is alive.
6331 // WARN( ... ); // TODO: Issue a warning.
6332 file_name = "unknown library";
6333 // Attention! Falling to the next case. That's intentional.
6334 case 1: { // Neighbor is alive.
6335 // Check it is allowed.
6336 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK");
6337 if (!__kmp_str_match_true(duplicate_ok)) {
6338 // That's not allowed. Issue fatal error.
6339 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name)__kmp_msg_format(kmp_i18n_msg_DuplicateLibrary, "libomp.so.5"
, file_name)
,
6340 KMP_HNT(DuplicateLibrary)__kmp_msg_format(kmp_i18n_hnt_DuplicateLibrary), __kmp_msg_null);
6341 }
6342 KMP_INTERNAL_FREE(duplicate_ok)free(duplicate_ok);
6343 __kmp_duplicate_library_ok = 1;
6344 done = 1; // Exit the loop.
6345 } break;
6346 case 2: { // Neighbor is dead.
6347 // Clear the variable and try to register library again.
6348 __kmp_env_unset(name);
6349 } break;
6350 default: { KMP_DEBUG_ASSERT(0)((0) ? 0 : __kmp_debug_assert("0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6350))
; } break;
6351 }
6352 }
6353 KMP_INTERNAL_FREE((void *)value)free((void *)value);
6354 }
6355 KMP_INTERNAL_FREE((void *)name)free((void *)name);
6356
6357} // func __kmp_register_library_startup
6358
6359void __kmp_unregister_library(void) {
6360
6361 char *name = __kmp_reg_status_name();
6362 char *value = __kmp_env_get(name);
6363
6364 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0)((__kmp_registration_flag != 0) ? 0 : __kmp_debug_assert("__kmp_registration_flag != 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6364))
;
6365 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL)((__kmp_registration_str != __null) ? 0 : __kmp_debug_assert(
"__kmp_registration_str != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6365))
;
6366 if (value != NULL__null && strcmp(value, __kmp_registration_str) == 0) {
6367 // Ok, this is our variable. Delete it.
6368 __kmp_env_unset(name);
6369 }
6370
6371 KMP_INTERNAL_FREE(__kmp_registration_str)free(__kmp_registration_str);
6372 KMP_INTERNAL_FREE(value)free(value);
6373 KMP_INTERNAL_FREE(name)free(name);
6374
6375 __kmp_registration_flag = 0;
6376 __kmp_registration_str = NULL__null;
6377
6378} // __kmp_unregister_library
6379
6380// End of Library registration stuff.
6381// -----------------------------------------------------------------------------
6382
6383#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))
6384
6385static void __kmp_check_mic_type() {
6386 kmp_cpuid_t cpuid_state = {0};
6387 kmp_cpuid_t *cs_p = &cpuid_state;
6388 __kmp_x86_cpuid(1, 0, cs_p);
6389 // We don't support mic1 at the moment
6390 if ((cs_p->eax & 0xff0) == 0xB10) {
6391 __kmp_mic_type = mic2;
6392 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6393 __kmp_mic_type = mic3;
6394 } else {
6395 __kmp_mic_type = non_mic;
6396 }
6397}
6398
6399#endif /* KMP_MIC_SUPPORTED */
6400
6401static void __kmp_do_serial_initialize(void) {
6402 int i, gtid;
6403 int size;
6404
6405 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize: enter\n"
); }
;
6406
6407 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4)((sizeof(kmp_int32) == 4) ? 0 : __kmp_debug_assert("sizeof(kmp_int32) == 4"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6407))
;
6408 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4)((sizeof(kmp_uint32) == 4) ? 0 : __kmp_debug_assert("sizeof(kmp_uint32) == 4"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6408))
;
6409 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8)((sizeof(kmp_int64) == 8) ? 0 : __kmp_debug_assert("sizeof(kmp_int64) == 8"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6409))
;
6410 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8)((sizeof(kmp_uint64) == 8) ? 0 : __kmp_debug_assert("sizeof(kmp_uint64) == 8"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6410))
;
6411 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *))((sizeof(kmp_intptr_t) == sizeof(void *)) ? 0 : __kmp_debug_assert
("sizeof(kmp_intptr_t) == sizeof(void *)", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6411))
;
6412
6413#if OMPT_SUPPORT1
6414 ompt_pre_init();
6415#endif
6416
6417 __kmp_validate_locks();
6418
6419 /* Initialize internal memory allocator */
6420 __kmp_init_allocator();
6421
6422 /* Register the library startup via an environment variable and check to see
6423 whether another copy of the library is already registered. */
6424
6425 __kmp_register_library_startup();
6426
6427 /* TODO reinitialization of library */
6428 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
6429 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize: reinitialization of library\n"
); }
;
6430 }
6431
6432 __kmp_global.g.g_abort = 0;
6433 TCW_SYNC_4(__kmp_global.g.g_done, FALSE)(__kmp_global.g.g_done) = (0);
6434
6435/* initialize the locks */
6436#if KMP_USE_ADAPTIVE_LOCKS(0 || 1) && !0
6437#if KMP_DEBUG_ADAPTIVE_LOCKS0
6438 __kmp_init_speculative_stats();
6439#endif
6440#endif
6441#if KMP_STATS_ENABLED0
6442 __kmp_stats_init();
6443#endif
6444 __kmp_init_lock(&__kmp_global_lock);
6445 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6446 __kmp_init_lock(&__kmp_debug_lock);
6447 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6448 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6449 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6450 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6451 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6452 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6453 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6454 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6455 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6456 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6457 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6458 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6459 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6460 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6461 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
6462#if KMP_USE_MONITOR
6463 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
6464#endif
6465 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
6466
6467 /* conduct initialization and initial setup of configuration */
6468
6469 __kmp_runtime_initialize();
6470
6471#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))
6472 __kmp_check_mic_type();
6473#endif
6474
6475// Some global variable initialization moved here from kmp_env_initialize()
6476#ifdef KMP_DEBUG1
6477 kmp_diag = 0;
6478#endif
6479 __kmp_abort_delay = 0;
6480
6481 // From __kmp_init_dflt_team_nth()
6482 /* assume the entire machine will be used */
6483 __kmp_dflt_team_nth_ub = __kmp_xproc;
6484 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH1) {
6485 __kmp_dflt_team_nth_ub = KMP_MIN_NTH1;
6486 }
6487 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6488 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6489 }
6490 __kmp_max_nth = __kmp_sys_max_nth;
6491 __kmp_cg_max_nth = __kmp_sys_max_nth;
6492 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default
6493 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6494 __kmp_teams_max_nth = __kmp_sys_max_nth;
6495 }
6496
6497 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME"
6498 // part
6499 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME(200);
6500#if KMP_USE_MONITOR
6501 __kmp_monitor_wakeups =
6502 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6503 __kmp_bt_intervals =
6504 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6505#endif
6506 // From "KMP_LIBRARY" part of __kmp_env_initialize()
6507 __kmp_library = library_throughput;
6508 // From KMP_SCHEDULE initialization
6509 __kmp_static = kmp_sch_static_balanced;
6510// AC: do not use analytical here, because it is non-monotonous
6511//__kmp_guided = kmp_sch_guided_iterative_chunked;
6512//__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no
6513// need to repeat assignment
6514// Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch
6515// bit control and barrier method control parts
6516#if KMP_FAST_REDUCTION_BARRIER1
6517#define kmp_reduction_barrier_gather_bb ((int)1)
6518#define kmp_reduction_barrier_release_bb ((int)1)
6519#define kmp_reduction_barrier_gather_pat bp_hyper_bar
6520#define kmp_reduction_barrier_release_pat bp_hyper_bar
6521#endif // KMP_FAST_REDUCTION_BARRIER
6522 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6523 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6524 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6525 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6526 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6527#if KMP_FAST_REDUCTION_BARRIER1
6528 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only (
6529 // lin_64 ): hyper,1
6530 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6531 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6532 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6533 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
6534 }
6535#endif // KMP_FAST_REDUCTION_BARRIER
6536 }
6537#if KMP_FAST_REDUCTION_BARRIER1
6538#undef kmp_reduction_barrier_release_pat
6539#undef kmp_reduction_barrier_gather_pat
6540#undef kmp_reduction_barrier_release_bb
6541#undef kmp_reduction_barrier_gather_bb
6542#endif // KMP_FAST_REDUCTION_BARRIER
6543#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))
6544 if (__kmp_mic_type == mic2) { // KNC
6545 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC
6546 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather
6547 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6548 1; // forkjoin release
6549 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6550 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6551 }
6552#if KMP_FAST_REDUCTION_BARRIER1
6553 if (__kmp_mic_type == mic2) { // KNC
6554 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6555 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6556 }
6557#endif // KMP_FAST_REDUCTION_BARRIER
6558#endif // KMP_MIC_SUPPORTED
6559
6560// From KMP_CHECKS initialization
6561#ifdef KMP_DEBUG1
6562 __kmp_env_checks = TRUE(!0); /* development versions have the extra checks */
6563#else
6564 __kmp_env_checks = FALSE0; /* port versions do not have the extra checks */
6565#endif
6566
6567 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization
6568 __kmp_foreign_tp = TRUE(!0);
6569
6570 __kmp_global.g.g_dynamic = FALSE0;
6571 __kmp_global.g.g_dynamic_mode = dynamic_default;
6572
6573 __kmp_env_initialize(NULL__null);
6574
6575// Print all messages in message catalog for testing purposes.
6576#ifdef KMP_DEBUG1
6577 char const *val = __kmp_env_get("KMP_DUMP_CATALOG");
6578 if (__kmp_str_match_true(val)) {
6579 kmp_str_buf_t buffer;
6580 __kmp_str_buf_init(&buffer){ (&buffer)->str = (&buffer)->bulk; (&buffer
)->size = sizeof((&buffer)->bulk); (&buffer)->
used = 0; (&buffer)->bulk[0] = 0; }
;
6581 __kmp_i18n_dump_catalog(&buffer);
6582 __kmp_printf("%s", buffer.str);
6583 __kmp_str_buf_free(&buffer);
6584 }
6585 __kmp_env_free(&val);
6586#endif
6587
6588 __kmp_threads_capacity =
6589 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6590 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part
6591 __kmp_tp_capacity = __kmp_default_tp_capacity(
6592 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
6593
6594 // If the library is shut down properly, both pools must be NULL. Just in
6595 // case, set them to NULL -- some memory may leak, but subsequent code will
6596 // work even if pools are not freed.
6597 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL)((__kmp_thread_pool == __null) ? 0 : __kmp_debug_assert("__kmp_thread_pool == __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6597))
;
6598 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL)((__kmp_thread_pool_insert_pt == __null) ? 0 : __kmp_debug_assert
("__kmp_thread_pool_insert_pt == __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6598))
;
6599 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL)((__kmp_team_pool == __null) ? 0 : __kmp_debug_assert("__kmp_team_pool == __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6599))
;
6600 __kmp_thread_pool = NULL__null;
6601 __kmp_thread_pool_insert_pt = NULL__null;
6602 __kmp_team_pool = NULL__null;
6603
6604 /* Allocate all of the variable sized records */
6605 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are
6606 * expandable */
6607 /* Since allocation is cache-aligned, just add extra padding at the end */
6608 size =
6609 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6610 CACHE_LINE64;
6611 __kmp_threads = (kmp_info_t **)__kmp_allocate(size)___kmp_allocate((size), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6611)
;
6612 __kmp_root = (kmp_root_t **)((char *)__kmp_threads +
6613 sizeof(kmp_info_t *) * __kmp_threads_capacity);
6614
6615 /* init thread counts */
6616 KMP_DEBUG_ASSERT(__kmp_all_nth ==((__kmp_all_nth == 0) ? 0 : __kmp_debug_assert("__kmp_all_nth == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6617))
6617 0)((__kmp_all_nth == 0) ? 0 : __kmp_debug_assert("__kmp_all_nth == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6617))
; // Asserts fail if the library is reinitializing and
6618 KMP_DEBUG_ASSERT(__kmp_nth == 0)((__kmp_nth == 0) ? 0 : __kmp_debug_assert("__kmp_nth == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6618))
; // something was wrong in termination.
6619 __kmp_all_nth = 0;
6620 __kmp_nth = 0;
6621
6622 /* setup the uber master thread and hierarchy */
6623 gtid = __kmp_register_root(TRUE(!0));
6624 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize T#%d\n"
, gtid); }
;
6625 KMP_ASSERT(KMP_UBER_GTID(gtid))((((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6625)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6625)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))) ? 0 : __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6625))
;
6626 KMP_ASSERT(KMP_INITIAL_GTID(gtid))((((gtid) == 0)) ? 0 : __kmp_debug_assert("KMP_INITIAL_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6626))
;
6627
6628 KMP_MB(); /* Flush all pending memory write invalidates. */
6629
6630 __kmp_common_initialize();
6631
6632#if KMP_OS_UNIX1
6633 /* invoke the child fork handler */
6634 __kmp_register_atfork();
6635#endif
6636
6637#if !defined KMP_DYNAMIC_LIB1
6638 {
6639 /* Invoke the exit handler when the program finishes, only for static
6640 library. For dynamic library, we already have _fini and DllMain. */
6641 int rc = atexit(__kmp_internal_end_atexit);
6642 if (rc != 0) {
6643 __kmp_fatal(KMP_MSG(FunctionError, "atexit()")__kmp_msg_format(kmp_i18n_msg_FunctionError, "atexit()"), KMP_ERR(rc)__kmp_msg_error_code(rc),
6644 __kmp_msg_null);
6645 }
6646 }
6647#endif
6648
6649#if KMP_HANDLE_SIGNALS(1 || 0)
6650#if KMP_OS_UNIX1
6651 /* NOTE: make sure that this is called before the user installs their own
6652 signal handlers so that the user handlers are called first. this way they
6653 can return false, not call our handler, avoid terminating the library, and
6654 continue execution where they left off. */
6655 __kmp_install_signals(FALSE0);
6656#endif /* KMP_OS_UNIX */
6657#if KMP_OS_WINDOWS0
6658 __kmp_install_signals(TRUE(!0));
6659#endif /* KMP_OS_WINDOWS */
6660#endif
6661
6662 /* we have finished the serial initialization */
6663 __kmp_init_counter++;
6664
6665 __kmp_init_serial = TRUE(!0);
6666
6667 if (__kmp_settings) {
6668 __kmp_env_print();
6669 }
6670
6671#if OMP_40_ENABLED(50 >= 40)
6672 if (__kmp_display_env || __kmp_display_env_verbose) {
6673 __kmp_env_print_2();
6674 }
6675#endif // OMP_40_ENABLED
6676
6677#if OMPT_SUPPORT1
6678 ompt_post_init();
6679#endif
6680
6681 KMP_MB();
6682
6683 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize: exit\n"
); }
;
6684}
6685
6686void __kmp_serial_initialize(void) {
6687 if (__kmp_init_serial) {
6688 return;
6689 }
6690 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6691 if (__kmp_init_serial) {
6692 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6693 return;
6694 }
6695 __kmp_do_serial_initialize();
6696 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6697}
6698
6699static void __kmp_do_middle_initialize(void) {
6700 int i, j;
6701 int prev_dflt_team_nth;
6702
6703 if (!__kmp_init_serial) {
6704 __kmp_do_serial_initialize();
6705 }
6706
6707 KA_TRACE(10, ("__kmp_middle_initialize: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_middle_initialize: enter\n"
); }
;
6708
6709 // Save the previous value for the __kmp_dflt_team_nth so that
6710 // we can avoid some reinitialization if it hasn't changed.
6711 prev_dflt_team_nth = __kmp_dflt_team_nth;
6712
6713#if KMP_AFFINITY_SUPPORTED1
6714 // __kmp_affinity_initialize() will try to set __kmp_ncores to the
6715 // number of cores on the machine.
6716 __kmp_affinity_initialize();
6717
6718 // Run through the __kmp_threads array and set the affinity mask
6719 // for each root thread that is currently registered with the RTL.
6720 for (i = 0; i < __kmp_threads_capacity; i++) {
6721 if (TCR_PTR(__kmp_threads[i])((void *)(__kmp_threads[i])) != NULL__null) {
6722 __kmp_affinity_set_init_mask(i, TRUE(!0));
6723 }
6724 }
6725#endif /* KMP_AFFINITY_SUPPORTED */
6726
6727 KMP_ASSERT(__kmp_xproc > 0)((__kmp_xproc > 0) ? 0 : __kmp_debug_assert("__kmp_xproc > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6727))
;
6728 if (__kmp_avail_proc == 0) {
6729 __kmp_avail_proc = __kmp_xproc;
6730 }
6731
6732 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3),
6733 // correct them now
6734 j = 0;
6735 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
6736 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
6737 __kmp_avail_proc;
6738 j++;
6739 }
6740
6741 if (__kmp_dflt_team_nth == 0) {
6742#ifdef KMP_DFLT_NTH_CORES
6743 // Default #threads = #cores
6744 __kmp_dflt_team_nth = __kmp_ncores;
6745 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
"__kmp_ncores (%d)\n", __kmp_dflt_team_nth); }
6746 "__kmp_ncores (%d)\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
"__kmp_ncores (%d)\n", __kmp_dflt_team_nth); }
6747 __kmp_dflt_team_nth))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
"__kmp_ncores (%d)\n", __kmp_dflt_team_nth); }
;
6748#else
6749 // Default #threads = #available OS procs
6750 __kmp_dflt_team_nth = __kmp_avail_proc;
6751 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
"__kmp_avail_proc(%d)\n", __kmp_dflt_team_nth); }
6752 "__kmp_avail_proc(%d)\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
"__kmp_avail_proc(%d)\n", __kmp_dflt_team_nth); }
6753 __kmp_dflt_team_nth))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
"__kmp_avail_proc(%d)\n", __kmp_dflt_team_nth); }
;
6754#endif /* KMP_DFLT_NTH_CORES */
6755 }
6756
6757 if (__kmp_dflt_team_nth < KMP_MIN_NTH1) {
6758 __kmp_dflt_team_nth = KMP_MIN_NTH1;
6759 }
6760 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
6761 __kmp_dflt_team_nth = __kmp_sys_max_nth;
6762 }
6763
6764 // There's no harm in continuing if the following check fails,
6765 // but it indicates an error in the previous logic.
6766 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub)((__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub) ? 0 : __kmp_debug_assert
("__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6766))
;
6767
6768 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
6769 // Run through the __kmp_threads array and set the num threads icv for each
6770 // root thread that is currently registered with the RTL (which has not
6771 // already explicitly set its nthreads-var with a call to
6772 // omp_set_num_threads()).
6773 for (i = 0; i < __kmp_threads_capacity; i++) {
6774 kmp_info_t *thread = __kmp_threads[i];
6775 if (thread == NULL__null)
6776 continue;
6777 if (thread->th.th_current_task->td_icvs.nproc != 0)
6778 continue;
6779
6780 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth)(((__kmp_threads[i])->th.th_current_task->td_icvs.nproc
) = (__kmp_dflt_team_nth))
;
6781 }
6782 }
6783 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); }
6784 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); }
6785 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); }
6786 __kmp_dflt_team_nth))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); }
;
6787
6788#ifdef KMP_ADJUST_BLOCKTIME1
6789 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */
6790 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
6791 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0)((__kmp_avail_proc > 0) ? 0 : __kmp_debug_assert("__kmp_avail_proc > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6791))
;
6792 if (__kmp_nth > __kmp_avail_proc) {
6793 __kmp_zero_bt = TRUE(!0);
6794 }
6795 }
6796#endif /* KMP_ADJUST_BLOCKTIME */
6797
6798 /* we have finished middle initialization */
6799 TCW_SYNC_4(__kmp_init_middle, TRUE)(__kmp_init_middle) = ((!0));
6800
6801 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_middle_initialize: exit\n"
); }
;
6802}
6803
6804void __kmp_middle_initialize(void) {
6805 if (__kmp_init_middle) {
6806 return;
6807 }
6808 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6809 if (__kmp_init_middle) {
6810 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6811 return;
6812 }
6813 __kmp_do_middle_initialize();
6814 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6815}
6816
6817void __kmp_parallel_initialize(void) {
6818 int gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg(); // this might be a new root
6819
6820 /* synchronize parallel initialization (for sibling) */
6821 if (TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
6822 return;
6823 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6824 if (TCR_4(__kmp_init_parallel)(__kmp_init_parallel)) {
6825 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6826 return;
6827 }
6828
6829 /* TODO reinitialization after we have already shut down */
6830 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
6831 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: attempt to init while shutting down\n"
); }
6832 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: attempt to init while shutting down\n"
); }
6833 ("__kmp_parallel_initialize: attempt to init while shutting down\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: attempt to init while shutting down\n"
); }
;
6834 __kmp_infinite_loop();
6835 }
6836
6837 /* jc: The lock __kmp_initz_lock is already held, so calling
6838 __kmp_serial_initialize would cause a deadlock. So we call
6839 __kmp_do_serial_initialize directly. */
6840 if (!__kmp_init_middle) {
6841 __kmp_do_middle_initialize();
6842 }
6843
6844 /* begin initialization */
6845 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: enter\n"
); }
;
6846 KMP_ASSERT(KMP_UBER_GTID(gtid))((((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6846)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6846)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))) ? 0 : __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6846))
;
6847
6848#if KMP_ARCH_X860 || KMP_ARCH_X86_641
6849 // Save the FP control regs.
6850 // Worker threads will set theirs to these values at thread startup.
6851 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
6852 __kmp_store_mxcsr(&__kmp_init_mxcsr);
6853 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
6854#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
6855
6856#if KMP_OS_UNIX1
6857#if KMP_HANDLE_SIGNALS(1 || 0)
6858 /* must be after __kmp_serial_initialize */
6859 __kmp_install_signals(TRUE(!0));
6860#endif
6861#endif
6862
6863 __kmp_suspend_initialize();
6864
6865#if defined(USE_LOAD_BALANCE1)
6866 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6867 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
6868 }
6869#else
6870 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6871 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
6872 }
6873#endif
6874
6875 if (__kmp_version) {
6876 __kmp_print_version_2();
6877 }
6878
6879 /* we have finished parallel initialization */
6880 TCW_SYNC_4(__kmp_init_parallel, TRUE)(__kmp_init_parallel) = ((!0));
6881
6882 KMP_MB();
6883 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: exit\n"
); }
;
6884
6885 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6886}
6887
6888/* ------------------------------------------------------------------------ */
6889
6890void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
6891 kmp_team_t *team) {
6892 kmp_disp_t *dispatch;
6893
6894 KMP_MB();
6895
6896 /* none of the threads have encountered any constructs, yet. */
6897 this_thr->th.th_local.this_construct = 0;
6898#if KMP_CACHE_MANAGE
6899 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
6900#endif /* KMP_CACHE_MANAGE */
6901 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch)((void *)(this_thr->th.th_dispatch));
6902 KMP_DEBUG_ASSERT(dispatch)((dispatch) ? 0 : __kmp_debug_assert("dispatch", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6902))
;
6903 KMP_DEBUG_ASSERT(team->t.t_dispatch)((team->t.t_dispatch) ? 0 : __kmp_debug_assert("team->t.t_dispatch"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6903))
;
6904 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[
6905 // this_thr->th.th_info.ds.ds_tid ] );
6906
6907 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */
6908#if OMP_45_ENABLED(50 >= 45)
6909 dispatch->th_doacross_buf_idx =
6910 0; /* reset the doacross dispatch buffer counter */
6911#endif
6912 if (__kmp_env_consistency_check)
6913 __kmp_push_parallel(gtid, team->t.t_ident);
6914
6915 KMP_MB(); /* Flush all pending memory write invalidates. */
6916}
6917
6918void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
6919 kmp_team_t *team) {
6920 if (__kmp_env_consistency_check)
6921 __kmp_pop_parallel(gtid, team->t.t_ident);
6922
6923 __kmp_finish_implicit_task(this_thr);
6924}
6925
6926int __kmp_invoke_task_func(int gtid) {
6927 int rc;
6928 int tid = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6928)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
6929 kmp_info_t *this_thr = __kmp_threads[gtid];
6930 kmp_team_t *team = this_thr->th.th_team;
6931
6932 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
6933#if USE_ITT_BUILD1
6934 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
6935 __kmp_itt_stack_callee_enter(
6936 (__itt_caller)
6937 team->t.t_stack_id); // inform ittnotify about entering user's code
6938 }
6939#endif /* USE_ITT_BUILD */
6940#if INCLUDE_SSC_MARKS(1 && 1)
6941 SSC_MARK_INVOKING()__asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
::"i"(0xd695) : "%ebx")
;
6942#endif
6943
6944#if OMPT_SUPPORT1
6945 void *dummy;
6946 void **exit_runtime_p;
6947 ompt_data_t *my_task_data;
6948 ompt_data_t *my_parallel_data;
6949 int ompt_team_size;
6950
6951 if (ompt_enabled.enabled) {
6952 exit_runtime_p = &(
6953 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame);
6954 } else {
6955 exit_runtime_p = &dummy;
6956 }
6957
6958 my_task_data =
6959 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
6960 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
6961 if (ompt_enabled.ompt_callback_implicit_task) {
6962 ompt_team_size = team->t.t_nproc;
6963 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
6964 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
6965 __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6965)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
);
6966 OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->thread_num = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 6966)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
6967 }
6968#endif
6969
6970 {
6971 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
6972 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
6973 rc =
6974 __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn)((void *)(team->t.t_pkfn)), gtid,
6975 tid, (int)team->t.t_argc, (void **)team->t.t_argv
6976#if OMPT_SUPPORT1
6977 ,
6978 exit_runtime_p
6979#endif
6980 );
6981#if OMPT_SUPPORT1
6982 *exit_runtime_p = NULL__null;
6983#endif
6984 }
6985
6986#if USE_ITT_BUILD1
6987 if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
6988 __kmp_itt_stack_callee_leave(
6989 (__itt_caller)
6990 team->t.t_stack_id); // inform ittnotify about leaving user's code
6991 }
6992#endif /* USE_ITT_BUILD */
6993 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
6994
6995 return rc;
6996}
6997
6998#if OMP_40_ENABLED(50 >= 40)
6999void __kmp_teams_master(int gtid) {
7000 // This routine is called by all master threads in teams construct
7001 kmp_info_t *thr = __kmp_threads[gtid];
7002 kmp_team_t *team = thr->th.th_team;
7003 ident_t *loc = team->t.t_ident;
7004 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7005 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask)((thr->th.th_teams_microtask) ? 0 : __kmp_debug_assert("thr->th.th_teams_microtask"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7005))
;
7006 KMP_DEBUG_ASSERT(thr->th.th_set_nproc)((thr->th.th_set_nproc) ? 0 : __kmp_debug_assert("thr->th.th_set_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7006))
;
7007 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n"
, gtid, ((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7008)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), thr
->th.th_teams_microtask); }
7008 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n"
, gtid, ((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7008)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid), thr
->th.th_teams_microtask); }
;
7009// Launch league of teams now, but not let workers execute
7010// (they hang on fork barrier until next parallel)
7011#if INCLUDE_SSC_MARKS(1 && 1)
7012 SSC_MARK_FORKING()__asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
::"i"(0xd693) : "%ebx")
;
7013#endif
7014 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
7015 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task
7016 VOLATILE_CAST(launch_t)(launch_t) __kmp_invoke_task_func, NULL__null);
7017#if INCLUDE_SSC_MARKS(1 && 1)
7018 SSC_MARK_JOINING()__asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
::"i"(0xd694) : "%ebx")
;
7019#endif
7020
7021 // AC: last parameter "1" eliminates join barrier which won't work because
7022 // worker threads are in a fork barrier waiting for more parallel regions
7023 __kmp_join_call(loc, gtid
7024#if OMPT_SUPPORT1
7025 ,
7026 fork_context_intel
7027#endif
7028 ,
7029 1);
7030}
7031
7032int __kmp_invoke_teams_master(int gtid) {
7033 kmp_info_t *this_thr = __kmp_threads[gtid];
7034 kmp_team_t *team = this_thr->th.th_team;
7035#if KMP_DEBUG1
7036 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7037 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==(((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == (
void *)__kmp_teams_master) ? 0 : __kmp_debug_assert("(void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == (void *)__kmp_teams_master"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7038))
7038 (void *)__kmp_teams_master)(((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == (
void *)__kmp_teams_master) ? 0 : __kmp_debug_assert("(void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == (void *)__kmp_teams_master"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7038))
;
7039#endif
7040 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7041 __kmp_teams_master(gtid);
7042 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7043 return 1;
7044}
7045#endif /* OMP_40_ENABLED */
7046
7047/* this sets the requested number of threads for the next parallel region
7048 encountered by this team. since this should be enclosed in the forkjoin
7049 critical section it should avoid race conditions with assymmetrical nested
7050 parallelism */
7051
7052void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) {
7053 kmp_info_t *thr = __kmp_threads[gtid];
7054
7055 if (num_threads > 0)
7056 thr->th.th_set_nproc = num_threads;
7057}
7058
7059#if OMP_40_ENABLED(50 >= 40)
7060
7061/* this sets the requested number of teams for the teams region and/or
7062 the number of threads for the next parallel region encountered */
7063void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams,
7064 int num_threads) {
7065 kmp_info_t *thr = __kmp_threads[gtid];
7066 KMP_DEBUG_ASSERT(num_teams >= 0)((num_teams >= 0) ? 0 : __kmp_debug_assert("num_teams >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7066))
;
7067 KMP_DEBUG_ASSERT(num_threads >= 0)((num_threads >= 0) ? 0 : __kmp_debug_assert("num_threads >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7067))
;
7068
7069 if (num_teams == 0)
7070 num_teams = 1; // default number of teams is 1.
7071 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested?
7072 if (!__kmp_reserve_warn) {
7073 __kmp_reserve_warn = 1;
7074 __kmp_msg(kmp_ms_warning,
7075 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, num_teams, __kmp_teams_max_nth
)
,
7076 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
7077 }
7078 num_teams = __kmp_teams_max_nth;
7079 }
7080 // Set number of teams (number of threads in the outer "parallel" of the
7081 // teams)
7082 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7083
7084 // Remember the number of threads for inner parallel regions
7085 if (num_threads == 0) {
7086 if (!TCR_4(__kmp_init_middle)(__kmp_init_middle))
7087 __kmp_middle_initialize(); // get __kmp_avail_proc calculated
7088 num_threads = __kmp_avail_proc / num_teams;
7089 if (num_teams * num_threads > __kmp_teams_max_nth) {
7090 // adjust num_threads w/o warning as it is not user setting
7091 num_threads = __kmp_teams_max_nth / num_teams;
7092 }
7093 } else {
7094 if (num_teams * num_threads > __kmp_teams_max_nth) {
7095 int new_threads = __kmp_teams_max_nth / num_teams;
7096 if (!__kmp_reserve_warn) { // user asked for too many threads
7097 __kmp_reserve_warn = 1; // that conflicts with KMP_TEAMS_THREAD_LIMIT
7098 __kmp_msg(kmp_ms_warning,
7099 KMP_MSG(CantFormThrTeam, num_threads, new_threads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, num_threads, new_threads
)
,
7100 KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
7101 }
7102 num_threads = new_threads;
7103 }
7104 }
7105 thr->th.th_teams_size.nth = num_threads;
7106}
7107
7108// Set the proc_bind var to use in the following parallel region.
7109void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) {
7110 kmp_info_t *thr = __kmp_threads[gtid];
7111 thr->th.th_set_proc_bind = proc_bind;
7112}
7113
7114#endif /* OMP_40_ENABLED */
7115
7116/* Launch the worker threads into the microtask. */
7117
7118void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) {
7119 kmp_info_t *this_thr = __kmp_threads[gtid];
7120
7121#ifdef KMP_DEBUG1
7122 int f;
7123#endif /* KMP_DEBUG */
7124
7125 KMP_DEBUG_ASSERT(team)((team) ? 0 : __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7125))
;
7126 KMP_DEBUG_ASSERT(this_thr->th.th_team == team)((this_thr->th.th_team == team) ? 0 : __kmp_debug_assert("this_thr->th.th_team == team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7126))
;
7127 KMP_ASSERT(KMP_MASTER_GTID(gtid))((((((((gtid)) >= 0) ? 0 : __kmp_debug_assert("((gtid)) >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7127)), __kmp_threads[((gtid))]->th.th_info.ds.ds_tid) ==
0)) ? 0 : __kmp_debug_assert("KMP_MASTER_GTID(gtid)", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7127))
;
7128 KMP_MB(); /* Flush all pending memory write invalidates. */
7129
7130 team->t.t_construct = 0; /* no single directives seen yet */
7131 team->t.t_ordered.dt.t_value =
7132 0; /* thread 0 enters the ordered section first */
7133
7134 /* Reset the identifiers on the dispatch buffer */
7135 KMP_DEBUG_ASSERT(team->t.t_disp_buffer)((team->t.t_disp_buffer) ? 0 : __kmp_debug_assert("team->t.t_disp_buffer"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7135))
;
7136 if (team->t.t_max_nproc > 1) {
7137 int i;
7138 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7139 team->t.t_disp_buffer[i].buffer_index = i;
7140#if OMP_45_ENABLED(50 >= 45)
7141 team->t.t_disp_buffer[i].doacross_buf_idx = i;
7142#endif
7143 }
7144 } else {
7145 team->t.t_disp_buffer[0].buffer_index = 0;
7146#if OMP_45_ENABLED(50 >= 45)
7147 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7148#endif
7149 }
7150
7151 KMP_MB(); /* Flush all pending memory write invalidates. */
7152 KMP_ASSERT(this_thr->th.th_team == team)((this_thr->th.th_team == team) ? 0 : __kmp_debug_assert("this_thr->th.th_team == team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7152))
;
7153
7154#ifdef KMP_DEBUG1
7155 for (f = 0; f < team->t.t_nproc; f++) {
7156 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7157))
7157 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc)((team->t.t_threads[f] && team->t.t_threads[f]->
th.th_team_nproc == team->t.t_nproc) ? 0 : __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7157))
;
7158 }
7159#endif /* KMP_DEBUG */
7160
7161 /* release the worker threads so they may begin working */
7162 __kmp_fork_barrier(gtid, 0);
7163}
7164
7165void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) {
7166 kmp_info_t *this_thr = __kmp_threads[gtid];
7167
7168 KMP_DEBUG_ASSERT(team)((team) ? 0 : __kmp_debug_assert("team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7168))
;
7169 KMP_DEBUG_ASSERT(this_thr->th.th_team == team)((this_thr->th.th_team == team) ? 0 : __kmp_debug_assert("this_thr->th.th_team == team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7169))
;
7170 KMP_ASSERT(KMP_MASTER_GTID(gtid))((((((((gtid)) >= 0) ? 0 : __kmp_debug_assert("((gtid)) >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7170)), __kmp_threads[((gtid))]->th.th_info.ds.ds_tid) ==
0)) ? 0 : __kmp_debug_assert("KMP_MASTER_GTID(gtid)", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7170))
;
7171 KMP_MB(); /* Flush all pending memory write invalidates. */
7172
7173/* Join barrier after fork */
7174
7175#ifdef KMP_DEBUG1
7176 if (__kmp_threads[gtid] &&
7177 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7178 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7179 __kmp_threads[gtid]);
7180 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "
7181 "team->t.t_nproc=%d\n",
7182 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7183 team->t.t_nproc);
7184 __kmp_print_structure();
7185 }
7186 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&((__kmp_threads[gtid] && __kmp_threads[gtid]->th.th_team_nproc
== team->t.t_nproc) ? 0 : __kmp_debug_assert("__kmp_threads[gtid] && __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7187))
7187 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc)((__kmp_threads[gtid] && __kmp_threads[gtid]->th.th_team_nproc
== team->t.t_nproc) ? 0 : __kmp_debug_assert("__kmp_threads[gtid] && __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7187))
;
7188#endif /* KMP_DEBUG */
7189
7190 __kmp_join_barrier(gtid); /* wait for everyone */
7191#if OMPT_SUPPORT1
7192 if (ompt_enabled.enabled &&
7193 this_thr->th.ompt_thread_info.state == omp_state_wait_barrier_implicit) {
7194 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7195 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
))
;
7196 this_thr->th.ompt_thread_info.state = omp_state_overhead;
7197#if OMPT_OPTIONAL1
7198 void *codeptr = NULL__null;
7199 if (KMP_MASTER_TID(ds_tid)((ds_tid) == 0) &&
7200 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback ||
7201 ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback))
7202 codeptr = OMPT_CUR_TEAM_INFO(this_thr)(&(this_thr->th.th_team->t.ompt_team_info))->master_return_address;
7203
7204 if (ompt_enabled.ompt_callback_sync_region_wait) {
7205 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback(
7206 ompt_sync_region_barrier, ompt_scope_end, NULL__null, task_data, codeptr);
7207 }
7208 if (ompt_enabled.ompt_callback_sync_region) {
7209 ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback(
7210 ompt_sync_region_barrier, ompt_scope_end, NULL__null, task_data, codeptr);
7211 }
7212#endif
7213 if (!KMP_MASTER_TID(ds_tid)((ds_tid) == 0) && ompt_enabled.ompt_callback_implicit_task) {
7214 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
7215 ompt_scope_end, NULL__null, task_data, 0, ds_tid);
7216 }
7217 }
7218#endif
7219
7220 KMP_MB(); /* Flush all pending memory write invalidates. */
7221 KMP_ASSERT(this_thr->th.th_team == team)((this_thr->th.th_team == team) ? 0 : __kmp_debug_assert("this_thr->th.th_team == team"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7221))
;
7222}
7223
7224/* ------------------------------------------------------------------------ */
7225
7226#ifdef USE_LOAD_BALANCE1
7227
7228// Return the worker threads actively spinning in the hot team, if we
7229// are at the outermost level of parallelism. Otherwise, return 0.
7230static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7231 int i;
7232 int retval;
7233 kmp_team_t *hot_team;
7234
7235 if (root->r.r_active) {
7236 return 0;
7237 }
7238 hot_team = root->r.r_hot_team;
7239 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME(2147483647)) {
7240 return hot_team->t.t_nproc - 1; // Don't count master thread
7241 }
7242
7243 // Skip the master thread - it is accounted for elsewhere.
7244 retval = 0;
7245 for (i = 1; i < hot_team->t.t_nproc; i++) {
7246 if (hot_team->t.t_threads[i]->th.th_active) {
7247 retval++;
7248 }
7249 }
7250 return retval;
7251}
7252
7253// Perform an automatic adjustment to the number of
7254// threads used by the next parallel region.
7255static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) {
7256 int retval;
7257 int pool_active;
7258 int hot_team_active;
7259 int team_curr_active;
7260 int system_active;
7261
7262 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n"
, root, set_nproc); }
7263 set_nproc))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n"
, root, set_nproc); }
;
7264 KMP_DEBUG_ASSERT(root)((root) ? 0 : __kmp_debug_assert("root", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7264))
;
7265 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]((root->r.r_root_team->t.t_threads[0] ->th.th_current_task
->td_icvs.dynamic == (!0)) ? 0 : __kmp_debug_assert("root->r.r_root_team->t.t_threads[0] ->th.th_current_task->td_icvs.dynamic == (!0)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7266))
7266 ->th.th_current_task->td_icvs.dynamic == TRUE)((root->r.r_root_team->t.t_threads[0] ->th.th_current_task
->td_icvs.dynamic == (!0)) ? 0 : __kmp_debug_assert("root->r.r_root_team->t.t_threads[0] ->th.th_current_task->td_icvs.dynamic == (!0)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7266))
;
7267 KMP_DEBUG_ASSERT(set_nproc > 1)((set_nproc > 1) ? 0 : __kmp_debug_assert("set_nproc > 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7267))
;
7268
7269 if (set_nproc == 1) {
7270 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n"))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: serial execution.\n"
); }
;
7271 return 1;
7272 }
7273
7274 // Threads that are active in the thread pool, active in the hot team for this
7275 // particular root (if we are at the outer par level), and the currently
7276 // executing thread (to become the master) are available to add to the new
7277 // team, but are currently contributing to the system load, and must be
7278 // accounted for.
7279 pool_active = __kmp_thread_pool_active_nth;
7280 hot_team_active = __kmp_active_hot_team_nproc(root);
7281 team_curr_active = pool_active + hot_team_active + 1;
7282
7283 // Check the system load.
7284 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7285 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d "if (kmp_b_debug >= 30) { __kmp_debug_printf ("__kmp_load_balance_nproc: system active = %d pool active = %d "
"hot team active = %d\n", system_active, pool_active, hot_team_active
); }
7286 "hot team active = %d\n",if (kmp_b_debug >= 30) { __kmp_debug_printf ("__kmp_load_balance_nproc: system active = %d pool active = %d "
"hot team active = %d\n", system_active, pool_active, hot_team_active
); }
7287 system_active, pool_active, hot_team_active))if (kmp_b_debug >= 30) { __kmp_debug_printf ("__kmp_load_balance_nproc: system active = %d pool active = %d "
"hot team active = %d\n", system_active, pool_active, hot_team_active
); }
;
7288
7289 if (system_active < 0) {
7290 // There was an error reading the necessary info from /proc, so use the
7291 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode
7292 // = dynamic_thread_limit, we shouldn't wind up getting back here.
7293 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7294 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit")__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_CantLoadBalUsing
, "KMP_DYNAMIC_MODE=thread limit"), __kmp_msg_null)
;
7295
7296 // Make this call behave like the thread limit algorithm.
7297 retval = __kmp_avail_proc - __kmp_nth +
7298 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7299 if (retval > set_nproc) {
7300 retval = set_nproc;
7301 }
7302 if (retval < KMP_MIN_NTH1) {
7303 retval = KMP_MIN_NTH1;
7304 }
7305
7306 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n",if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n"
, retval); }
7307 retval))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n"
, retval); }
;
7308 return retval;
7309 }
7310
7311 // There is a slight delay in the load balance algorithm in detecting new
7312 // running procs. The real system load at this instant should be at least as
7313 // large as the #active omp thread that are available to add to the team.
7314 if (system_active < team_curr_active) {
7315 system_active = team_curr_active;
7316 }
7317 retval = __kmp_avail_proc - system_active + team_curr_active;
7318 if (retval > set_nproc) {
7319 retval = set_nproc;
7320 }
7321 if (retval < KMP_MIN_NTH1) {
7322 retval = KMP_MIN_NTH1;
7323 }
7324
7325 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: exit. retval:%d\n"
, retval); }
;
7326 return retval;
7327} // __kmp_load_balance_nproc()
7328
7329#endif /* USE_LOAD_BALANCE */
7330
7331/* ------------------------------------------------------------------------ */
7332
7333/* NOTE: this is called with the __kmp_init_lock held */
7334void __kmp_cleanup(void) {
7335 int f;
7336
7337 KA_TRACE(10, ("__kmp_cleanup: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_cleanup: enter\n"
); }
;
7338
7339 if (TCR_4(__kmp_init_parallel)(__kmp_init_parallel)) {
7340#if KMP_HANDLE_SIGNALS(1 || 0)
7341 __kmp_remove_signals();
7342#endif
7343 TCW_4(__kmp_init_parallel, FALSE)(__kmp_init_parallel) = (0);
7344 }
7345
7346 if (TCR_4(__kmp_init_middle)(__kmp_init_middle)) {
7347#if KMP_AFFINITY_SUPPORTED1
7348 __kmp_affinity_uninitialize();
7349#endif /* KMP_AFFINITY_SUPPORTED */
7350 __kmp_cleanup_hierarchy();
7351 TCW_4(__kmp_init_middle, FALSE)(__kmp_init_middle) = (0);
7352 }
7353
7354 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_cleanup: go serial cleanup\n"
); }
;
7355
7356 if (__kmp_init_serial) {
7357 __kmp_runtime_destroy();
7358 __kmp_init_serial = FALSE0;
7359 }
7360
7361 __kmp_cleanup_threadprivate_caches();
7362
7363 for (f = 0; f < __kmp_threads_capacity; f++) {
7364 if (__kmp_root[f] != NULL__null) {
7365 __kmp_free(__kmp_root[f])___kmp_free((__kmp_root[f]), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7365)
;
7366 __kmp_root[f] = NULL__null;
7367 }
7368 }
7369 __kmp_free(__kmp_threads)___kmp_free((__kmp_threads), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7369)
;
7370 // __kmp_threads and __kmp_root were allocated at once, as single block, so
7371 // there is no need in freeing __kmp_root.
7372 __kmp_threads = NULL__null;
7373 __kmp_root = NULL__null;
7374 __kmp_threads_capacity = 0;
7375
7376#if KMP_USE_DYNAMIC_LOCK1
7377 __kmp_cleanup_indirect_user_locks();
7378#else
7379 __kmp_cleanup_user_locks();
7380#endif
7381
7382#if KMP_AFFINITY_SUPPORTED1
7383 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file))free(const_cast<char *>(__kmp_cpuinfo_file));
7384 __kmp_cpuinfo_file = NULL__null;
7385#endif /* KMP_AFFINITY_SUPPORTED */
7386
7387#if KMP_USE_ADAPTIVE_LOCKS(0 || 1) && !0
7388#if KMP_DEBUG_ADAPTIVE_LOCKS0
7389 __kmp_print_speculative_stats();
7390#endif
7391#endif
7392 KMP_INTERNAL_FREE(__kmp_nested_nth.nth)free(__kmp_nested_nth.nth);
7393 __kmp_nested_nth.nth = NULL__null;
7394 __kmp_nested_nth.size = 0;
7395 __kmp_nested_nth.used = 0;
7396 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types)free(__kmp_nested_proc_bind.bind_types);
7397 __kmp_nested_proc_bind.bind_types = NULL__null;
7398 __kmp_nested_proc_bind.size = 0;
7399 __kmp_nested_proc_bind.used = 0;
7400
7401 __kmp_i18n_catclose();
7402
7403#if KMP_USE_HIER_SCHED0
7404 __kmp_hier_scheds.deallocate();
7405#endif
7406
7407#if KMP_STATS_ENABLED0
7408 __kmp_stats_fini();
7409#endif
7410
7411 KA_TRACE(10, ("__kmp_cleanup: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_cleanup: exit\n"
); }
;
7412}
7413
7414/* ------------------------------------------------------------------------ */
7415
7416int __kmp_ignore_mppbeg(void) {
7417 char *env;
7418
7419 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL__null) {
7420 if (__kmp_str_match_false(env))
7421 return FALSE0;
7422 }
7423 // By default __kmpc_begin() is no-op.
7424 return TRUE(!0);
7425}
7426
7427int __kmp_ignore_mppend(void) {
7428 char *env;
7429
7430 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL__null) {
7431 if (__kmp_str_match_false(env))
7432 return FALSE0;
7433 }
7434 // By default __kmpc_end() is no-op.
7435 return TRUE(!0);
7436}
7437
7438void __kmp_internal_begin(void) {
7439 int gtid;
7440 kmp_root_t *root;
7441
7442 /* this is a very important step as it will register new sibling threads
7443 and assign these new uber threads a new gtid */
7444 gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
7445 root = __kmp_threads[gtid]->th.th_root;
7446 KMP_ASSERT(KMP_UBER_GTID(gtid))((((((gtid) >= (-6)) ? 0 : __kmp_debug_assert("(gtid) >= (-6)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7446)), (((gtid) < __kmp_threads_capacity) ? 0 : __kmp_debug_assert
("(gtid) < __kmp_threads_capacity", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7446)), (gtid) >= 0 && __kmp_root[(gtid)] &&
__kmp_threads[(gtid)] && (__kmp_threads[(gtid)] == __kmp_root
[(gtid)]->r.r_uber_thread))) ? 0 : __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7446))
;
7447
7448 if (root->r.r_begin)
7449 return;
7450 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7451 if (root->r.r_begin) {
7452 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7453 return;
7454 }
7455
7456 root->r.r_begin = TRUE(!0);
7457
7458 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7459}
7460
7461/* ------------------------------------------------------------------------ */
7462
7463void __kmp_user_set_library(enum library_type arg) {
7464 int gtid;
7465 kmp_root_t *root;
7466 kmp_info_t *thread;
7467
7468 /* first, make sure we are initialized so we can get our gtid */
7469
7470 gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
7471 thread = __kmp_threads[gtid];
7472
7473 root = thread->th.th_root;
7474
7475 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n"
, gtid, arg, library_serial); }
7476 library_serial))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n"
, gtid, arg, library_serial); }
;
7477 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level
7478 thread */
7479 KMP_WARNING(SetLibraryIncorrectCall)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_SetLibraryIncorrectCall
), __kmp_msg_null)
;
7480 return;
7481 }
7482
7483 switch (arg) {
7484 case library_serial:
7485 thread->th.th_set_nproc = 0;
7486 set__nproc(thread, 1)(((thread)->th.th_current_task->td_icvs.nproc) = (1));
7487 break;
7488 case library_turnaround:
7489 thread->th.th_set_nproc = 0;
7490 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub))
7491 : __kmp_dflt_team_nth_ub)(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub))
;
7492 break;
7493 case library_throughput:
7494 thread->th.th_set_nproc = 0;
7495 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub))
7496 : __kmp_dflt_team_nth_ub)(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub))
;
7497 break;
7498 default:
7499 KMP_FATAL(UnknownLibraryType, arg)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownLibraryType,
arg), __kmp_msg_null)
;
7500 }
7501
7502 __kmp_aux_set_library(arg);
7503}
7504
7505void __kmp_aux_set_stacksize(size_t arg) {
7506 if (!__kmp_init_serial)
7507 __kmp_serial_initialize();
7508
7509#if KMP_OS_DARWIN0
7510 if (arg & (0x1000 - 1)) {
7511 arg &= ~(0x1000 - 1);
7512 if (arg + 0x1000) /* check for overflow if we round up */
7513 arg += 0x1000;
7514 }
7515#endif
7516 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7517
7518 /* only change the default stacksize before the first parallel region */
7519 if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel)) {
7520 size_t value = arg; /* argument is in bytes */
7521
7522 if (value < __kmp_sys_min_stksize)
7523 value = __kmp_sys_min_stksize;
7524 else if (value > KMP_MAX_STKSIZE(~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)
))
)
7525 value = KMP_MAX_STKSIZE(~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)
))
;
7526
7527 __kmp_stksize = value;
7528
7529 __kmp_env_stksize = TRUE(!0); /* was KMP_STACKSIZE specified? */
7530 }
7531
7532 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7533}
7534
7535/* set the behaviour of the runtime library */
7536/* TODO this can cause some odd behaviour with sibling parallelism... */
7537void __kmp_aux_set_library(enum library_type arg) {
7538 __kmp_library = arg;
7539
7540 switch (__kmp_library) {
7541 case library_serial: {
7542 KMP_INFORM(LibraryIsSerial)__kmp_msg(kmp_ms_inform, __kmp_msg_format(kmp_i18n_msg_LibraryIsSerial
), __kmp_msg_null)
;
7543 (void)__kmp_change_library(TRUE(!0));
7544 } break;
7545 case library_turnaround:
7546 (void)__kmp_change_library(TRUE(!0));
7547 break;
7548 case library_throughput:
7549 (void)__kmp_change_library(FALSE0);
7550 break;
7551 default:
7552 KMP_FATAL(UnknownLibraryType, arg)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownLibraryType,
arg), __kmp_msg_null)
;
7553 }
7554}
7555
7556/* ------------------------------------------------------------------------ */
7557
7558void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) {
7559 int blocktime = arg; /* argument is in milliseconds */
7560#if KMP_USE_MONITOR
7561 int bt_intervals;
7562#endif
7563 int bt_set;
7564
7565 __kmp_save_internal_controls(thread);
7566
7567 /* Normalize and set blocktime for the teams */
7568 if (blocktime < KMP_MIN_BLOCKTIME(0))
7569 blocktime = KMP_MIN_BLOCKTIME(0);
7570 else if (blocktime > KMP_MAX_BLOCKTIME(2147483647))
7571 blocktime = KMP_MAX_BLOCKTIME(2147483647);
7572
7573 set__blocktime_team(thread->th.th_team, tid, blocktime)(((thread->th.th_team)->t.t_threads[(tid)]->th.th_current_task
->td_icvs.blocktime) = (blocktime))
;
7574 set__blocktime_team(thread->th.th_serial_team, 0, blocktime)(((thread->th.th_serial_team)->t.t_threads[(0)]->th.
th_current_task->td_icvs.blocktime) = (blocktime))
;
7575
7576#if KMP_USE_MONITOR
7577 /* Calculate and set blocktime intervals for the teams */
7578 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
7579
7580 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
7581 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
7582#endif
7583
7584 /* Set whether blocktime has been set to "TRUE" */
7585 bt_set = TRUE(!0);
7586
7587 set__bt_set_team(thread->th.th_team, tid, bt_set)(((thread->th.th_team)->t.t_threads[(tid)]->th.th_current_task
->td_icvs.bt_set) = (bt_set))
;
7588 set__bt_set_team(thread->th.th_serial_team, 0, bt_set)(((thread->th.th_serial_team)->t.t_threads[(0)]->th.
th_current_task->td_icvs.bt_set) = (bt_set))
;
7589#if KMP_USE_MONITOR
7590 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
"bt_intervals=%d, monitor_updates=%d\n", ((((tid) >= 0 &&
(thread->th.th_team) != __null) ? 0 : __kmp_debug_assert(
"(tid) >= 0 && (thread->th.th_team) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7592)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
, bt_intervals, __kmp_monitor_wakeups); }
7591 "bt_intervals=%d, monitor_updates=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
"bt_intervals=%d, monitor_updates=%d\n", ((((tid) >= 0 &&
(thread->th.th_team) != __null) ? 0 : __kmp_debug_assert(
"(tid) >= 0 && (thread->th.th_team) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7592)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
, bt_intervals, __kmp_monitor_wakeups); }
7592 __kmp_gtid_from_tid(tid, thread->th.th_team),if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
"bt_intervals=%d, monitor_updates=%d\n", ((((tid) >= 0 &&
(thread->th.th_team) != __null) ? 0 : __kmp_debug_assert(
"(tid) >= 0 && (thread->th.th_team) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7592)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
, bt_intervals, __kmp_monitor_wakeups); }
7593 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
"bt_intervals=%d, monitor_updates=%d\n", ((((tid) >= 0 &&
(thread->th.th_team) != __null) ? 0 : __kmp_debug_assert(
"(tid) >= 0 && (thread->th.th_team) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7592)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
, bt_intervals, __kmp_monitor_wakeups); }
7594 __kmp_monitor_wakeups))if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
"bt_intervals=%d, monitor_updates=%d\n", ((((tid) >= 0 &&
(thread->th.th_team) != __null) ? 0 : __kmp_debug_assert(
"(tid) >= 0 && (thread->th.th_team) != __null",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7592)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
, bt_intervals, __kmp_monitor_wakeups); }
;
7595#else
7596 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n"
, ((((tid) >= 0 && (thread->th.th_team) != __null
) ? 0 : __kmp_debug_assert("(tid) >= 0 && (thread->th.th_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7597)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
); }
7597 __kmp_gtid_from_tid(tid, thread->th.th_team),if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n"
, ((((tid) >= 0 && (thread->th.th_team) != __null
) ? 0 : __kmp_debug_assert("(tid) >= 0 && (thread->th.th_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7597)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
); }
7598 thread->th.th_team->t.t_id, tid, blocktime))if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n"
, ((((tid) >= 0 && (thread->th.th_team) != __null
) ? 0 : __kmp_debug_assert("(tid) >= 0 && (thread->th.th_team) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7597)), thread->th.th_team->t.t_threads[(tid)]->th
.th_info.ds.ds_gtid), thread->th.th_team->t.t_id, tid, blocktime
); }
;
7599#endif
7600}
7601
7602void __kmp_aux_set_defaults(char const *str, int len) {
7603 if (!__kmp_init_serial) {
7604 __kmp_serial_initialize();
7605 }
7606 __kmp_env_initialize(str);
7607
7608 if (__kmp_settings
7609#if OMP_40_ENABLED(50 >= 40)
7610 || __kmp_display_env || __kmp_display_env_verbose
7611#endif // OMP_40_ENABLED
7612 ) {
7613 __kmp_env_print();
7614 }
7615} // __kmp_aux_set_defaults
7616
7617/* ------------------------------------------------------------------------ */
7618/* internal fast reduction routines */
7619
7620PACKED_REDUCTION_METHOD_T
7621__kmp_determine_reduction_method(
7622 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
7623 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
7624 kmp_critical_name *lck) {
7625
7626 // Default reduction method: critical construct ( lck != NULL, like in current
7627 // PAROPT )
7628 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method
7629 // can be selected by RTL
7630 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method
7631 // can be selected by RTL
7632 // Finally, it's up to OpenMP RTL to make a decision on which method to select
7633 // among generated by PAROPT.
7634
7635 PACKED_REDUCTION_METHOD_T retval;
7636
7637 int team_size;
7638
7639 KMP_DEBUG_ASSERT(loc)((loc) ? 0 : __kmp_debug_assert("loc", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7639))
; // it would be nice to test ( loc != 0 )
7640 KMP_DEBUG_ASSERT(lck)((lck) ? 0 : __kmp_debug_assert("lck", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7640))
; // it would be nice to test ( lck != 0 )
7641
7642#define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \
7643 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE0x10)) == (KMP_IDENT_ATOMIC_REDUCE0x10))
7644#define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func))
7645
7646 retval = critical_reduce_block;
7647
7648 // another choice of getting a team size (with 1 dynamic deference) is slower
7649 team_size = __kmp_get_team_num_threads(global_tid)(__kmp_threads[(global_tid)]->th.th_team->t.t_nproc);
7650 if (team_size == 1) {
7651
7652 retval = empty_reduce_block;
7653
7654 } else {
7655
7656 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
7657 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
7658
7659#if KMP_ARCH_X86_641 || KMP_ARCH_PPC64(0 || 0) || KMP_ARCH_AARCH640 || KMP_ARCH_MIPS640
7660
7661#if KMP_OS_LINUX1 || KMP_OS_FREEBSD0 || KMP_OS_NETBSD0 || KMP_OS_WINDOWS0 || \
7662 KMP_OS_DARWIN0
7663
7664 int teamsize_cutoff = 4;
7665
7666#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))
7667 if (__kmp_mic_type != non_mic) {
7668 teamsize_cutoff = 8;
7669 }
7670#endif
7671 if (tree_available) {
7672 if (team_size <= teamsize_cutoff) {
7673 if (atomic_available) {
7674 retval = atomic_reduce_block;
7675 }
7676 } else {
7677 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER(((tree_reduce_block) | (bs_reduction_barrier)));
7678 }
7679 } else if (atomic_available) {
7680 retval = atomic_reduce_block;
7681 }
7682#else
7683#error "Unknown or unsupported OS"
7684#endif // KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS ||
7685// KMP_OS_DARWIN
7686
7687#elif KMP_ARCH_X860 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS0
7688
7689#if KMP_OS_LINUX1 || KMP_OS_WINDOWS0
7690
7691 // basic tuning
7692
7693 if (atomic_available) {
7694 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ???
7695 retval = atomic_reduce_block;
7696 }
7697 } // otherwise: use critical section
7698
7699#elif KMP_OS_DARWIN0
7700
7701 if (atomic_available && (num_vars <= 3)) {
7702 retval = atomic_reduce_block;
7703 } else if (tree_available) {
7704 if ((reduce_size > (9 * sizeof(kmp_real64))) &&
7705 (reduce_size < (2000 * sizeof(kmp_real64)))) {
7706 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER(((tree_reduce_block) | (bs_plain_barrier)));
7707 }
7708 } // otherwise: use critical section
7709
7710#else
7711#error "Unknown or unsupported OS"
7712#endif
7713
7714#else
7715#error "Unknown or unsupported architecture"
7716#endif
7717 }
7718
7719 // KMP_FORCE_REDUCTION
7720
7721 // If the team is serialized (team_size == 1), ignore the forced reduction
7722 // method and stay with the unsynchronized method (empty_reduce_block)
7723 if (__kmp_force_reduction_method != reduction_method_not_defined &&
7724 team_size != 1) {
7725
7726 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
7727
7728 int atomic_available, tree_available;
7729
7730 switch ((forced_retval = __kmp_force_reduction_method)) {
7731 case critical_reduce_block:
7732 KMP_ASSERT(lck)((lck) ? 0 : __kmp_debug_assert("lck", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7732))
; // lck should be != 0
7733 break;
7734
7735 case atomic_reduce_block:
7736 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
7737 if (!atomic_available) {
7738 KMP_WARNING(RedMethodNotSupported, "atomic")__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_RedMethodNotSupported
, "atomic"), __kmp_msg_null)
;
7739 forced_retval = critical_reduce_block;
7740 }
7741 break;
7742
7743 case tree_reduce_block:
7744 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
7745 if (!tree_available) {
7746 KMP_WARNING(RedMethodNotSupported, "tree")__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_RedMethodNotSupported
, "tree"), __kmp_msg_null)
;
7747 forced_retval = critical_reduce_block;
7748 } else {
7749#if KMP_FAST_REDUCTION_BARRIER1
7750 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER(((tree_reduce_block) | (bs_reduction_barrier)));
7751#endif
7752 }
7753 break;
7754
7755 default:
7756 KMP_ASSERT(0)((0) ? 0 : __kmp_debug_assert("0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_runtime.cpp"
, 7756))
; // "unsupported method specified"
7757 }
7758
7759 retval = forced_retval;
7760 }
7761
7762 KA_TRACE(10, ("reduction method selected=%08x\n", retval))if (kmp_a_debug >= 10) { __kmp_debug_printf ("reduction method selected=%08x\n"
, retval); }
;
7763
7764#undef FAST_REDUCTION_TREE_METHOD_GENERATED
7765#undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
7766
7767 return (retval);
7768}
7769
7770// this function is for testing set/get/determine reduce method
7771kmp_int32 __kmp_get_reduce_method(void) {
7772 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
7773}