Bug Summary

File:projects/openmp/runtime/src/kmp_tasking.cpp
Warning:line 1432, column 65
The right operand of '-' is a garbage value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name kmp_tasking.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D omp_EXPORTS -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/projects/openmp/runtime/src -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn338205/include -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/i18n -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/include/50 -I /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/thirdparty/ittnotify -U NDEBUG -D _GNU_SOURCE -D _REENTRANT -D _FORTIFY_SOURCE=2 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-comment -Wno-sign-compare -Wno-unused-function -Wno-unused-value -Wno-unused-variable -Wno-switch -Wno-unknown-pragmas -Wno-missing-field-initializers -Wno-missing-braces -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/projects/openmp/runtime/src -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fno-rtti -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-07-29-043837-17923-1 -x c++ /build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp -faddrsig
1/*
2 * kmp_tasking.cpp -- OpenMP 3.0 tasking support.
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// The LLVM Compiler Infrastructure
8//
9// This file is dual licensed under the MIT and the University of Illinois Open
10// Source Licenses. See LICENSE.txt for details.
11//
12//===----------------------------------------------------------------------===//
13
14#include "kmp.h"
15#include "kmp_i18n.h"
16#include "kmp_itt.h"
17#include "kmp_stats.h"
18#include "kmp_wait_release.h"
19
20#if OMPT_SUPPORT1
21#include "ompt-specific.h"
22#endif
23
24#include "tsan_annotations.h"
25
26/* forward declaration */
27static void __kmp_enable_tasking(kmp_task_team_t *task_team,
28 kmp_info_t *this_thr);
29static void __kmp_alloc_task_deque(kmp_info_t *thread,
30 kmp_thread_data_t *thread_data);
31static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
32 kmp_task_team_t *task_team);
33
34#ifdef OMP_45_ENABLED(50 >= 45)
35static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
36#endif
37
38#ifdef BUILD_TIED_TASK_STACK
39
40// __kmp_trace_task_stack: print the tied tasks from the task stack in order
41// from top do bottom
42//
43// gtid: global thread identifier for thread containing stack
44// thread_data: thread data for task team thread containing stack
45// threshold: value above which the trace statement triggers
46// location: string identifying call site of this function (for trace)
47static void __kmp_trace_task_stack(kmp_int32 gtid,
48 kmp_thread_data_t *thread_data,
49 int threshold, char *location) {
50 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
51 kmp_taskdata_t **stack_top = task_stack->ts_top;
52 kmp_int32 entries = task_stack->ts_entries;
53 kmp_taskdata_t *tied_task;
54
55 KA_TRACE(if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
"first_block = %p, stack_top = %p \n", location, gtid, entries
, task_stack->ts_first_block, stack_top); }
56 threshold,if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
"first_block = %p, stack_top = %p \n", location, gtid, entries
, task_stack->ts_first_block, stack_top); }
57 ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
"first_block = %p, stack_top = %p \n", location, gtid, entries
, task_stack->ts_first_block, stack_top); }
58 "first_block = %p, stack_top = %p \n",if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
"first_block = %p, stack_top = %p \n", location, gtid, entries
, task_stack->ts_first_block, stack_top); }
59 location, gtid, entries, task_stack->ts_first_block, stack_top))if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
"first_block = %p, stack_top = %p \n", location, gtid, entries
, task_stack->ts_first_block, stack_top); }
;
60
61 KMP_DEBUG_ASSERT(stack_top != NULL)((stack_top != __null) ? 0 : __kmp_debug_assert("stack_top != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 61))
;
62 KMP_DEBUG_ASSERT(entries > 0)((entries > 0) ? 0 : __kmp_debug_assert("entries > 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 62))
;
63
64 while (entries != 0) {
65 KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0])((stack_top != &task_stack->ts_first_block.sb_block[0]
) ? 0 : __kmp_debug_assert("stack_top != &task_stack->ts_first_block.sb_block[0]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 65))
;
66 // fix up ts_top if we need to pop from previous block
67 if (entries & TASK_STACK_INDEX_MASK == 0) {
68 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top);
69
70 stack_block = stack_block->sb_prev;
71 stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
72 }
73
74 // finish bookkeeping
75 stack_top--;
76 entries--;
77
78 tied_task = *stack_top;
79
80 KMP_DEBUG_ASSERT(tied_task != NULL)((tied_task != __null) ? 0 : __kmp_debug_assert("tied_task != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 80))
;
81 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED)((tied_task->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("tied_task->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 81))
;
82
83 KA_TRACE(threshold,if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
"stack_top=%p, tied_task=%p\n", location, gtid, entries, stack_top
, tied_task); }
84 ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
"stack_top=%p, tied_task=%p\n", location, gtid, entries, stack_top
, tied_task); }
85 "stack_top=%p, tied_task=%p\n",if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
"stack_top=%p, tied_task=%p\n", location, gtid, entries, stack_top
, tied_task); }
86 location, gtid, entries, stack_top, tied_task))if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
"stack_top=%p, tied_task=%p\n", location, gtid, entries, stack_top
, tied_task); }
;
87 }
88 KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0])((stack_top == &task_stack->ts_first_block.sb_block[0]
) ? 0 : __kmp_debug_assert("stack_top == &task_stack->ts_first_block.sb_block[0]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 88))
;
89
90 KA_TRACE(threshold,if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n"
, location, gtid); }
91 ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n",if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n"
, location, gtid); }
92 location, gtid))if (kmp_a_debug >= threshold) { __kmp_debug_printf ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n"
, location, gtid); }
;
93}
94
95// __kmp_init_task_stack: initialize the task stack for the first time
96// after a thread_data structure is created.
97// It should not be necessary to do this again (assuming the stack works).
98//
99// gtid: global thread identifier of calling thread
100// thread_data: thread data for task team thread containing stack
101static void __kmp_init_task_stack(kmp_int32 gtid,
102 kmp_thread_data_t *thread_data) {
103 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
104 kmp_stack_block_t *first_block;
105
106 // set up the first block of the stack
107 first_block = &task_stack->ts_first_block;
108 task_stack->ts_top = (kmp_taskdata_t **)first_block;
109 memset((void *)first_block, '\0',
110 TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *));
111
112 // initialize the stack to be empty
113 task_stack->ts_entries = TASK_STACK_EMPTY;
114 first_block->sb_next = NULL__null;
115 first_block->sb_prev = NULL__null;
116}
117
118// __kmp_free_task_stack: free the task stack when thread_data is destroyed.
119//
120// gtid: global thread identifier for calling thread
121// thread_data: thread info for thread containing stack
122static void __kmp_free_task_stack(kmp_int32 gtid,
123 kmp_thread_data_t *thread_data) {
124 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
125 kmp_stack_block_t *stack_block = &task_stack->ts_first_block;
126
127 KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY)((task_stack->ts_entries == TASK_STACK_EMPTY) ? 0 : __kmp_debug_assert
("task_stack->ts_entries == TASK_STACK_EMPTY", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 127))
;
128 // free from the second block of the stack
129 while (stack_block != NULL__null) {
130 kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL__null;
131
132 stack_block->sb_next = NULL__null;
133 stack_block->sb_prev = NULL__null;
134 if (stack_block != &task_stack->ts_first_block) {
135 __kmp_thread_free(thread,___kmp_thread_free((thread), (stack_block), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 136)
136 stack_block)___kmp_thread_free((thread), (stack_block), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 136)
; // free the block, if not the first
137 }
138 stack_block = next_block;
139 }
140 // initialize the stack to be empty
141 task_stack->ts_entries = 0;
142 task_stack->ts_top = NULL__null;
143}
144
145// __kmp_push_task_stack: Push the tied task onto the task stack.
146// Grow the stack if necessary by allocating another block.
147//
148// gtid: global thread identifier for calling thread
149// thread: thread info for thread containing stack
150// tied_task: the task to push on the stack
151static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread,
152 kmp_taskdata_t *tied_task) {
153 // GEH - need to consider what to do if tt_threads_data not allocated yet
154 kmp_thread_data_t *thread_data =
155 &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 155)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
];
156 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
157
158 if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) {
159 return; // Don't push anything on stack if team or team tasks are serialized
160 }
161
162 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED)((tied_task->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("tied_task->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 162))
;
163 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL)((task_stack->ts_top != __null) ? 0 : __kmp_debug_assert("task_stack->ts_top != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 163))
;
164
165 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n"
, gtid, thread, tied_task); }
166 ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n"
, gtid, thread, tied_task); }
167 gtid, thread, tied_task))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n"
, gtid, thread, tied_task); }
;
168 // Store entry
169 *(task_stack->ts_top) = tied_task;
170
171 // Do bookkeeping for next push
172 task_stack->ts_top++;
173 task_stack->ts_entries++;
174
175 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
176 // Find beginning of this task block
177 kmp_stack_block_t *stack_block =
178 (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE);
179
180 // Check if we already have a block
181 if (stack_block->sb_next !=
182 NULL__null) { // reset ts_top to beginning of next block
183 task_stack->ts_top = &stack_block->sb_next->sb_block[0];
184 } else { // Alloc new block and link it up
185 kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc(
186 thread, sizeof(kmp_stack_block_t));
187
188 task_stack->ts_top = &new_block->sb_block[0];
189 stack_block->sb_next = new_block;
190 new_block->sb_prev = stack_block;
191 new_block->sb_next = NULL__null;
192
193 KA_TRACE(if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n"
, gtid, tied_task, new_block); }
194 30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n"
, gtid, tied_task, new_block); }
195 ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n"
, gtid, tied_task, new_block); }
196 gtid, tied_task, new_block))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n"
, gtid, tied_task, new_block); }
;
197 }
198 }
199 KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n"
, gtid, tied_task); }
200 tied_task))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n"
, gtid, tied_task); }
;
201}
202
203// __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return
204// the task, just check to make sure it matches the ending task passed in.
205//
206// gtid: global thread identifier for the calling thread
207// thread: thread info structure containing stack
208// tied_task: the task popped off the stack
209// ending_task: the task that is ending (should match popped task)
210static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread,
211 kmp_taskdata_t *ending_task) {
212 // GEH - need to consider what to do if tt_threads_data not allocated yet
213 kmp_thread_data_t *thread_data =
214 &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 214)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
];
215 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
216 kmp_taskdata_t *tied_task;
217
218 if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) {
219 // Don't pop anything from stack if team or team tasks are serialized
220 return;
221 }
222
223 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL)((task_stack->ts_top != __null) ? 0 : __kmp_debug_assert("task_stack->ts_top != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 223))
;
224 KMP_DEBUG_ASSERT(task_stack->ts_entries > 0)((task_stack->ts_entries > 0) ? 0 : __kmp_debug_assert(
"task_stack->ts_entries > 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 224))
;
225
226 KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n"
, gtid, thread); }
227 thread))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n"
, gtid, thread); }
;
228
229 // fix up ts_top if we need to pop from previous block
230 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
231 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top);
232
233 stack_block = stack_block->sb_prev;
234 task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
235 }
236
237 // finish bookkeeping
238 task_stack->ts_top--;
239 task_stack->ts_entries--;
240
241 tied_task = *(task_stack->ts_top);
242
243 KMP_DEBUG_ASSERT(tied_task != NULL)((tied_task != __null) ? 0 : __kmp_debug_assert("tied_task != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 243))
;
244 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED)((tied_task->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("tied_task->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 244))
;
245 KMP_DEBUG_ASSERT(tied_task == ending_task)((tied_task == ending_task) ? 0 : __kmp_debug_assert("tied_task == ending_task"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 245))
; // If we built the stack correctly
246
247 KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n"
, gtid, tied_task); }
248 tied_task))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n"
, gtid, tied_task); }
;
249 return;
250}
251#endif /* BUILD_TIED_TASK_STACK */
252
253// __kmp_push_task: Add a task to the thread's deque
254static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
255 kmp_info_t *thread = __kmp_threads[gtid];
256 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
257 kmp_task_team_t *task_team = thread->th.th_task_team;
258 kmp_int32 tid = __kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 258)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
;
259 kmp_thread_data_t *thread_data;
260
261 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d trying to push task %p.\n"
, gtid, taskdata); }
262 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d trying to push task %p.\n"
, gtid, taskdata); }
;
263
264 if (taskdata->td_flags.tiedness == TASK_UNTIED0) {
265 // untied task needs to increment counter so that the task structure is not
266 // freed prematurely
267 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count)(&taskdata->td_untied_count)->fetch_add(1, std::memory_order_acq_rel
)
;
268 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n"
, gtid, counter, taskdata); }
269 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n"
, gtid, counter, taskdata); }
270 ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n"
, gtid, counter, taskdata); }
271 gtid, counter, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n"
, gtid, counter, taskdata); }
;
272 }
273
274 // The first check avoids building task_team thread data if serialized
275 if (taskdata->td_flags.task_serial) {
276 KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d team serialized; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
277 "TASK_NOT_PUSHED for task %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d team serialized; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
278 gtid, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d team serialized; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
;
279 return TASK_NOT_PUSHED1;
280 }
281
282 // Now that serialized tasks have returned, we can assume that we are not in
283 // immediate exec mode
284 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec)((__kmp_tasking_mode != tskm_immediate_exec) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode != tskm_immediate_exec", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 284))
;
285 if (!KMP_TASKING_ENABLED(task_team)(((task_team)->tt.tt_found_tasks) == (!0))) {
286 __kmp_enable_tasking(task_team, thread);
287 }
288 KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE)(((task_team->tt.tt_found_tasks) == (!0)) ? 0 : __kmp_debug_assert
("(task_team->tt.tt_found_tasks) == (!0)", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 288))
;
289 KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL)((((void *)(task_team->tt.tt_threads_data)) != __null) ? 0
: __kmp_debug_assert("((void *)(task_team->tt.tt_threads_data)) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 289))
;
290
291 // Find tasking deque specific to encountering thread
292 thread_data = &task_team->tt.tt_threads_data[tid];
293
294 // No lock needed since only owner can allocate
295 if (thread_data->td.td_deque == NULL__null) {
296 __kmp_alloc_task_deque(thread, thread_data);
297 }
298
299 // Check if deque is full
300 if (TCR_4(thread_data->td.td_deque_ntasks)(thread_data->td.td_deque_ntasks) >=
301 TASK_DEQUE_SIZE(thread_data->td)((thread_data->td).td_deque_size)) {
302 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d deque is full; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
303 "TASK_NOT_PUSHED for task %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d deque is full; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
304 gtid, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d deque is full; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
;
305 return TASK_NOT_PUSHED1;
306 }
307
308 // Lock the deque for the task push operation
309 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
310
311#if OMP_45_ENABLED(50 >= 45)
312 // Need to recheck as we can get a proxy task from a thread outside of OpenMP
313 if (TCR_4(thread_data->td.td_deque_ntasks)(thread_data->td.td_deque_ntasks) >=
314 TASK_DEQUE_SIZE(thread_data->td)((thread_data->td).td_deque_size)) {
315 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
316 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; returning "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d deque is full on 2nd check; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
317 "TASK_NOT_PUSHED for task %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d deque is full on 2nd check; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
318 gtid, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d deque is full on 2nd check; returning "
"TASK_NOT_PUSHED for task %p\n", gtid, taskdata); }
;
319 return TASK_NOT_PUSHED1;
320 }
321#else
322 // Must have room since no thread can add tasks but calling thread
323 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) <(((thread_data->td.td_deque_ntasks) < ((thread_data->
td).td_deque_size)) ? 0 : __kmp_debug_assert("(thread_data->td.td_deque_ntasks) < ((thread_data->td).td_deque_size)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 324))
324 TASK_DEQUE_SIZE(thread_data->td))(((thread_data->td.td_deque_ntasks) < ((thread_data->
td).td_deque_size)) ? 0 : __kmp_debug_assert("(thread_data->td.td_deque_ntasks) < ((thread_data->td).td_deque_size)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 324))
;
325#endif
326
327 thread_data->td.td_deque[thread_data->td.td_deque_tail] =
328 taskdata; // Push taskdata
329 // Wrap index.
330 thread_data->td.td_deque_tail =
331 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td)((thread_data->td).td_deque_size - 1);
332 TCW_4(thread_data->td.td_deque_ntasks,(thread_data->td.td_deque_ntasks) = ((thread_data->td.td_deque_ntasks
) + 1)
333 TCR_4(thread_data->td.td_deque_ntasks) + 1)(thread_data->td.td_deque_ntasks) = ((thread_data->td.td_deque_ntasks
) + 1)
; // Adjust task count
334
335 KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
"task=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data
->td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
336 "task=%p ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
"task=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data
->td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
337 gtid, taskdata, thread_data->td.td_deque_ntasks,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
"task=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data
->td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
338 thread_data->td.td_deque_head, thread_data->td.td_deque_tail))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
"task=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data
->td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
;
339
340 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
341
342 return TASK_SUCCESSFULLY_PUSHED0;
343}
344
345// __kmp_pop_current_task_from_thread: set up current task from called thread
346// when team ends
347//
348// this_thr: thread structure to set current_task in.
349void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) {
350 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(enter): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
351 "this_thread=%p, curtask=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(enter): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
352 "curtask_parent=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(enter): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
353 0, this_thr, this_thr->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(enter): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
354 this_thr->th.th_current_task->td_parent))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(enter): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
;
355
356 this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent;
357
358 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(exit): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
359 "this_thread=%p, curtask=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(exit): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
360 "curtask_parent=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(exit): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
361 0, this_thr, this_thr->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(exit): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
362 this_thr->th.th_current_task->td_parent))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_pop_current_task_from_thread(exit): T#%d "
"this_thread=%p, curtask=%p, " "curtask_parent=%p\n", 0, this_thr
, this_thr->th.th_current_task, this_thr->th.th_current_task
->td_parent); }
;
363}
364
365// __kmp_push_current_task_to_thread: set up current task in called thread for a
366// new team
367//
368// this_thr: thread structure to set up
369// team: team for implicit task data
370// tid: thread within team to set up
371void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team,
372 int tid) {
373 // current task of the thread is a parent of the new just created implicit
374 // tasks of new team
375 KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
376 "curtask=%p "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
377 "parent_task=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
378 tid, this_thr, this_thr->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
379 team->t.t_implicit_task_taskdata[tid].td_parent))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
;
380
381 KMP_DEBUG_ASSERT(this_thr != NULL)((this_thr != __null) ? 0 : __kmp_debug_assert("this_thr != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 381))
;
382
383 if (tid == 0) {
384 if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) {
385 team->t.t_implicit_task_taskdata[0].td_parent =
386 this_thr->th.th_current_task;
387 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0];
388 }
389 } else {
390 team->t.t_implicit_task_taskdata[tid].td_parent =
391 team->t.t_implicit_task_taskdata[0].td_parent;
392 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid];
393 }
394
395 KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
396 "curtask=%p "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
397 "parent_task=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
398 tid, this_thr, this_thr->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
399 team->t.t_implicit_task_taskdata[tid].td_parent))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
"curtask=%p " "parent_task=%p\n", tid, this_thr, this_thr->
th.th_current_task, team->t.t_implicit_task_taskdata[tid].
td_parent); }
;
400}
401
402// __kmp_task_start: bookkeeping for a task starting execution
403//
404// GTID: global thread id of calling thread
405// task: task starting execution
406// current_task: task suspending
407static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task,
408 kmp_taskdata_t *current_task) {
409 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
410 kmp_info_t *thread = __kmp_threads[gtid];
411
412 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n"
, gtid, taskdata, current_task); }
413 ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n"
, gtid, taskdata, current_task); }
414 gtid, taskdata, current_task))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n"
, gtid, taskdata, current_task); }
;
415
416 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT)((taskdata->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 416))
;
417
418 // mark currently executing task as suspended
419 // TODO: GEH - make sure root team implicit task is initialized properly.
420 // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 );
421 current_task->td_flags.executing = 0;
422
423// Add task to stack if tied
424#ifdef BUILD_TIED_TASK_STACK
425 if (taskdata->td_flags.tiedness == TASK_TIED1) {
426 __kmp_push_task_stack(gtid, thread, taskdata);
427 }
428#endif /* BUILD_TIED_TASK_STACK */
429
430 // mark starting task as executing and as current task
431 thread->th.th_current_task = taskdata;
432
433 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 ||((taskdata->td_flags.started == 0 || taskdata->td_flags
.tiedness == 0) ? 0 : __kmp_debug_assert("taskdata->td_flags.started == 0 || taskdata->td_flags.tiedness == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 434))
434 taskdata->td_flags.tiedness == TASK_UNTIED)((taskdata->td_flags.started == 0 || taskdata->td_flags
.tiedness == 0) ? 0 : __kmp_debug_assert("taskdata->td_flags.started == 0 || taskdata->td_flags.tiedness == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 434))
;
435 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 ||((taskdata->td_flags.executing == 0 || taskdata->td_flags
.tiedness == 0) ? 0 : __kmp_debug_assert("taskdata->td_flags.executing == 0 || taskdata->td_flags.tiedness == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 436))
436 taskdata->td_flags.tiedness == TASK_UNTIED)((taskdata->td_flags.executing == 0 || taskdata->td_flags
.tiedness == 0) ? 0 : __kmp_debug_assert("taskdata->td_flags.executing == 0 || taskdata->td_flags.tiedness == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 436))
;
437 taskdata->td_flags.started = 1;
438 taskdata->td_flags.executing = 1;
439 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0)((taskdata->td_flags.complete == 0) ? 0 : __kmp_debug_assert
("taskdata->td_flags.complete == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 439))
;
440 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0)((taskdata->td_flags.freed == 0) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.freed == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 440))
;
441
442 // GEH TODO: shouldn't we pass some sort of location identifier here?
443 // APT: yes, we will pass location here.
444 // need to store current thread state (in a thread or taskdata structure)
445 // before setting work_state, otherwise wrong state is set after end of task
446
447 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_start(exit): T#%d task=%p\n"
, gtid, taskdata); }
;
448
449 return;
450}
451
452#if OMPT_SUPPORT1
453//------------------------------------------------------------------------------
454// __ompt_task_init:
455// Initialize OMPT fields maintained by a task. This will only be called after
456// ompt_start_tool, so we already know whether ompt is enabled or not.
457
458static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) {
459 // The calls to __ompt_task_init already have the ompt_enabled condition.
460 task->ompt_task_info.task_data.value = 0;
461 task->ompt_task_info.frame.exit_frame = NULL__null;
462 task->ompt_task_info.frame.enter_frame = NULL__null;
463#if OMP_40_ENABLED(50 >= 40)
464 task->ompt_task_info.ndeps = 0;
465 task->ompt_task_info.deps = NULL__null;
466#endif /* OMP_40_ENABLED */
467}
468
469// __ompt_task_start:
470// Build and trigger task-begin event
471static inline void __ompt_task_start(kmp_task_t *task,
472 kmp_taskdata_t *current_task,
473 kmp_int32 gtid) {
474 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
475 ompt_task_status_t status = ompt_task_others;
476 if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) {
477 status = ompt_task_yield;
478 __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0;
479 }
480 /* let OMPT know that we're about to run this task */
481 if (ompt_enabled.ompt_callback_task_schedule) {
482 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)ompt_callback_task_schedule_callback(
483 &(current_task->ompt_task_info.task_data), status,
484 &(taskdata->ompt_task_info.task_data));
485 }
486 taskdata->ompt_task_info.scheduling_parent = current_task;
487}
488
489// __ompt_task_finish:
490// Build and trigger final task-schedule event
491static inline void
492__ompt_task_finish(kmp_task_t *task, kmp_taskdata_t *resumed_task,
493 ompt_task_status_t status = ompt_task_complete) {
494 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
495 if (__kmp_omp_cancellation && taskdata->td_taskgroup &&
496 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) {
497 status = ompt_task_cancel;
498 }
499
500 /* let OMPT know that we're returning to the callee task */
501 if (ompt_enabled.ompt_callback_task_schedule) {
502 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)ompt_callback_task_schedule_callback(
503 &(taskdata->ompt_task_info.task_data), status,
504 &((resumed_task ? resumed_task
505 : (taskdata->ompt_task_info.scheduling_parent
506 ? taskdata->ompt_task_info.scheduling_parent
507 : taskdata->td_parent))
508 ->ompt_task_info.task_data));
509 }
510}
511#endif
512
513template <bool ompt>
514static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid,
515 kmp_task_t *task,
516 void *frame_address,
517 void *return_address) {
518 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
519 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
520
521 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
"current_task=%p\n", gtid, loc_ref, taskdata, current_task);
}
522 "current_task=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
"current_task=%p\n", gtid, loc_ref, taskdata, current_task);
}
523 gtid, loc_ref, taskdata, current_task))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
"current_task=%p\n", gtid, loc_ref, taskdata, current_task);
}
;
524
525 if (taskdata->td_flags.tiedness == TASK_UNTIED0) {
526 // untied task needs to increment counter so that the task structure is not
527 // freed prematurely
528 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count)(&taskdata->td_untied_count)->fetch_add(1, std::memory_order_acq_rel
)
;
529 KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "
"incremented for task %p\n", gtid, counter, taskdata); }
530 "incremented for task %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "
"incremented for task %p\n", gtid, counter, taskdata); }
531 gtid, counter, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "
"incremented for task %p\n", gtid, counter, taskdata); }
;
532 }
533
534 taskdata->td_flags.task_serial =
535 1; // Execute this task immediately, not deferred.
536 __kmp_task_start(gtid, task, current_task);
537
538#if OMPT_SUPPORT1
539 if (ompt) {
540 if (current_task->ompt_task_info.frame.enter_frame == NULL__null) {
541 current_task->ompt_task_info.frame.enter_frame =
542 taskdata->ompt_task_info.frame.exit_frame = frame_address;
543 }
544 if (ompt_enabled.ompt_callback_task_create) {
545 ompt_task_info_t *parent_info = &(current_task->ompt_task_info);
546 ompt_callbacks.ompt_callback(ompt_callback_task_create)ompt_callback_task_create_callback(
547 &(parent_info->task_data), &(parent_info->frame),
548 &(taskdata->ompt_task_info.task_data),
549 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata)((taskdata->td_flags.task_serial || taskdata->td_flags.
tasking_ser) ? ompt_task_undeferred : 0x0) | ((!(taskdata->
td_flags.tiedness)) ? ompt_task_untied : 0x0) | (taskdata->
td_flags.final ? ompt_task_final : 0x0) | (taskdata->td_flags
.merged_if0 ? ompt_task_mergeable : 0x0)
, 0,
550 return_address);
551 }
552 __ompt_task_start(task, current_task, gtid);
553 }
554#endif // OMPT_SUPPORT
555
556 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n"
, gtid, loc_ref, taskdata); }
557 loc_ref, taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n"
, gtid, loc_ref, taskdata); }
;
558}
559
560#if OMPT_SUPPORT1
561OMPT_NOINLINE__attribute__((noinline))
562static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
563 kmp_task_t *task,
564 void *frame_address,
565 void *return_address) {
566 __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address,
567 return_address);
568}
569#endif // OMPT_SUPPORT
570
571// __kmpc_omp_task_begin_if0: report that a given serialized task has started
572// execution
573//
574// loc_ref: source location information; points to beginning of task block.
575// gtid: global thread number.
576// task: task thunk for the started task.
577void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid,
578 kmp_task_t *task) {
579#if OMPT_SUPPORT1
580 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
581 OMPT_STORE_RETURN_ADDRESS(gtid)if (ompt_enabled.enabled && gtid >= 0 && __kmp_threads
[gtid] && !__kmp_threads[gtid]->th.ompt_thread_info
.return_address) __kmp_threads[gtid]->th.ompt_thread_info.
return_address = __builtin_return_address(0)
;
582 __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task,
583 OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1),
584 OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid));
585 return;
586 }
587#endif
588 __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL__null, NULL__null);
589}
590
591#ifdef TASK_UNUSED
592// __kmpc_omp_task_begin: report that a given task has started execution
593// NEVER GENERATED BY COMPILER, DEPRECATED!!!
594void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) {
595 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
596
597 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1), current_task
); }
598 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1), current_task
); }
599 ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1), current_task
); }
600 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1), current_task
); }
;
601
602 __kmp_task_start(gtid, task, current_task);
603
604 KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
605 loc_ref, KMP_TASK_TO_TASKDATA(task)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
;
606 return;
607}
608#endif // TASK_UNUSED
609
610// __kmp_free_task: free the current task space and the space for shareds
611//
612// gtid: Global thread ID of calling thread
613// taskdata: task to free
614// thread: thread data structure of caller
615static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
616 kmp_info_t *thread) {
617 KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_free_task: T#%d freeing data from task %p\n"
, gtid, taskdata); }
618 taskdata))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_free_task: T#%d freeing data from task %p\n"
, gtid, taskdata); }
;
619
620 // Check to make sure all flags and counters have the correct values
621 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT)((taskdata->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 621))
;
622 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0)((taskdata->td_flags.executing == 0) ? 0 : __kmp_debug_assert
("taskdata->td_flags.executing == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 622))
;
623 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1)((taskdata->td_flags.complete == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.complete == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 623))
;
624 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0)((taskdata->td_flags.freed == 0) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.freed == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 624))
;
625 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 ||((taskdata->td_allocated_child_tasks == 0 || taskdata->
td_flags.task_serial == 1) ? 0 : __kmp_debug_assert("taskdata->td_allocated_child_tasks == 0 || taskdata->td_flags.task_serial == 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 626))
626 taskdata->td_flags.task_serial == 1)((taskdata->td_allocated_child_tasks == 0 || taskdata->
td_flags.task_serial == 1) ? 0 : __kmp_debug_assert("taskdata->td_allocated_child_tasks == 0 || taskdata->td_flags.task_serial == 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 626))
;
627 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0)((taskdata->td_incomplete_child_tasks == 0) ? 0 : __kmp_debug_assert
("taskdata->td_incomplete_child_tasks == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 627))
;
628
629 taskdata->td_flags.freed = 1;
630 ANNOTATE_HAPPENS_BEFORE(taskdata);
631// deallocate the taskdata and shared variable blocks associated with this task
632#if USE_FAST_MEMORY3
633 __kmp_fast_free(thread, taskdata)___kmp_fast_free((thread), (taskdata), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 633)
;
634#else /* ! USE_FAST_MEMORY */
635 __kmp_thread_free(thread, taskdata)___kmp_thread_free((thread), (taskdata), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 635)
;
636#endif
637
638 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task: T#%d freed task %p\n"
, gtid, taskdata); }
;
639}
640
641// __kmp_free_task_and_ancestors: free the current task and ancestors without
642// children
643//
644// gtid: Global thread ID of calling thread
645// taskdata: task to free
646// thread: thread data structure of caller
647static void __kmp_free_task_and_ancestors(kmp_int32 gtid,
648 kmp_taskdata_t *taskdata,
649 kmp_info_t *thread) {
650#if OMP_45_ENABLED(50 >= 45)
651 // Proxy tasks must always be allowed to free their parents
652 // because they can be run in background even in serial mode.
653 kmp_int32 team_serial =
654 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) &&
655 !taskdata->td_flags.proxy;
656#else
657 kmp_int32 team_serial =
658 taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser;
659#endif
660 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT)((taskdata->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 660))
;
661
662 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks)(&taskdata->td_allocated_child_tasks)->fetch_sub(1,
std::memory_order_acq_rel)
- 1;
663 KMP_DEBUG_ASSERT(children >= 0)((children >= 0) ? 0 : __kmp_debug_assert("children >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 663))
;
664
665 // Now, go up the ancestor tree to see if any ancestors can now be freed.
666 while (children == 0) {
667 kmp_taskdata_t *parent_taskdata = taskdata->td_parent;
668
669 KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
"and freeing itself\n", gtid, taskdata); }
670 "and freeing itself\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
"and freeing itself\n", gtid, taskdata); }
671 gtid, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
"and freeing itself\n", gtid, taskdata); }
;
672
673 // --- Deallocate my ancestor task ---
674 __kmp_free_task(gtid, taskdata, thread);
675
676 taskdata = parent_taskdata;
677
678 // Stop checking ancestors at implicit task instead of walking up ancestor
679 // tree to avoid premature deallocation of ancestors.
680 if (team_serial || taskdata->td_flags.tasktype == TASK_IMPLICIT0)
681 return;
682
683 // Predecrement simulated by "- 1" calculation
684 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks)(&taskdata->td_allocated_child_tasks)->fetch_sub(1,
std::memory_order_acq_rel)
- 1;
685 KMP_DEBUG_ASSERT(children >= 0)((children >= 0) ? 0 : __kmp_debug_assert("children >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 685))
;
686 }
687
688 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
"not freeing it yet\n", gtid, taskdata, children); }
689 20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
"not freeing it yet\n", gtid, taskdata, children); }
690 "not freeing it yet\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
"not freeing it yet\n", gtid, taskdata, children); }
691 gtid, taskdata, children))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
"not freeing it yet\n", gtid, taskdata, children); }
;
692}
693
694// __kmp_task_finish: bookkeeping to do when a task finishes execution
695//
696// gtid: global thread ID for calling thread
697// task: task to be finished
698// resumed_task: task to be resumed. (may be NULL if task is serialized)
699template <bool ompt>
700static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
701 kmp_taskdata_t *resumed_task) {
702 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
703 kmp_info_t *thread = __kmp_threads[gtid];
704 kmp_task_team_t *task_team =
705 thread->th.th_task_team; // might be NULL for serial teams...
706 kmp_int32 children = 0;
707
708 KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "
"task %p\n", gtid, taskdata, resumed_task); }
709 "task %p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "
"task %p\n", gtid, taskdata, resumed_task); }
710 gtid, taskdata, resumed_task))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "
"task %p\n", gtid, taskdata, resumed_task); }
;
711
712 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT)((taskdata->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 712))
;
713
714// Pop task from stack if tied
715#ifdef BUILD_TIED_TASK_STACK
716 if (taskdata->td_flags.tiedness == TASK_TIED1) {
717 __kmp_pop_task_stack(gtid, thread, taskdata);
718 }
719#endif /* BUILD_TIED_TASK_STACK */
720
721 if (taskdata->td_flags.tiedness == TASK_UNTIED0) {
722 // untied task needs to check the counter so that the task structure is not
723 // freed prematurely
724 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count)(&taskdata->td_untied_count)->fetch_sub(1, std::memory_order_acq_rel
)
- 1;
725 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n"
, gtid, counter, taskdata); }
726 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n"
, gtid, counter, taskdata); }
727 ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n"
, gtid, counter, taskdata); }
728 gtid, counter, taskdata))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n"
, gtid, counter, taskdata); }
;
729 if (counter > 0) {
730 // untied task is not done, to be continued possibly by other thread, do
731 // not free it now
732 if (resumed_task == NULL__null) {
733 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial)((taskdata->td_flags.task_serial) ? 0 : __kmp_debug_assert
("taskdata->td_flags.task_serial", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 733))
;
734 resumed_task = taskdata->td_parent; // In a serialized task, the resumed
735 // task is the parent
736 }
737 thread->th.th_current_task = resumed_task; // restore current_task
738 resumed_task->td_flags.executing = 1; // resume previous task
739 KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(exit): T#%d partially done task %p, "
"resuming task %p\n", gtid, taskdata, resumed_task); }
740 "resuming task %p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(exit): T#%d partially done task %p, "
"resuming task %p\n", gtid, taskdata, resumed_task); }
741 gtid, taskdata, resumed_task))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(exit): T#%d partially done task %p, "
"resuming task %p\n", gtid, taskdata, resumed_task); }
;
742 return;
743 }
744 }
745#if OMPT_SUPPORT1
746 if (ompt)
747 __ompt_task_finish(task, resumed_task);
748#endif
749
750 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0)((taskdata->td_flags.complete == 0) ? 0 : __kmp_debug_assert
("taskdata->td_flags.complete == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 750))
;
751 taskdata->td_flags.complete = 1; // mark the task as completed
752 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1)((taskdata->td_flags.started == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.started == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 752))
;
753 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0)((taskdata->td_flags.freed == 0) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.freed == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 753))
;
754
755 // Only need to keep track of count if team parallel and tasking not
756 // serialized
757 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
758 // Predecrement simulated by "- 1" calculation
759 children =
760 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks)(&taskdata->td_parent->td_incomplete_child_tasks)->
fetch_sub(1, std::memory_order_acq_rel)
- 1;
761 KMP_DEBUG_ASSERT(children >= 0)((children >= 0) ? 0 : __kmp_debug_assert("children >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 761))
;
762#if OMP_40_ENABLED(50 >= 40)
763 if (taskdata->td_taskgroup)
764 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count)(&taskdata->td_taskgroup->count)->fetch_sub(1, std
::memory_order_acq_rel)
;
765#if OMP_45_ENABLED(50 >= 45)
766 }
767 // if we found proxy tasks there could exist a dependency chain
768 // with the proxy task as origin
769 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) ||
770 (task_team && task_team->tt.tt_found_proxy_tasks)) {
771#endif
772 __kmp_release_deps(gtid, taskdata);
773#endif
774 }
775
776 // td_flags.executing must be marked as 0 after __kmp_release_deps has been
777 // called. Othertwise, if a task is executed immediately from the release_deps
778 // code, the flag will be reset to 1 again by this same function
779 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1)((taskdata->td_flags.executing == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.executing == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 779))
;
780 taskdata->td_flags.executing = 0; // suspend the finishing task
781
782 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n"
, gtid, taskdata, children); }
783 20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n"
, gtid, taskdata, children); }
784 gtid, taskdata, children))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n"
, gtid, taskdata, children); }
;
785
786#if OMP_40_ENABLED(50 >= 40)
787 /* If the tasks' destructor thunk flag has been set, we need to invoke the
788 destructor thunk that has been generated by the compiler. The code is
789 placed here, since at this point other tasks might have been released
790 hence overlapping the destructor invokations with some other work in the
791 released tasks. The OpenMP spec is not specific on when the destructors
792 are invoked, so we should be free to choose. */
793 if (taskdata->td_flags.destructors_thunk) {
794 kmp_routine_entry_t destr_thunk = task->data1.destructors;
795 KMP_ASSERT(destr_thunk)((destr_thunk) ? 0 : __kmp_debug_assert("destr_thunk", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 795))
;
796 destr_thunk(gtid, task);
797 }
798#endif // OMP_40_ENABLED
799
800 // bookkeeping for resuming task:
801 // GEH - note tasking_ser => task_serial
802 KMP_DEBUG_ASSERT((((taskdata->td_flags.tasking_ser || taskdata->td_flags
.task_serial) == taskdata->td_flags.task_serial) ? 0 : __kmp_debug_assert
("(taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == taskdata->td_flags.task_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 804))
803 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==(((taskdata->td_flags.tasking_ser || taskdata->td_flags
.task_serial) == taskdata->td_flags.task_serial) ? 0 : __kmp_debug_assert
("(taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == taskdata->td_flags.task_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 804))
804 taskdata->td_flags.task_serial)(((taskdata->td_flags.tasking_ser || taskdata->td_flags
.task_serial) == taskdata->td_flags.task_serial) ? 0 : __kmp_debug_assert
("(taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == taskdata->td_flags.task_serial"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 804))
;
805 if (taskdata->td_flags.task_serial) {
806 if (resumed_task == NULL__null) {
807 resumed_task = taskdata->td_parent; // In a serialized task, the resumed
808 // task is the parent
809 }
810 } else {
811 KMP_DEBUG_ASSERT(resumed_task !=((resumed_task != __null) ? 0 : __kmp_debug_assert("resumed_task != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 812))
812 NULL)((resumed_task != __null) ? 0 : __kmp_debug_assert("resumed_task != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 812))
; // verify that resumed task is passed as arguemnt
813 }
814
815 // Free this task and then ancestor tasks if they have no children.
816 // Restore th_current_task first as suggested by John:
817 // johnmc: if an asynchronous inquiry peers into the runtime system
818 // it doesn't see the freed task as the current task.
819 thread->th.th_current_task = resumed_task;
820 __kmp_free_task_and_ancestors(gtid, taskdata, thread);
821
822 // TODO: GEH - make sure root team implicit task is initialized properly.
823 // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 );
824 resumed_task->td_flags.executing = 1; // resume previous task
825
826 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n"
, gtid, taskdata, resumed_task); }
827 10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n"
, gtid, taskdata, resumed_task); }
828 gtid, taskdata, resumed_task))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n"
, gtid, taskdata, resumed_task); }
;
829
830 return;
831}
832
833template <bool ompt>
834static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref,
835 kmp_int32 gtid,
836 kmp_task_t *task) {
837 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
838 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
;
839 // this routine will provide task to resume
840 __kmp_task_finish<ompt>(gtid, task, NULL__null);
841
842 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
843 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
;
844
845#if OMPT_SUPPORT1
846 if (ompt) {
847 omp_frame_t *ompt_frame;
848 __ompt_get_task_info_internal(0, NULL__null, NULL__null, &ompt_frame, NULL__null, NULL__null);
849 ompt_frame->enter_frame = NULL__null;
850 }
851#endif
852
853 return;
854}
855
856#if OMPT_SUPPORT1
857OMPT_NOINLINE__attribute__((noinline))
858void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
859 kmp_task_t *task) {
860 __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task);
861}
862#endif // OMPT_SUPPORT
863
864// __kmpc_omp_task_complete_if0: report that a task has completed execution
865//
866// loc_ref: source location information; points to end of task block.
867// gtid: global thread number.
868// task: task thunk for the completed task.
869void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid,
870 kmp_task_t *task) {
871#if OMPT_SUPPORT1
872 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
873 __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task);
874 return;
875 }
876#endif
877 __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task);
878}
879
880#ifdef TASK_UNUSED
881// __kmpc_omp_task_complete: report that a task has completed execution
882// NEVER GENERATED BY COMPILER, DEPRECATED!!!
883void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid,
884 kmp_task_t *task) {
885 KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
886 loc_ref, KMP_TASK_TO_TASKDATA(task)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
;
887
888 __kmp_task_finish<false>(gtid, task,
889 NULL__null); // Not sure how to find task to resume
890
891 KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
892 loc_ref, KMP_TASK_TO_TASKDATA(task)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n"
, gtid, loc_ref, (((kmp_taskdata_t *)task) - 1)); }
;
893 return;
894}
895#endif // TASK_UNUSED
896
897// __kmp_init_implicit_task: Initialize the appropriate fields in the implicit
898// task for a given thread
899//
900// loc_ref: reference to source location of parallel region
901// this_thr: thread data structure corresponding to implicit task
902// team: team for this_thr
903// tid: thread id of given thread within team
904// set_curr_task: TRUE if need to push current task to thread
905// NOTE: Routine does not set up the implicit task ICVS. This is assumed to
906// have already been done elsewhere.
907// TODO: Get better loc_ref. Value passed in may be NULL
908void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
909 kmp_team_t *team, int tid, int set_curr_task) {
910 kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid];
911
912 KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n"
, tid, team, task, set_curr_task ? "TRUE" : "FALSE"); }
913 10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n"
, tid, team, task, set_curr_task ? "TRUE" : "FALSE"); }
914 ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n"
, tid, team, task, set_curr_task ? "TRUE" : "FALSE"); }
915 tid, team, task, set_curr_task ? "TRUE" : "FALSE"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n"
, tid, team, task, set_curr_task ? "TRUE" : "FALSE"); }
;
916
917 task->td_task_id = KMP_GEN_TASK_ID()(~0);
918 task->td_team = team;
919 // task->td_parent = NULL; // fix for CQ230101 (broken parent task info
920 // in debugger)
921 task->td_ident = loc_ref;
922 task->td_taskwait_ident = NULL__null;
923 task->td_taskwait_counter = 0;
924 task->td_taskwait_thread = 0;
925
926 task->td_flags.tiedness = TASK_TIED1;
927 task->td_flags.tasktype = TASK_IMPLICIT0;
928#if OMP_45_ENABLED(50 >= 45)
929 task->td_flags.proxy = TASK_FULL0;
930#endif
931
932 // All implicit tasks are executed immediately, not deferred
933 task->td_flags.task_serial = 1;
934 task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
935 task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
936
937 task->td_flags.started = 1;
938 task->td_flags.executing = 1;
939 task->td_flags.complete = 0;
940 task->td_flags.freed = 0;
941
942#if OMP_40_ENABLED(50 >= 40)
943 task->td_depnode = NULL__null;
944#endif
945 task->td_last_tied = task;
946
947 if (set_curr_task) { // only do this init first time thread is created
948 KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0)(&task->td_incomplete_child_tasks)->store(0, std::memory_order_release
)
;
949 // Not used: don't need to deallocate implicit task
950 KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0)(&task->td_allocated_child_tasks)->store(0, std::memory_order_release
)
;
951#if OMP_40_ENABLED(50 >= 40)
952 task->td_taskgroup = NULL__null; // An implicit task does not have taskgroup
953 task->td_dephash = NULL__null;
954#endif
955 __kmp_push_current_task_to_thread(this_thr, team, tid);
956 } else {
957 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0)((task->td_incomplete_child_tasks == 0) ? 0 : __kmp_debug_assert
("task->td_incomplete_child_tasks == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 957))
;
958 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0)((task->td_allocated_child_tasks == 0) ? 0 : __kmp_debug_assert
("task->td_allocated_child_tasks == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 958))
;
959 }
960
961#if OMPT_SUPPORT1
962 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0))
963 __ompt_task_init(task, tid);
964#endif
965
966 KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n"
, tid, team, task); }
967 team, task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n"
, tid, team, task); }
;
968}
969
970// __kmp_finish_implicit_task: Release resources associated to implicit tasks
971// at the end of parallel regions. Some resources are kept for reuse in the next
972// parallel region.
973//
974// thread: thread data structure corresponding to implicit task
975void __kmp_finish_implicit_task(kmp_info_t *thread) {
976 kmp_taskdata_t *task = thread->th.th_current_task;
977 if (task->td_dephash)
978 __kmp_dephash_free_entries(thread, task->td_dephash);
979}
980
981// __kmp_free_implicit_task: Release resources associated to implicit tasks
982// when these are destroyed regions
983//
984// thread: thread data structure corresponding to implicit task
985void __kmp_free_implicit_task(kmp_info_t *thread) {
986 kmp_taskdata_t *task = thread->th.th_current_task;
987 if (task && task->td_dephash) {
988 __kmp_dephash_free(thread, task->td_dephash);
989 task->td_dephash = NULL__null;
990 }
991}
992
993// Round up a size to a power of two specified by val: Used to insert padding
994// between structures co-allocated using a single malloc() call
995static size_t __kmp_round_up_to_val(size_t size, size_t val) {
996 if (size & (val - 1)) {
997 size &= ~(val - 1);
998 if (size <= KMP_SIZE_T_MAX(0xFFFFFFFFFFFFFFFF) - val) {
999 size += val; // Round up if there is no overflow.
1000 }
1001 }
1002 return size;
1003} // __kmp_round_up_to_va
1004
1005// __kmp_task_alloc: Allocate the taskdata and task data structures for a task
1006//
1007// loc_ref: source location information
1008// gtid: global thread number.
1009// flags: include tiedness & task type (explicit vs. implicit) of the ''new''
1010// task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine.
1011// sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including
1012// private vars accessed in task.
1013// sizeof_shareds: Size in bytes of array of pointers to shared vars accessed
1014// in task.
1015// task_entry: Pointer to task code entry point generated by compiler.
1016// returns: a pointer to the allocated kmp_task_t structure (task).
1017kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1018 kmp_tasking_flags_t *flags,
1019 size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1020 kmp_routine_entry_t task_entry) {
1021 kmp_task_t *task;
1022 kmp_taskdata_t *taskdata;
1023 kmp_info_t *thread = __kmp_threads[gtid];
1024 kmp_team_t *team = thread->th.th_team;
1025 kmp_taskdata_t *parent_task = thread->th.th_current_task;
1026 size_t shareds_offset;
1027
1028 KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, *((kmp_int32 *)flags), sizeof_kmp_task_t, sizeof_shareds, task_entry
); }
1029 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, *((kmp_int32 *)flags), sizeof_kmp_task_t, sizeof_shareds, task_entry
); }
1030 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, *((kmp_int32 *)flags), sizeof_kmp_task_t, sizeof_shareds, task_entry
); }
1031 sizeof_shareds, task_entry))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, *((kmp_int32 *)flags), sizeof_kmp_task_t, sizeof_shareds, task_entry
); }
;
1032
1033 if (parent_task->td_flags.final) {
1034 if (flags->merged_if0) {
1035 }
1036 flags->final = 1;
1037 }
1038 if (flags->tiedness == TASK_UNTIED0 && !team->t.t_serialized) {
1039 // Untied task encountered causes the TSC algorithm to check entire deque of
1040 // the victim thread. If no untied task encountered, then checking the head
1041 // of the deque should be enough.
1042 KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1)if ((thread->th.th_task_team->tt.tt_untied_task_encountered
) != (1)) (thread->th.th_task_team->tt.tt_untied_task_encountered
) = (1)
;
1043 }
1044
1045#if OMP_45_ENABLED(50 >= 45)
1046 if (flags->proxy == TASK_PROXY1) {
1047 flags->tiedness = TASK_UNTIED0;
1048 flags->merged_if0 = 1;
1049
1050 /* are we running in a sequential parallel or tskm_immediate_exec... we need
1051 tasking support enabled */
1052 if ((thread->th.th_task_team) == NULL__null) {
1053 /* This should only happen if the team is serialized
1054 setup a task team and propagate it to the thread */
1055 KMP_DEBUG_ASSERT(team->t.t_serialized)((team->t.t_serialized) ? 0 : __kmp_debug_assert("team->t.t_serialized"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1055))
;
1056 KA_TRACE(30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("T#%d creating task team in __kmp_task_alloc for proxy task\n"
, gtid); }
1057 ("T#%d creating task team in __kmp_task_alloc for proxy task\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("T#%d creating task team in __kmp_task_alloc for proxy task\n"
, gtid); }
1058 gtid))if (kmp_a_debug >= 30) { __kmp_debug_printf ("T#%d creating task team in __kmp_task_alloc for proxy task\n"
, gtid); }
;
1059 __kmp_task_team_setup(
1060 thread, team,
1061 1); // 1 indicates setup the current team regardless of nthreads
1062 thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state];
1063 }
1064 kmp_task_team_t *task_team = thread->th.th_task_team;
1065
1066 /* tasking must be enabled now as the task might not be pushed */
1067 if (!KMP_TASKING_ENABLED(task_team)(((task_team)->tt.tt_found_tasks) == (!0))) {
1068 KA_TRACE(if (kmp_a_debug >= 30) { __kmp_debug_printf ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n"
, gtid); }
1069 30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n"
, gtid); }
1070 ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid))if (kmp_a_debug >= 30) { __kmp_debug_printf ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n"
, gtid); }
;
1071 __kmp_enable_tasking(task_team, thread);
1072 kmp_int32 tid = thread->th.th_info.ds.ds_tid;
1073 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
1074 // No lock needed since only owner can allocate
1075 if (thread_data->td.td_deque == NULL__null) {
1076 __kmp_alloc_task_deque(thread, thread_data);
1077 }
1078 }
1079
1080 if (task_team->tt.tt_found_proxy_tasks == FALSE0)
1081 TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE)(task_team->tt.tt_found_proxy_tasks) = ((!0));
1082 }
1083#endif
1084
1085 // Calculate shared structure offset including padding after kmp_task_t struct
1086 // to align pointers in shared struct
1087 shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
1088 shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *));
1089
1090 // Allocate a kmp_taskdata_t block and a kmp_task_t block.
1091 KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_task_alloc: T#%d First malloc size: %ld\n"
, gtid, shareds_offset); }
1092 shareds_offset))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_task_alloc: T#%d First malloc size: %ld\n"
, gtid, shareds_offset); }
;
1093 KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_task_alloc: T#%d Second malloc size: %ld\n"
, gtid, sizeof_shareds); }
1094 sizeof_shareds))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_task_alloc: T#%d Second malloc size: %ld\n"
, gtid, sizeof_shareds); }
;
1095
1096// Avoid double allocation here by combining shareds with taskdata
1097#if USE_FAST_MEMORY3
1098 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset +___kmp_fast_allocate((thread), (shareds_offset + sizeof_shareds
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1099)
1099 sizeof_shareds)___kmp_fast_allocate((thread), (shareds_offset + sizeof_shareds
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1099)
;
1100#else /* ! USE_FAST_MEMORY */
1101 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset +___kmp_thread_malloc((thread), (shareds_offset + sizeof_shareds
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1102)
1102 sizeof_shareds)___kmp_thread_malloc((thread), (shareds_offset + sizeof_shareds
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1102)
;
1103#endif /* USE_FAST_MEMORY */
1104 ANNOTATE_HAPPENS_AFTER(taskdata);
1105
1106 task = KMP_TASKDATA_TO_TASK(taskdata)(kmp_task_t *)(taskdata + 1);
1107
1108// Make sure task & taskdata are aligned appropriately
1109#if KMP_ARCH_X860 || KMP_ARCH_PPC64(0 || 0) || !KMP_HAVE_QUAD0
1110 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0)(((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0
) ? 0 : __kmp_debug_assert("(((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1110))
;
1111 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0)(((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0) ? 0
: __kmp_debug_assert("(((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1111))
;
1112#else
1113 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0)(((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0)
? 0 : __kmp_debug_assert("(((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1113))
;
1114 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0)(((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0) ? 0
: __kmp_debug_assert("(((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1114))
;
1115#endif
1116 if (sizeof_shareds > 0) {
1117 // Avoid double allocation here by combining shareds with taskdata
1118 task->shareds = &((char *)taskdata)[shareds_offset];
1119 // Make sure shareds struct is aligned to pointer size
1120 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==(((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1
)) == 0) ? 0 : __kmp_debug_assert("(((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1121))
1121 0)(((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1
)) == 0) ? 0 : __kmp_debug_assert("(((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1121))
;
1122 } else {
1123 task->shareds = NULL__null;
1124 }
1125 task->routine = task_entry;
1126 task->part_id = 0; // AC: Always start with 0 part id
1127
1128 taskdata->td_task_id = KMP_GEN_TASK_ID()(~0);
1129 taskdata->td_team = team;
1130 taskdata->td_alloc_thread = thread;
1131 taskdata->td_parent = parent_task;
1132 taskdata->td_level = parent_task->td_level + 1; // increment nesting level
1133 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0)(&taskdata->td_untied_count)->store(0, std::memory_order_relaxed
)
;
1134 taskdata->td_ident = loc_ref;
1135 taskdata->td_taskwait_ident = NULL__null;
1136 taskdata->td_taskwait_counter = 0;
1137 taskdata->td_taskwait_thread = 0;
1138 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL)((taskdata->td_parent != __null) ? 0 : __kmp_debug_assert(
"taskdata->td_parent != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1138))
;
1139#if OMP_45_ENABLED(50 >= 45)
1140 // avoid copying icvs for proxy tasks
1141 if (flags->proxy == TASK_FULL0)
1142#endif
1143 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs);
1144
1145 taskdata->td_flags.tiedness = flags->tiedness;
1146 taskdata->td_flags.final = flags->final;
1147 taskdata->td_flags.merged_if0 = flags->merged_if0;
1148#if OMP_40_ENABLED(50 >= 40)
1149 taskdata->td_flags.destructors_thunk = flags->destructors_thunk;
1150#endif // OMP_40_ENABLED
1151#if OMP_45_ENABLED(50 >= 45)
1152 taskdata->td_flags.proxy = flags->proxy;
1153 taskdata->td_task_team = thread->th.th_task_team;
1154 taskdata->td_size_alloc = shareds_offset + sizeof_shareds;
1155#endif
1156 taskdata->td_flags.tasktype = TASK_EXPLICIT1;
1157
1158 // GEH - TODO: fix this to copy parent task's value of tasking_ser flag
1159 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1160
1161 // GEH - TODO: fix this to copy parent task's value of team_serial flag
1162 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1163
1164 // GEH - Note we serialize the task if the team is serialized to make sure
1165 // implicit parallel region tasks are not left until program termination to
1166 // execute. Also, it helps locality to execute immediately.
1167
1168 taskdata->td_flags.task_serial =
1169 (parent_task->td_flags.final || taskdata->td_flags.team_serial ||
1170 taskdata->td_flags.tasking_ser);
1171
1172 taskdata->td_flags.started = 0;
1173 taskdata->td_flags.executing = 0;
1174 taskdata->td_flags.complete = 0;
1175 taskdata->td_flags.freed = 0;
1176
1177 taskdata->td_flags.native = flags->native;
1178
1179 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0)(&taskdata->td_incomplete_child_tasks)->store(0, std
::memory_order_relaxed)
;
1180 // start at one because counts current task and children
1181 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1)(&taskdata->td_allocated_child_tasks)->store(1, std
::memory_order_relaxed)
;
1182#if OMP_40_ENABLED(50 >= 40)
1183 taskdata->td_taskgroup =
1184 parent_task->td_taskgroup; // task inherits taskgroup from the parent task
1185 taskdata->td_dephash = NULL__null;
1186 taskdata->td_depnode = NULL__null;
1187#endif
1188 if (flags->tiedness == TASK_UNTIED0)
1189 taskdata->td_last_tied = NULL__null; // will be set when the task is scheduled
1190 else
1191 taskdata->td_last_tied = taskdata;
1192
1193#if OMPT_SUPPORT1
1194 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0))
1195 __ompt_task_init(taskdata, gtid);
1196#endif
1197// Only need to keep track of child task counts if team parallel and tasking not
1198// serialized or if it is a proxy task
1199#if OMP_45_ENABLED(50 >= 45)
1200 if (flags->proxy == TASK_PROXY1 ||
1201 !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser))
1202#else
1203 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser))
1204#endif
1205 {
1206 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks)(&parent_task->td_incomplete_child_tasks)->fetch_add
(1, std::memory_order_acq_rel)
;
1207#if OMP_40_ENABLED(50 >= 40)
1208 if (parent_task->td_taskgroup)
1209 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count)(&parent_task->td_taskgroup->count)->fetch_add(1
, std::memory_order_acq_rel)
;
1210#endif
1211 // Only need to keep track of allocated child tasks for explicit tasks since
1212 // implicit not deallocated
1213 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT1) {
1214 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks)(&taskdata->td_parent->td_allocated_child_tasks)->
fetch_add(1, std::memory_order_acq_rel)
;
1215 }
1216 }
1217
1218 KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n"
, gtid, taskdata, taskdata->td_parent); }
1219 gtid, taskdata, taskdata->td_parent))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n"
, gtid, taskdata, taskdata->td_parent); }
;
1220 ANNOTATE_HAPPENS_BEFORE(task);
1221
1222 return task;
1223}
1224
1225kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1226 kmp_int32 flags, size_t sizeof_kmp_task_t,
1227 size_t sizeof_shareds,
1228 kmp_routine_entry_t task_entry) {
1229 kmp_task_t *retval;
1230 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
1231
1232 input_flags->native = FALSE0;
1233// __kmp_task_alloc() sets up all other runtime flags
1234
1235#if OMP_45_ENABLED(50 >= 45)
1236 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", input_flags
->proxy ? "proxy" : "", sizeof_kmp_task_t, sizeof_shareds,
task_entry); }
1237 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", input_flags
->proxy ? "proxy" : "", sizeof_kmp_task_t, sizeof_shareds,
task_entry); }
1238 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", input_flags
->proxy ? "proxy" : "", sizeof_kmp_task_t, sizeof_shareds,
task_entry); }
1239 input_flags->proxy ? "proxy" : "", sizeof_kmp_task_t,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", input_flags
->proxy ? "proxy" : "", sizeof_kmp_task_t, sizeof_shareds,
task_entry); }
1240 sizeof_shareds, task_entry))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", input_flags
->proxy ? "proxy" : "", sizeof_kmp_task_t, sizeof_shareds,
task_entry); }
;
1241#else
1242 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", sizeof_kmp_task_t
, sizeof_shareds, task_entry); }
1243 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", sizeof_kmp_task_t
, sizeof_shareds, task_entry); }
1244 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", sizeof_kmp_task_t
, sizeof_shareds, task_entry); }
1245 sizeof_kmp_task_t, sizeof_shareds, task_entry))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
"sizeof_task=%ld sizeof_shared=%ld entry=%p\n", gtid, loc_ref
, input_flags->tiedness ? "tied " : "untied", sizeof_kmp_task_t
, sizeof_shareds, task_entry); }
;
1246#endif
1247
1248 retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t,
1249 sizeof_shareds, task_entry);
1250
1251 KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n"
, gtid, retval); }
;
1252
1253 return retval;
1254}
1255
1256// __kmp_invoke_task: invoke the specified task
1257//
1258// gtid: global thread ID of caller
1259// task: the task to invoke
1260// current_task: the task to resume after task invokation
1261static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
1262 kmp_taskdata_t *current_task) {
1263 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
1264 kmp_uint64 cur_time;
36
'cur_time' declared without an initial value
1265#if OMP_40_ENABLED(50 >= 40)
1266 int discard = 0 /* false */;
1267#endif
1268 KA_TRACE(if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n"
, gtid, taskdata, current_task); }
1269 30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n"
, gtid, taskdata, current_task); }
1270 gtid, taskdata, current_task))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n"
, gtid, taskdata, current_task); }
;
1271 KMP_DEBUG_ASSERT(task)((task) ? 0 : __kmp_debug_assert("task", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1271))
;
1272#if OMP_45_ENABLED(50 >= 45)
1273 if (taskdata->td_flags.proxy == TASK_PROXY1 &&
1274 taskdata->td_flags.complete == 1) {
1275 // This is a proxy task that was already completed but it needs to run
1276 // its bottom-half finish
1277 KA_TRACE(if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n"
, gtid, taskdata); }
1278 30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n"
, gtid, taskdata); }
1279 ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n"
, gtid, taskdata); }
1280 gtid, taskdata))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n"
, gtid, taskdata); }
;
1281
1282 __kmp_bottom_half_finish_proxy(gtid, task);
1283
1284 KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for "if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(exit): T#%d completed bottom finish for "
"proxy task %p, resuming task %p\n", gtid, taskdata, current_task
); }
1285 "proxy task %p, resuming task %p\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(exit): T#%d completed bottom finish for "
"proxy task %p, resuming task %p\n", gtid, taskdata, current_task
); }
1286 gtid, taskdata, current_task))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(exit): T#%d completed bottom finish for "
"proxy task %p, resuming task %p\n", gtid, taskdata, current_task
); }
;
1287
1288 return;
1289 }
1290#endif
1291
1292#if USE_ITT_BUILD1 && USE_ITT_NOTIFY1
1293 if (__kmp_forkjoin_frames_mode == 3) {
37
Assuming '__kmp_forkjoin_frames_mode' is not equal to 3
38
Taking false branch
1294 // Get the current time stamp to measure task execution time to correct
1295 // barrier imbalance time
1296 cur_time = __itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0();
1297 }
1298#endif
1299
1300#if OMPT_SUPPORT1
1301 // For untied tasks, the first task executed only calls __kmpc_omp_task and
1302 // does not execute code.
1303 ompt_thread_info_t oldInfo;
1304 kmp_info_t *thread;
1305 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
39
Taking false branch
1306 // Store the threads states and restore them after the task
1307 thread = __kmp_threads[gtid];
1308 oldInfo = thread->th.ompt_thread_info;
1309 thread->th.ompt_thread_info.wait_id = 0;
1310 thread->th.ompt_thread_info.state = (thread->th.th_team_serialized)
1311 ? omp_state_work_serial
1312 : omp_state_work_parallel;
1313 taskdata->ompt_task_info.frame.exit_frame = OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
1314 }
1315#endif
1316
1317#if OMP_45_ENABLED(50 >= 45)
1318 // Proxy tasks are not handled by the runtime
1319 if (taskdata->td_flags.proxy != TASK_PROXY1) {
40
Taking true branch
1320#endif
1321 ANNOTATE_HAPPENS_AFTER(task);
1322 __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded
1323#if OMP_45_ENABLED(50 >= 45)
1324 }
1325#endif
1326
1327#if OMP_40_ENABLED(50 >= 40)
1328 // TODO: cancel tasks if the parallel region has also been cancelled
1329 // TODO: check if this sequence can be hoisted above __kmp_task_start
1330 // if cancellation has been enabled for this run ...
1331 if (__kmp_omp_cancellation) {
41
Assuming '__kmp_omp_cancellation' is 0
42
Taking false branch
1332 kmp_info_t *this_thr = __kmp_threads[gtid];
1333 kmp_team_t *this_team = this_thr->th.th_team;
1334 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
1335 if ((taskgroup && taskgroup->cancel_request) ||
1336 (this_team->t.t_cancel_request == cancel_parallel)) {
1337#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
1338 ompt_data_t *task_data;
1339 if (UNLIKELY(ompt_enabled.ompt_callback_cancel)__builtin_expect(!!(ompt_enabled.ompt_callback_cancel), 0)) {
1340 __ompt_get_task_info_internal(0, NULL__null, &task_data, NULL__null, NULL__null, NULL__null);
1341 ompt_callbacks.ompt_callback(ompt_callback_cancel)ompt_callback_cancel_callback(
1342 task_data,
1343 ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup
1344 : ompt_cancel_parallel) |
1345 ompt_cancel_discarded_task,
1346 NULL__null);
1347 }
1348#endif
1349 KMP_COUNT_BLOCK(TASK_cancelled)((void)0);
1350 // this task belongs to a task group and we need to cancel it
1351 discard = 1 /* true */;
1352 }
1353 }
1354
1355 // Invoke the task routine and pass in relevant data.
1356 // Thunks generated by gcc take a different argument list.
1357 if (!discard) {
43
Taking true branch
1358 if (taskdata->td_flags.tiedness == TASK_UNTIED0) {
44
Assuming the condition is false
45
Taking false branch
1359 taskdata->td_last_tied = current_task->td_last_tied;
1360 KMP_DEBUG_ASSERT(taskdata->td_last_tied)((taskdata->td_last_tied) ? 0 : __kmp_debug_assert("taskdata->td_last_tied"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1360))
;
1361 }
1362#if KMP_STATS_ENABLED0
1363 KMP_COUNT_BLOCK(TASK_executed)((void)0);
1364 switch (KMP_GET_THREAD_STATE()((void)0)) {
1365 case FORK_JOIN_BARRIER:
1366 KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar)((void)0);
1367 break;
1368 case PLAIN_BARRIER:
1369 KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar)((void)0);
1370 break;
1371 case TASKYIELD:
1372 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield)((void)0);
1373 break;
1374 case TASKWAIT:
1375 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait)((void)0);
1376 break;
1377 case TASKGROUP:
1378 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup)((void)0);
1379 break;
1380 default:
1381 KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate)((void)0);
1382 break;
1383 }
1384#endif // KMP_STATS_ENABLED
1385#endif // OMP_40_ENABLED
1386
1387// OMPT task begin
1388#if OMPT_SUPPORT1
1389 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0))
46
Taking false branch
1390 __ompt_task_start(task, current_task, gtid);
1391#endif
1392
1393#ifdef KMP_GOMP_COMPAT
1394 if (taskdata->td_flags.native) {
47
Assuming the condition is false
48
Taking false branch
1395 ((void (*)(void *))(*(task->routine)))(task->shareds);
1396 } else
1397#endif /* KMP_GOMP_COMPAT */
1398 {
1399 (*(task->routine))(gtid, task);
1400 }
1401 KMP_POP_PARTITIONED_TIMER()((void)0);
1402
1403#if OMP_40_ENABLED(50 >= 40)
1404 }
1405#endif // OMP_40_ENABLED
1406
1407
1408#if OMP_45_ENABLED(50 >= 45)
1409 // Proxy tasks are not handled by the runtime
1410 if (taskdata->td_flags.proxy != TASK_PROXY1) {
49
Assuming the condition is true
50
Taking true branch
1411#endif
1412 ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent);
1413#if OMPT_SUPPORT1
1414 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
51
Taking false branch
1415 thread->th.ompt_thread_info = oldInfo;
1416 if (taskdata->td_flags.tiedness == TASK_TIED1) {
1417 taskdata->ompt_task_info.frame.exit_frame = NULL__null;
1418 }
1419 __kmp_task_finish<true>(gtid, task, current_task);
1420 } else
1421#endif
1422 __kmp_task_finish<false>(gtid, task, current_task);
1423#if OMP_45_ENABLED(50 >= 45)
1424 }
1425#endif
1426
1427#if USE_ITT_BUILD1 && USE_ITT_NOTIFY1
1428 // Barrier imbalance - correct arrive time after the task finished
1429 if (__kmp_forkjoin_frames_mode == 3) {
52
Assuming '__kmp_forkjoin_frames_mode' is equal to 3
53
Taking true branch
1430 kmp_info_t *this_thr = __kmp_threads[gtid];
1431 if (this_thr->th.th_bar_arrive_time) {
54
Assuming the condition is true
55
Taking true branch
1432 this_thr->th.th_bar_arrive_time += (__itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0() - cur_time);
56
The right operand of '-' is a garbage value
1433 }
1434 }
1435#endif
1436 KA_TRACE(if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n"
, gtid, taskdata, current_task); }
1437 30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n"
, gtid, taskdata, current_task); }
1438 ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n"
, gtid, taskdata, current_task); }
1439 gtid, taskdata, current_task))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n"
, gtid, taskdata, current_task); }
;
1440 return;
1441}
1442
1443// __kmpc_omp_task_parts: Schedule a thread-switchable task for execution
1444//
1445// loc_ref: location of original task pragma (ignored)
1446// gtid: Global Thread ID of encountering thread
1447// new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task''
1448// Returns:
1449// TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1450// be resumed later.
1451// TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1452// resumed later.
1453kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid,
1454 kmp_task_t *new_task) {
1455 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task)(((kmp_taskdata_t *)new_task) - 1);
1456
1457 KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, new_taskdata); }
1458 loc_ref, new_taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, new_taskdata); }
;
1459
1460#if OMPT_SUPPORT1
1461 kmp_taskdata_t *parent;
1462 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
1463 parent = new_taskdata->td_parent;
1464 if (ompt_enabled.ompt_callback_task_create) {
1465 ompt_data_t task_data = ompt_data_none;
1466 ompt_callbacks.ompt_callback(ompt_callback_task_create)ompt_callback_task_create_callback(
1467 parent ? &(parent->ompt_task_info.task_data) : &task_data,
1468 parent ? &(parent->ompt_task_info.frame) : NULL__null,
1469 &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0,
1470 OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0));
1471 }
1472 }
1473#endif
1474
1475 /* Should we execute the new task or queue it? For now, let's just always try
1476 to queue it. If the queue fills up, then we'll execute it. */
1477
1478 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED1) // if cannot defer
1479 { // Execute this task immediately
1480 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1481 new_taskdata->td_flags.task_serial = 1;
1482 __kmp_invoke_task(gtid, new_task, current_task);
1483 }
1484
1485 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
"loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref
, new_taskdata); }
1486 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
"loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref
, new_taskdata); }
1487 ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
"loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref
, new_taskdata); }
1488 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
"loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref
, new_taskdata); }
1489 gtid, loc_ref, new_taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
"loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", gtid, loc_ref
, new_taskdata); }
;
1490
1491 ANNOTATE_HAPPENS_BEFORE(new_task);
1492#if OMPT_SUPPORT1
1493 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
1494 parent->ompt_task_info.frame.enter_frame = NULL__null;
1495 }
1496#endif
1497 return TASK_CURRENT_NOT_QUEUED0;
1498}
1499
1500// __kmp_omp_task: Schedule a non-thread-switchable task for execution
1501//
1502// gtid: Global Thread ID of encountering thread
1503// new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc()
1504// serialize_immediate: if TRUE then if the task is executed immediately its
1505// execution will be serialized
1506// Returns:
1507// TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1508// be resumed later.
1509// TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1510// resumed later.
1511kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
1512 bool serialize_immediate) {
1513 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task)(((kmp_taskdata_t *)new_task) - 1);
1514
1515/* Should we execute the new task or queue it? For now, let's just always try to
1516 queue it. If the queue fills up, then we'll execute it. */
1517#if OMP_45_ENABLED(50 >= 45)
1518 if (new_taskdata->td_flags.proxy == TASK_PROXY1 ||
32
Assuming the condition is false
33
Taking true branch
1519 __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED1) // if cannot defer
1520#else
1521 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED1) // if cannot defer
1522#endif
1523 { // Execute this task immediately
1524 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1525 if (serialize_immediate)
34
Taking true branch
1526 new_taskdata->td_flags.task_serial = 1;
1527 __kmp_invoke_task(gtid, new_task, current_task);
35
Calling '__kmp_invoke_task'
1528 }
1529
1530 ANNOTATE_HAPPENS_BEFORE(new_task);
1531 return TASK_CURRENT_NOT_QUEUED0;
1532}
1533
1534// __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a
1535// non-thread-switchable task from the parent thread only!
1536//
1537// loc_ref: location of original task pragma (ignored)
1538// gtid: Global Thread ID of encountering thread
1539// new_task: non-thread-switchable task thunk allocated by
1540// __kmp_omp_task_alloc()
1541// Returns:
1542// TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1543// be resumed later.
1544// TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1545// resumed later.
1546kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
1547 kmp_task_t *new_task) {
1548 kmp_int32 res;
1549 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK)((void)0);
1550
1551#if KMP_DEBUG1 || OMPT_SUPPORT1
1552 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task)(((kmp_taskdata_t *)new_task) - 1);
1553#endif
1554 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, new_taskdata); }
1555 new_taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, new_taskdata); }
;
1556
1557#if OMPT_SUPPORT1
1558 kmp_taskdata_t *parent = NULL__null;
1559 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
1560 if (!new_taskdata->td_flags.started) {
1561 OMPT_STORE_RETURN_ADDRESS(gtid)if (ompt_enabled.enabled && gtid >= 0 && __kmp_threads
[gtid] && !__kmp_threads[gtid]->th.ompt_thread_info
.return_address) __kmp_threads[gtid]->th.ompt_thread_info.
return_address = __builtin_return_address(0)
;
1562 parent = new_taskdata->td_parent;
1563 if (!parent->ompt_task_info.frame.enter_frame) {
1564 parent->ompt_task_info.frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1565 }
1566 if (ompt_enabled.ompt_callback_task_create) {
1567 ompt_data_t task_data = ompt_data_none;
1568 ompt_callbacks.ompt_callback(ompt_callback_task_create)ompt_callback_task_create_callback(
1569 parent ? &(parent->ompt_task_info.task_data) : &task_data,
1570 parent ? &(parent->ompt_task_info.frame) : NULL__null,
1571 &(new_taskdata->ompt_task_info.task_data),
1572 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata)((new_taskdata->td_flags.task_serial || new_taskdata->td_flags
.tasking_ser) ? ompt_task_undeferred : 0x0) | ((!(new_taskdata
->td_flags.tiedness)) ? ompt_task_untied : 0x0) | (new_taskdata
->td_flags.final ? ompt_task_final : 0x0) | (new_taskdata->
td_flags.merged_if0 ? ompt_task_mergeable : 0x0)
, 0,
1573 OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid));
1574 }
1575 } else {
1576 // We are scheduling the continuation of an UNTIED task.
1577 // Scheduling back to the parent task.
1578 __ompt_task_finish(new_task,
1579 new_taskdata->ompt_task_info.scheduling_parent,
1580 ompt_task_others);
1581 new_taskdata->ompt_task_info.frame.exit_frame = NULL__null;
1582 }
1583 }
1584#endif
1585
1586 res = __kmp_omp_task(gtid, new_task, true);
1587
1588 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(exit): T#%d returning "
"TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", gtid, loc_ref, new_taskdata
); }
1589 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(exit): T#%d returning "
"TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", gtid, loc_ref, new_taskdata
); }
1590 gtid, loc_ref, new_taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(exit): T#%d returning "
"TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", gtid, loc_ref, new_taskdata
); }
;
1591#if OMPT_SUPPORT1
1592 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)__builtin_expect(!!(ompt_enabled.enabled && parent !=
__null), 0)
) {
1593 parent->ompt_task_info.frame.enter_frame = NULL__null;
1594 }
1595#endif
1596 return res;
1597}
1598
1599// __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule
1600// a taskloop task with the correct OMPT return address
1601//
1602// loc_ref: location of original task pragma (ignored)
1603// gtid: Global Thread ID of encountering thread
1604// new_task: non-thread-switchable task thunk allocated by
1605// __kmp_omp_task_alloc()
1606// codeptr_ra: return address for OMPT callback
1607// Returns:
1608// TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1609// be resumed later.
1610// TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1611// resumed later.
1612kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid,
1613 kmp_task_t *new_task, void *codeptr_ra) {
1614 kmp_int32 res;
1615 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK)((void)0);
1616
1617#if KMP_DEBUG1 || OMPT_SUPPORT1
1618 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task)(((kmp_taskdata_t *)new_task) - 1);
1619#endif
1620 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, new_taskdata); }
1621 new_taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n"
, gtid, loc_ref, new_taskdata); }
;
1622
1623#if OMPT_SUPPORT1
1624 kmp_taskdata_t *parent = NULL__null;
1625 if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)__builtin_expect(!!(ompt_enabled.enabled && !new_taskdata
->td_flags.started), 0)
) {
30
Taking false branch
1626 parent = new_taskdata->td_parent;
1627 if (!parent->ompt_task_info.frame.enter_frame)
1628 parent->ompt_task_info.frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1);
1629 if (ompt_enabled.ompt_callback_task_create) {
1630 ompt_data_t task_data = ompt_data_none;
1631 ompt_callbacks.ompt_callback(ompt_callback_task_create)ompt_callback_task_create_callback(
1632 parent ? &(parent->ompt_task_info.task_data) : &task_data,
1633 parent ? &(parent->ompt_task_info.frame) : NULL__null,
1634 &(new_taskdata->ompt_task_info.task_data),
1635 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata)((new_taskdata->td_flags.task_serial || new_taskdata->td_flags
.tasking_ser) ? ompt_task_undeferred : 0x0) | ((!(new_taskdata
->td_flags.tiedness)) ? ompt_task_untied : 0x0) | (new_taskdata
->td_flags.final ? ompt_task_final : 0x0) | (new_taskdata->
td_flags.merged_if0 ? ompt_task_mergeable : 0x0)
, 0,
1636 codeptr_ra);
1637 }
1638 }
1639#endif
1640
1641 res = __kmp_omp_task(gtid, new_task, true);
31
Calling '__kmp_omp_task'
1642
1643 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(exit): T#%d returning "
"TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", gtid, loc_ref, new_taskdata
); }
1644 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(exit): T#%d returning "
"TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", gtid, loc_ref, new_taskdata
); }
1645 gtid, loc_ref, new_taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_task(exit): T#%d returning "
"TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", gtid, loc_ref, new_taskdata
); }
;
1646#if OMPT_SUPPORT1
1647 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)__builtin_expect(!!(ompt_enabled.enabled && parent !=
__null), 0)
) {
1648 parent->ompt_task_info.frame.enter_frame = NULL__null;
1649 }
1650#endif
1651 return res;
1652}
1653
1654template <bool ompt>
1655static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
1656 void *frame_address,
1657 void *return_address) {
1658 kmp_taskdata_t *taskdata;
1659 kmp_info_t *thread;
1660 int thread_finished = FALSE0;
1661 KMP_SET_THREAD_STATE_BLOCK(TASKWAIT)((void)0);
1662
1663 KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n"
, gtid, loc_ref); }
;
1664
1665 if (__kmp_tasking_mode != tskm_immediate_exec) {
1666 thread = __kmp_threads[gtid];
1667 taskdata = thread->th.th_current_task;
1668
1669#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
1670 ompt_data_t *my_task_data;
1671 ompt_data_t *my_parallel_data;
1672
1673 if (ompt) {
1674 my_task_data = &(taskdata->ompt_task_info.task_data);
1675 my_parallel_data = OMPT_CUR_TEAM_DATA(thread)(&(thread->th.th_team->t.ompt_team_info.parallel_data
))
;
1676
1677 taskdata->ompt_task_info.frame.enter_frame = frame_address;
1678
1679 if (ompt_enabled.ompt_callback_sync_region) {
1680 ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback(
1681 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
1682 my_task_data, return_address);
1683 }
1684
1685 if (ompt_enabled.ompt_callback_sync_region_wait) {
1686 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback(
1687 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
1688 my_task_data, return_address);
1689 }
1690 }
1691#endif // OMPT_SUPPORT && OMPT_OPTIONAL
1692
1693// Debugger: The taskwait is active. Store location and thread encountered the
1694// taskwait.
1695#if USE_ITT_BUILD1
1696// Note: These values are used by ITT events as well.
1697#endif /* USE_ITT_BUILD */
1698 taskdata->td_taskwait_counter += 1;
1699 taskdata->td_taskwait_ident = loc_ref;
1700 taskdata->td_taskwait_thread = gtid + 1;
1701
1702#if USE_ITT_BUILD1
1703 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
1704 if (itt_sync_obj != NULL__null)
1705 __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
1706#endif /* USE_ITT_BUILD */
1707
1708 bool must_wait =
1709 !taskdata->td_flags.team_serial && !taskdata->td_flags.final;
1710
1711#if OMP_45_ENABLED(50 >= 45)
1712 must_wait = must_wait || (thread->th.th_task_team != NULL__null &&
1713 thread->th.th_task_team->tt.tt_found_proxy_tasks);
1714#endif
1715 if (must_wait) {
1716 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *,reinterpret_cast<std::atomic<kmp_uint32> *>(&
(taskdata->td_incomplete_child_tasks))
1717 &(taskdata->td_incomplete_child_tasks))reinterpret_cast<std::atomic<kmp_uint32> *>(&
(taskdata->td_incomplete_child_tasks))
,
1718 0U);
1719 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks)(&taskdata->td_incomplete_child_tasks)->load(std::memory_order_acquire
)
!= 0) {
1720 flag.execute_tasks(thread, gtid, FALSE0,
1721 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), itt_sync_obj,
1722 __kmp_task_stealing_constraint);
1723 }
1724 }
1725#if USE_ITT_BUILD1
1726 if (itt_sync_obj != NULL__null)
1727 __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
1728#endif /* USE_ITT_BUILD */
1729
1730 // Debugger: The taskwait is completed. Location remains, but thread is
1731 // negated.
1732 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
1733
1734#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
1735 if (ompt) {
1736 if (ompt_enabled.ompt_callback_sync_region_wait) {
1737 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback(
1738 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
1739 my_task_data, return_address);
1740 }
1741 if (ompt_enabled.ompt_callback_sync_region) {
1742 ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback(
1743 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
1744 my_task_data, return_address);
1745 }
1746 taskdata->ompt_task_info.frame.enter_frame = NULL__null;
1747 }
1748#endif // OMPT_SUPPORT && OMPT_OPTIONAL
1749
1750 ANNOTATE_HAPPENS_AFTER(taskdata);
1751 }
1752
1753 KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
"returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata); }
1754 "returning TASK_CURRENT_NOT_QUEUED\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
"returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata); }
1755 gtid, taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
"returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata); }
;
1756
1757 return TASK_CURRENT_NOT_QUEUED0;
1758}
1759
1760#if OMPT_SUPPORT1
1761OMPT_NOINLINE__attribute__((noinline))
1762static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid,
1763 void *frame_address,
1764 void *return_address) {
1765 return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address,
1766 return_address);
1767}
1768#endif // OMPT_SUPPORT
1769
1770// __kmpc_omp_taskwait: Wait until all tasks generated by the current task are
1771// complete
1772kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) {
1773#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
1774 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
1775 OMPT_STORE_RETURN_ADDRESS(gtid)if (ompt_enabled.enabled && gtid >= 0 && __kmp_threads
[gtid] && !__kmp_threads[gtid]->th.ompt_thread_info
.return_address) __kmp_threads[gtid]->th.ompt_thread_info.
return_address = __builtin_return_address(0)
;
1776 return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(1)__builtin_frame_address(1),
1777 OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid));
1778 }
1779#endif
1780 return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL__null, NULL__null);
1781}
1782
1783// __kmpc_omp_taskyield: switch to a different task
1784kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
1785 kmp_taskdata_t *taskdata;
1786 kmp_info_t *thread;
1787 int thread_finished = FALSE0;
1788
1789 KMP_COUNT_BLOCK(OMP_TASKYIELD)((void)0);
1790 KMP_SET_THREAD_STATE_BLOCK(TASKYIELD)((void)0);
1791
1792 KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n"
, gtid, loc_ref, end_part); }
1793 gtid, loc_ref, end_part))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n"
, gtid, loc_ref, end_part); }
;
1794
1795 if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) {
1796 thread = __kmp_threads[gtid];
1797 taskdata = thread->th.th_current_task;
1798// Should we model this as a task wait or not?
1799// Debugger: The taskwait is active. Store location and thread encountered the
1800// taskwait.
1801#if USE_ITT_BUILD1
1802// Note: These values are used by ITT events as well.
1803#endif /* USE_ITT_BUILD */
1804 taskdata->td_taskwait_counter += 1;
1805 taskdata->td_taskwait_ident = loc_ref;
1806 taskdata->td_taskwait_thread = gtid + 1;
1807
1808#if USE_ITT_BUILD1
1809 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
1810 if (itt_sync_obj != NULL__null)
1811 __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
1812#endif /* USE_ITT_BUILD */
1813 if (!taskdata->td_flags.team_serial) {
1814 kmp_task_team_t *task_team = thread->th.th_task_team;
1815 if (task_team != NULL__null) {
1816 if (KMP_TASKING_ENABLED(task_team)(((task_team)->tt.tt_found_tasks) == (!0))) {
1817#if OMPT_SUPPORT1
1818 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0))
1819 thread->th.ompt_thread_info.ompt_task_yielded = 1;
1820#endif
1821 __kmp_execute_tasks_32(
1822 thread, gtid, NULL__null, FALSE0,
1823 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), itt_sync_obj,
1824 __kmp_task_stealing_constraint);
1825#if OMPT_SUPPORT1
1826 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0))
1827 thread->th.ompt_thread_info.ompt_task_yielded = 0;
1828#endif
1829 }
1830 }
1831 }
1832#if USE_ITT_BUILD1
1833 if (itt_sync_obj != NULL__null)
1834 __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
1835#endif /* USE_ITT_BUILD */
1836
1837 // Debugger: The taskwait is completed. Location remains, but thread is
1838 // negated.
1839 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
1840 }
1841
1842 KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
"returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata); }
1843 "returning TASK_CURRENT_NOT_QUEUED\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
"returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata); }
1844 gtid, taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
"returning TASK_CURRENT_NOT_QUEUED\n", gtid, taskdata); }
;
1845
1846 return TASK_CURRENT_NOT_QUEUED0;
1847}
1848
1849// TODO: change to OMP_50_ENABLED, need to change build tools for this to work
1850#if OMP_45_ENABLED(50 >= 45)
1851// Task Reduction implementation
1852
1853typedef struct kmp_task_red_flags {
1854 unsigned lazy_priv : 1; // hint: (1) use lazy allocation (big objects)
1855 unsigned reserved31 : 31;
1856} kmp_task_red_flags_t;
1857
1858// internal structure for reduction data item related info
1859typedef struct kmp_task_red_data {
1860 void *reduce_shar; // shared reduction item
1861 size_t reduce_size; // size of data item
1862 void *reduce_priv; // thread specific data
1863 void *reduce_pend; // end of private data for comparison op
1864 void *reduce_init; // data initialization routine
1865 void *reduce_fini; // data finalization routine
1866 void *reduce_comb; // data combiner routine
1867 kmp_task_red_flags_t flags; // flags for additional info from compiler
1868} kmp_task_red_data_t;
1869
1870// structure sent us by compiler - one per reduction item
1871typedef struct kmp_task_red_input {
1872 void *reduce_shar; // shared reduction item
1873 size_t reduce_size; // size of data item
1874 void *reduce_init; // data initialization routine
1875 void *reduce_fini; // data finalization routine
1876 void *reduce_comb; // data combiner routine
1877 kmp_task_red_flags_t flags; // flags for additional info from compiler
1878} kmp_task_red_input_t;
1879
1880/*!
1881@ingroup TASKING
1882@param gtid Global thread ID
1883@param num Number of data items to reduce
1884@param data Array of data for reduction
1885@return The taskgroup identifier
1886
1887Initialize task reduction for the taskgroup.
1888*/
1889void *__kmpc_task_reduction_init(int gtid, int num, void *data) {
1890 kmp_info_t *thread = __kmp_threads[gtid];
1891 kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup;
1892 kmp_int32 nth = thread->th.th_team_nproc;
1893 kmp_task_red_input_t *input = (kmp_task_red_input_t *)data;
1894 kmp_task_red_data_t *arr;
1895
1896 // check input data just in case
1897 KMP_ASSERT(tg != NULL)((tg != __null) ? 0 : __kmp_debug_assert("tg != NULL", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1897))
;
1898 KMP_ASSERT(data != NULL)((data != __null) ? 0 : __kmp_debug_assert("data != NULL", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1898))
;
1899 KMP_ASSERT(num > 0)((num > 0) ? 0 : __kmp_debug_assert("num > 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1899))
;
1900 if (nth == 1) {
1901 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n"
, gtid, tg); }
1902 gtid, tg))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n"
, gtid, tg); }
;
1903 return (void *)tg;
1904 }
1905 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n"
, gtid, tg, num); }
1906 gtid, tg, num))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n"
, gtid, tg, num); }
;
1907 arr = (kmp_task_red_data_t *)__kmp_thread_malloc(___kmp_thread_malloc((thread), (num * sizeof(kmp_task_red_data_t
)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1908)
1908 thread, num * sizeof(kmp_task_red_data_t))___kmp_thread_malloc((thread), (num * sizeof(kmp_task_red_data_t
)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1908)
;
1909 for (int i = 0; i < num; ++i) {
1910 void (*f_init)(void *) = (void (*)(void *))(input[i].reduce_init);
1911 size_t size = input[i].reduce_size - 1;
1912 // round the size up to cache line per thread-specific item
1913 size += CACHE_LINE64 - size % CACHE_LINE64;
1914 KMP_ASSERT(input[i].reduce_comb != NULL)((input[i].reduce_comb != __null) ? 0 : __kmp_debug_assert("input[i].reduce_comb != NULL"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1914))
; // combiner is mandatory
1915 arr[i].reduce_shar = input[i].reduce_shar;
1916 arr[i].reduce_size = size;
1917 arr[i].reduce_init = input[i].reduce_init;
1918 arr[i].reduce_fini = input[i].reduce_fini;
1919 arr[i].reduce_comb = input[i].reduce_comb;
1920 arr[i].flags = input[i].flags;
1921 if (!input[i].flags.lazy_priv) {
1922 // allocate cache-line aligned block and fill it with zeros
1923 arr[i].reduce_priv = __kmp_allocate(nth * size)___kmp_allocate((nth * size), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1923)
;
1924 arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size;
1925 if (f_init != NULL__null) {
1926 // initialize thread-specific items
1927 for (int j = 0; j < nth; ++j) {
1928 f_init((char *)(arr[i].reduce_priv) + j * size);
1929 }
1930 }
1931 } else {
1932 // only allocate space for pointers now,
1933 // objects will be lazily allocated/initialized once requested
1934 arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *))___kmp_allocate((nth * sizeof(void *)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1934)
;
1935 }
1936 }
1937 tg->reduce_data = (void *)arr;
1938 tg->reduce_num_data = num;
1939 return (void *)tg;
1940}
1941
1942/*!
1943@ingroup TASKING
1944@param gtid Global thread ID
1945@param tskgrp The taskgroup ID (optional)
1946@param data Shared location of the item
1947@return The pointer to per-thread data
1948
1949Get thread-specific location of data item
1950*/
1951void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
1952 kmp_info_t *thread = __kmp_threads[gtid];
1953 kmp_int32 nth = thread->th.th_team_nproc;
1954 if (nth == 1)
1955 return data; // nothing to do
1956
1957 kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp;
1958 if (tg == NULL__null)
1959 tg = thread->th.th_current_task->td_taskgroup;
1960 KMP_ASSERT(tg != NULL)((tg != __null) ? 0 : __kmp_debug_assert("tg != NULL", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1960))
;
1961 kmp_task_red_data_t *arr = (kmp_task_red_data_t *)(tg->reduce_data);
1962 kmp_int32 num = tg->reduce_num_data;
1963 kmp_int32 tid = thread->th.th_info.ds.ds_tid;
1964
1965 KMP_ASSERT(data != NULL)((data != __null) ? 0 : __kmp_debug_assert("data != NULL", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1965))
;
1966 while (tg != NULL__null) {
1967 for (int i = 0; i < num; ++i) {
1968 if (!arr[i].flags.lazy_priv) {
1969 if (data == arr[i].reduce_shar ||
1970 (data >= arr[i].reduce_priv && data < arr[i].reduce_pend))
1971 return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size;
1972 } else {
1973 // check shared location first
1974 void **p_priv = (void **)(arr[i].reduce_priv);
1975 if (data == arr[i].reduce_shar)
1976 goto found;
1977 // check if we get some thread specific location as parameter
1978 for (int j = 0; j < nth; ++j)
1979 if (data == p_priv[j])
1980 goto found;
1981 continue; // not found, continue search
1982 found:
1983 if (p_priv[tid] == NULL__null) {
1984 // allocate thread specific object lazily
1985 void (*f_init)(void *) = (void (*)(void *))(arr[i].reduce_init);
1986 p_priv[tid] = __kmp_allocate(arr[i].reduce_size)___kmp_allocate((arr[i].reduce_size), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1986)
;
1987 if (f_init != NULL__null) {
1988 f_init(p_priv[tid]);
1989 }
1990 }
1991 return p_priv[tid];
1992 }
1993 }
1994 tg = tg->parent;
1995 arr = (kmp_task_red_data_t *)(tg->reduce_data);
1996 num = tg->reduce_num_data;
1997 }
1998 KMP_ASSERT2(0, "Unknown task reduction item")((0) ? 0 : __kmp_debug_assert(("Unknown task reduction item")
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 1998))
;
1999 return NULL__null; // ERROR, this line never executed
2000}
2001
2002// Finalize task reduction.
2003// Called from __kmpc_end_taskgroup()
2004static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) {
2005 kmp_int32 nth = th->th.th_team_nproc;
2006 KMP_DEBUG_ASSERT(nth > 1)((nth > 1) ? 0 : __kmp_debug_assert("nth > 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2006))
; // should not be called if nth == 1
2007 kmp_task_red_data_t *arr = (kmp_task_red_data_t *)tg->reduce_data;
2008 kmp_int32 num = tg->reduce_num_data;
2009 for (int i = 0; i < num; ++i) {
2010 void *sh_data = arr[i].reduce_shar;
2011 void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini);
2012 void (*f_comb)(void *, void *) =
2013 (void (*)(void *, void *))(arr[i].reduce_comb);
2014 if (!arr[i].flags.lazy_priv) {
2015 void *pr_data = arr[i].reduce_priv;
2016 size_t size = arr[i].reduce_size;
2017 for (int j = 0; j < nth; ++j) {
2018 void *priv_data = (char *)pr_data + j * size;
2019 f_comb(sh_data, priv_data); // combine results
2020 if (f_fini)
2021 f_fini(priv_data); // finalize if needed
2022 }
2023 } else {
2024 void **pr_data = (void **)(arr[i].reduce_priv);
2025 for (int j = 0; j < nth; ++j) {
2026 if (pr_data[j] != NULL__null) {
2027 f_comb(sh_data, pr_data[j]); // combine results
2028 if (f_fini)
2029 f_fini(pr_data[j]); // finalize if needed
2030 __kmp_free(pr_data[j])___kmp_free((pr_data[j]), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2030)
;
2031 }
2032 }
2033 }
2034 __kmp_free(arr[i].reduce_priv)___kmp_free((arr[i].reduce_priv), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2034)
;
2035 }
2036 __kmp_thread_free(th, arr)___kmp_thread_free((th), (arr), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2036)
;
2037 tg->reduce_data = NULL__null;
2038 tg->reduce_num_data = 0;
2039}
2040#endif
2041
2042#if OMP_40_ENABLED(50 >= 40)
2043// __kmpc_taskgroup: Start a new taskgroup
2044void __kmpc_taskgroup(ident_t *loc, int gtid) {
2045 kmp_info_t *thread = __kmp_threads[gtid];
2046 kmp_taskdata_t *taskdata = thread->th.th_current_task;
2047 kmp_taskgroup_t *tg_new =
2048 (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t))___kmp_thread_malloc((thread), (sizeof(kmp_taskgroup_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2048)
;
2049 KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_taskgroup: T#%d loc=%p group=%p\n"
, gtid, loc, tg_new); }
;
2050 KMP_ATOMIC_ST_RLX(&tg_new->count, 0)(&tg_new->count)->store(0, std::memory_order_relaxed
)
;
2051 KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq)(&tg_new->cancel_request)->store(cancel_noreq, std::
memory_order_relaxed)
;
2052 tg_new->parent = taskdata->td_taskgroup;
2053// TODO: change to OMP_50_ENABLED, need to change build tools for this to work
2054#if OMP_45_ENABLED(50 >= 45)
2055 tg_new->reduce_data = NULL__null;
2056 tg_new->reduce_num_data = 0;
2057#endif
2058 taskdata->td_taskgroup = tg_new;
2059
2060#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
2061 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)__builtin_expect(!!(ompt_enabled.ompt_callback_sync_region), 0
)
) {
2062 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid);
2063 if (!codeptr)
2064 codeptr = OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0);
2065 kmp_team_t *team = thread->th.th_team;
2066 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data;
2067 // FIXME: I think this is wrong for lwt!
2068 ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data;
2069
2070 ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback(
2071 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2072 &(my_task_data), codeptr);
2073 }
2074#endif
2075}
2076
2077// __kmpc_end_taskgroup: Wait until all tasks generated by the current task
2078// and its descendants are complete
2079void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
2080 kmp_info_t *thread = __kmp_threads[gtid];
2081 kmp_taskdata_t *taskdata = thread->th.th_current_task;
2082 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
2083 int thread_finished = FALSE0;
2084
2085#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
2086 kmp_team_t *team;
2087 ompt_data_t my_task_data;
2088 ompt_data_t my_parallel_data;
2089 void *codeptr;
2090 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0)) {
2091 team = thread->th.th_team;
2092 my_task_data = taskdata->ompt_task_info.task_data;
2093 // FIXME: I think this is wrong for lwt!
2094 my_parallel_data = team->t.ompt_team_info.parallel_data;
2095 codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid);
2096 if (!codeptr)
2097 codeptr = OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0);
2098 }
2099#endif
2100
2101 KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n"
, gtid, loc); }
;
2102 KMP_DEBUG_ASSERT(taskgroup != NULL)((taskgroup != __null) ? 0 : __kmp_debug_assert("taskgroup != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2102))
;
2103 KMP_SET_THREAD_STATE_BLOCK(TASKGROUP)((void)0);
2104
2105 if (__kmp_tasking_mode != tskm_immediate_exec) {
2106 // mark task as waiting not on a barrier
2107 taskdata->td_taskwait_counter += 1;
2108 taskdata->td_taskwait_ident = loc;
2109 taskdata->td_taskwait_thread = gtid + 1;
2110#if USE_ITT_BUILD1
2111 // For ITT the taskgroup wait is similar to taskwait until we need to
2112 // distinguish them
2113 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
2114 if (itt_sync_obj != NULL__null)
2115 __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
2116#endif /* USE_ITT_BUILD */
2117
2118#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
2119 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)__builtin_expect(!!(ompt_enabled.ompt_callback_sync_region_wait
), 0)
) {
2120 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback(
2121 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2122 &(my_task_data), codeptr);
2123 }
2124#endif
2125
2126#if OMP_45_ENABLED(50 >= 45)
2127 if (!taskdata->td_flags.team_serial ||
2128 (thread->th.th_task_team != NULL__null &&
2129 thread->th.th_task_team->tt.tt_found_proxy_tasks))
2130#else
2131 if (!taskdata->td_flags.team_serial)
2132#endif
2133 {
2134 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count))reinterpret_cast<std::atomic<kmp_uint32> *>(&
(taskgroup->count))
,
2135 0U);
2136 while (KMP_ATOMIC_LD_ACQ(&taskgroup->count)(&taskgroup->count)->load(std::memory_order_acquire
)
!= 0) {
2137 flag.execute_tasks(thread, gtid, FALSE0,
2138 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), itt_sync_obj,
2139 __kmp_task_stealing_constraint);
2140 }
2141 }
2142 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting
2143
2144#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
2145 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)__builtin_expect(!!(ompt_enabled.ompt_callback_sync_region_wait
), 0)
) {
2146 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback(
2147 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2148 &(my_task_data), codeptr);
2149 }
2150#endif
2151
2152#if USE_ITT_BUILD1
2153 if (itt_sync_obj != NULL__null)
2154 __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
2155#endif /* USE_ITT_BUILD */
2156 }
2157 KMP_DEBUG_ASSERT(taskgroup->count == 0)((taskgroup->count == 0) ? 0 : __kmp_debug_assert("taskgroup->count == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2157))
;
2158
2159// TODO: change to OMP_50_ENABLED, need to change build tools for this to work
2160#if OMP_45_ENABLED(50 >= 45)
2161 if (taskgroup->reduce_data != NULL__null) // need to reduce?
2162 __kmp_task_reduction_fini(thread, taskgroup);
2163#endif
2164 // Restore parent taskgroup for the current task
2165 taskdata->td_taskgroup = taskgroup->parent;
2166 __kmp_thread_free(thread, taskgroup)___kmp_thread_free((thread), (taskgroup), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2166)
;
2167
2168 KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n"
, gtid, taskdata); }
2169 gtid, taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n"
, gtid, taskdata); }
;
2170 ANNOTATE_HAPPENS_AFTER(taskdata);
2171
2172#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
2173 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)__builtin_expect(!!(ompt_enabled.ompt_callback_sync_region), 0
)
) {
2174 ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback(
2175 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2176 &(my_task_data), codeptr);
2177 }
2178#endif
2179}
2180#endif
2181
2182// __kmp_remove_my_task: remove a task from my own deque
2183static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid,
2184 kmp_task_team_t *task_team,
2185 kmp_int32 is_constrained) {
2186 kmp_task_t *task;
2187 kmp_taskdata_t *taskdata;
2188 kmp_thread_data_t *thread_data;
2189 kmp_uint32 tail;
2190
2191 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec)((__kmp_tasking_mode != tskm_immediate_exec) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode != tskm_immediate_exec", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2191))
;
2192 KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data !=((task_team->tt.tt_threads_data != __null) ? 0 : __kmp_debug_assert
("task_team->tt.tt_threads_data != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2193))
2193 NULL)((task_team->tt.tt_threads_data != __null) ? 0 : __kmp_debug_assert
("task_team->tt.tt_threads_data != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2193))
; // Caller should check this condition
2194
2195 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2195)), __kmp_threads[(gtid)]->th.th_info.ds.ds_tid)
];
2196
2197 KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n"
, gtid, thread_data->td.td_deque_ntasks, thread_data->td
.td_deque_head, thread_data->td.td_deque_tail); }
2198 gtid, thread_data->td.td_deque_ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n"
, gtid, thread_data->td.td_deque_ntasks, thread_data->td
.td_deque_head, thread_data->td.td_deque_tail); }
2199 thread_data->td.td_deque_head, thread_data->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n"
, gtid, thread_data->td.td_deque_ntasks, thread_data->td
.td_deque_head, thread_data->td.td_deque_tail); }
;
2200
2201 if (TCR_4(thread_data->td.td_deque_ntasks)(thread_data->td.td_deque_ntasks) == 0) {
2202 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2203 ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2204 "ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2205 gtid, thread_data->td.td_deque_ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2206 thread_data->td.td_deque_head, thread_data->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
;
2207 return NULL__null;
2208 }
2209
2210 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2211
2212 if (TCR_4(thread_data->td.td_deque_ntasks)(thread_data->td.td_deque_ntasks) == 0) {
2213 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2214 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2215 ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2216 "ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2217 gtid, thread_data->td.td_deque_ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2218 thread_data->td.td_deque_head, thread_data->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
;
2219 return NULL__null;
2220 }
2221
2222 tail = (thread_data->td.td_deque_tail - 1) &
2223 TASK_DEQUE_MASK(thread_data->td)((thread_data->td).td_deque_size - 1); // Wrap index.
2224 taskdata = thread_data->td.td_deque[tail];
2225
2226 if (is_constrained && (taskdata->td_flags.tiedness == TASK_TIED1)) {
2227 // we need to check if the candidate obeys task scheduling constraint (TSC)
2228 // only descendant of all deferred tied tasks can be scheduled, checking
2229 // the last one is enough, as it in turn is the descendant of all others
2230 kmp_taskdata_t *current = thread->th.th_current_task->td_last_tied;
2231 KMP_DEBUG_ASSERT(current != NULL)((current != __null) ? 0 : __kmp_debug_assert("current != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2231))
;
2232 // check if last tied task is not suspended on barrier
2233 if (current->td_flags.tasktype == TASK_EXPLICIT1 ||
2234 current->td_taskwait_thread > 0) { // <= 0 on barrier
2235 kmp_int32 level = current->td_level;
2236 kmp_taskdata_t *parent = taskdata->td_parent;
2237 while (parent != current && parent->td_level > level) {
2238 parent = parent->td_parent; // check generation up to the level of the
2239 // current task
2240 KMP_DEBUG_ASSERT(parent != NULL)((parent != __null) ? 0 : __kmp_debug_assert("parent != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2240))
;
2241 }
2242 if (parent != current) {
2243 // The TSC does not allow to steal victim task
2244 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2245 KA_TRACE(10, ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2246 "ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2247 gtid, thread_data->td.td_deque_ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2248 thread_data->td.td_deque_head,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
2249 thread_data->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
"ntasks=%d head=%u tail=%u\n", gtid, thread_data->td.td_deque_ntasks
, thread_data->td.td_deque_head, thread_data->td.td_deque_tail
); }
;
2250 return NULL__null;
2251 }
2252 }
2253 }
2254
2255 thread_data->td.td_deque_tail = tail;
2256 TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1)(thread_data->td.td_deque_ntasks) = (thread_data->td.td_deque_ntasks
- 1)
;
2257
2258 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2259
2260 KA_TRACE(10, ("__kmp_remove_my_task(exit #2): T#%d task %p removed: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d task %p removed: "
"ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data->
td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
2261 "ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d task %p removed: "
"ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data->
td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
2262 gtid, taskdata, thread_data->td.td_deque_ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d task %p removed: "
"ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data->
td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
2263 thread_data->td.td_deque_head, thread_data->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_remove_my_task(exit #2): T#%d task %p removed: "
"ntasks=%d head=%u tail=%u\n", gtid, taskdata, thread_data->
td.td_deque_ntasks, thread_data->td.td_deque_head, thread_data
->td.td_deque_tail); }
;
2264
2265 task = KMP_TASKDATA_TO_TASK(taskdata)(kmp_task_t *)(taskdata + 1);
2266 return task;
2267}
2268
2269// __kmp_steal_task: remove a task from another thread's deque
2270// Assume that calling thread has already checked existence of
2271// task_team thread_data before calling this routine.
2272static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid,
2273 kmp_task_team_t *task_team,
2274 std::atomic<kmp_int32> *unfinished_threads,
2275 int *thread_finished,
2276 kmp_int32 is_constrained) {
2277 kmp_task_t *task;
2278 kmp_taskdata_t *taskdata;
2279 kmp_taskdata_t *current;
2280 kmp_thread_data_t *victim_td, *threads_data;
2281 kmp_int32 level, target;
2282 kmp_int32 victim_tid;
2283
2284 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec)((__kmp_tasking_mode != tskm_immediate_exec) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode != tskm_immediate_exec", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2284))
;
2285
2286 threads_data = task_team->tt.tt_threads_data;
2287 KMP_DEBUG_ASSERT(threads_data != NULL)((threads_data != __null) ? 0 : __kmp_debug_assert("threads_data != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2287))
; // Caller should check this condition
2288
2289 victim_tid = victim_thr->th.th_info.ds.ds_tid;
2290 victim_td = &threads_data[victim_tid];
2291
2292 KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2294)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2293 "task_team=%p ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2294)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2294 gtid, __kmp_gtid_from_thread(victim_thr), task_team,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2294)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2295 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2294)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2296 victim_td->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2294)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
;
2297
2298 if (TCR_4(victim_td->td.td_deque_ntasks)(victim_td->td.td_deque_ntasks) == 0) {
2299 KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2301)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2300 "task_team=%p ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2301)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2301 gtid, __kmp_gtid_from_thread(victim_thr), task_team,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2301)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2302 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2301)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
2303 victim_td->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2301)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head
, victim_td->td.td_deque_tail); }
;
2304 return NULL__null;
2305 }
2306
2307 __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock);
2308
2309 int ntasks = TCR_4(victim_td->td.td_deque_ntasks)(victim_td->td.td_deque_ntasks);
2310 // Check again after we acquire the lock
2311 if (ntasks == 0) {
2312 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2313 KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2315)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2314 "task_team=%p ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2315)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2315 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2315)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2316 victim_td->td.td_deque_head, victim_td->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((victim_thr
) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2315)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
;
2317 return NULL__null;
2318 }
2319
2320 KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL)((victim_td->td.td_deque != __null) ? 0 : __kmp_debug_assert
("victim_td->td.td_deque != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2320))
;
2321
2322 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head];
2323 if (is_constrained && (taskdata->td_flags.tiedness == TASK_TIED1)) {
2324 // we need to check if the candidate obeys task scheduling constraint (TSC)
2325 // only descendant of all deferred tied tasks can be scheduled, checking
2326 // the last one is enough, as it in turn is the descendant of all others
2327 current = __kmp_threads[gtid]->th.th_current_task->td_last_tied;
2328 KMP_DEBUG_ASSERT(current != NULL)((current != __null) ? 0 : __kmp_debug_assert("current != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2328))
;
2329 // check if last tied task is not suspended on barrier
2330 if (current->td_flags.tasktype == TASK_EXPLICIT1 ||
2331 current->td_taskwait_thread > 0) { // <= 0 on barrier
2332 level = current->td_level;
2333 kmp_taskdata_t *parent = taskdata->td_parent;
2334 while (parent != current && parent->td_level > level) {
2335 parent = parent->td_parent; // check generation up to the level of the
2336 // current task
2337 KMP_DEBUG_ASSERT(parent != NULL)((parent != __null) ? 0 : __kmp_debug_assert("parent != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2337))
;
2338 }
2339 if (parent != current) {
2340 if (!task_team->tt.tt_untied_task_encountered) {
2341 // The TSC does not allow to steal victim task
2342 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2343 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #3): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2346)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2344 ("__kmp_steal_task(exit #3): T#%d could not steal from "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #3): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2346)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2345 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #3): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2346)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2346 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #3): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2346)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2347 victim_td->td.td_deque_head, victim_td->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #3): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2346)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
;
2348 return NULL__null;
2349 }
2350 taskdata = NULL__null; // will check other tasks in victim's deque
2351 }
2352 }
2353 }
2354 if (taskdata != NULL__null) {
2355 // Bump head pointer and Wrap.
2356 victim_td->td.td_deque_head =
2357 (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td)((victim_td->td).td_deque_size - 1);
2358 } else {
2359 int i;
2360 // walk through victim's deque trying to steal any task
2361 target = victim_td->td.td_deque_head;
2362 for (i = 1; i < ntasks; ++i) {
2363 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td)((victim_td->td).td_deque_size - 1);
2364 taskdata = victim_td->td.td_deque[target];
2365 if (taskdata->td_flags.tiedness == TASK_TIED1) {
2366 // check if the candidate obeys the TSC
2367 kmp_taskdata_t *parent = taskdata->td_parent;
2368 // check generation up to the level of the current task
2369 while (parent != current && parent->td_level > level) {
2370 parent = parent->td_parent;
2371 KMP_DEBUG_ASSERT(parent != NULL)((parent != __null) ? 0 : __kmp_debug_assert("parent != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2371))
;
2372 }
2373 if (parent != current) {
2374 // The TSC does not allow to steal the candidate
2375 taskdata = NULL__null;
2376 continue;
2377 } else {
2378 // found victim tied task
2379 break;
2380 }
2381 } else {
2382 // found victim untied task
2383 break;
2384 }
2385 }
2386 if (taskdata == NULL__null) {
2387 // No appropriate candidate to steal found
2388 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2389 KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #4): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2391)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2390 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #4): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2391)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2391 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #4): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2391)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2392 victim_td->td.td_deque_head, victim_td->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #4): T#%d could not steal from "
"T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", gtid, ((((
victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2391)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
;
2393 return NULL__null;
2394 }
2395 int prev = target;
2396 for (i = i + 1; i < ntasks; ++i) {
2397 // shift remaining tasks in the deque left by 1
2398 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td)((victim_td->td).td_deque_size - 1);
2399 victim_td->td.td_deque[prev] = victim_td->td.td_deque[target];
2400 prev = target;
2401 }
2402 KMP_DEBUG_ASSERT(victim_td->td.td_deque_tail ==((victim_td->td.td_deque_tail == ((target + 1) & ((victim_td
->td).td_deque_size - 1))) ? 0 : __kmp_debug_assert("victim_td->td.td_deque_tail == ((target + 1) & ((victim_td->td).td_deque_size - 1))"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2403))
2403 ((target + 1) & TASK_DEQUE_MASK(victim_td->td)))((victim_td->td.td_deque_tail == ((target + 1) & ((victim_td
->td).td_deque_size - 1))) ? 0 : __kmp_debug_assert("victim_td->td.td_deque_tail == ((target + 1) & ((victim_td->td).td_deque_size - 1))"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2403))
;
2404 victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped))
2405 }
2406 if (*thread_finished) {
2407 // We need to un-mark this victim as a finished victim. This must be done
2408 // before releasing the lock, or else other threads (starting with the
2409 // master victim) might be prematurely released from the barrier!!!
2410 kmp_int32 count;
2411
2412 count = KMP_ATOMIC_INC(unfinished_threads)(unfinished_threads)->fetch_add(1, std::memory_order_acq_rel
)
;
2413
2414 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n"
, gtid, count + 1, task_team); }
2415 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n"
, gtid, count + 1, task_team); }
2416 ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n"
, gtid, count + 1, task_team); }
2417 gtid, count + 1, task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n"
, gtid, count + 1, task_team); }
;
2418
2419 *thread_finished = FALSE0;
2420 }
2421 TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1)(victim_td->td.td_deque_ntasks) = (ntasks - 1);
2422
2423 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2424
2425 KMP_COUNT_BLOCK(TASK_stolen)((void)0);
2426 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, (
(((victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2429)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2427 ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, (
(((victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2429)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2428 "task_team=%p ntasks=%d head=%u tail=%u\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, (
(((victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2429)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2429 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, (
(((victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2429)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
2430 ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
"task_team=%p ntasks=%d head=%u tail=%u\n", gtid, taskdata, (
(((victim_thr) != __null) ? 0 : __kmp_debug_assert("(victim_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2429)), (victim_thr)->th.th_info.ds.ds_gtid), task_team,
ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail
); }
;
2431
2432 task = KMP_TASKDATA_TO_TASK(taskdata)(kmp_task_t *)(taskdata + 1);
2433 return task;
2434}
2435
2436// __kmp_execute_tasks_template: Choose and execute tasks until either the
2437// condition is statisfied (return true) or there are none left (return false).
2438//
2439// final_spin is TRUE if this is the spin at the release barrier.
2440// thread_finished indicates whether the thread is finished executing all
2441// the tasks it has on its deque, and is at the release barrier.
2442// spinner is the location on which to spin.
2443// spinner == NULL means only execute a single task and return.
2444// checker is the value to check to terminate the spin.
2445template <class C>
2446static inline int __kmp_execute_tasks_template(
2447 kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin,
2448 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), void *itt_sync_obj,
2449 kmp_int32 is_constrained) {
2450 kmp_task_team_t *task_team = thread->th.th_task_team;
2451 kmp_thread_data_t *threads_data;
2452 kmp_task_t *task;
2453 kmp_info_t *other_thread;
2454 kmp_taskdata_t *current_task = thread->th.th_current_task;
2455 std::atomic<kmp_int32> *unfinished_threads;
2456 kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0,
2457 tid = thread->th.th_info.ds.ds_tid;
2458
2459 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec)((__kmp_tasking_mode != tskm_immediate_exec) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode != tskm_immediate_exec", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2459))
;
2460 KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid])((thread == __kmp_threads[gtid]) ? 0 : __kmp_debug_assert("thread == __kmp_threads[gtid]"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2460))
;
2461
2462 if (task_team == NULL__null)
2463 return FALSE0;
2464
2465 KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d "if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d "
"*thread_finished=%d\n", gtid, final_spin, *thread_finished)
; }
2466 "*thread_finished=%d\n",if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d "
"*thread_finished=%d\n", gtid, final_spin, *thread_finished)
; }
2467 gtid, final_spin, *thread_finished))if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d "
"*thread_finished=%d\n", gtid, final_spin, *thread_finished)
; }
;
2468
2469 thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP0;
2470 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data)((void *)(task_team->tt.tt_threads_data));
2471 KMP_DEBUG_ASSERT(threads_data != NULL)((threads_data != __null) ? 0 : __kmp_debug_assert("threads_data != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2471))
;
2472
2473 nthreads = task_team->tt.tt_nproc;
2474 unfinished_threads = &(task_team->tt.tt_unfinished_threads);
2475#if OMP_45_ENABLED(50 >= 45)
2476 KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks)((nthreads > 1 || task_team->tt.tt_found_proxy_tasks) ?
0 : __kmp_debug_assert("nthreads > 1 || task_team->tt.tt_found_proxy_tasks"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2476))
;
2477#else
2478 KMP_DEBUG_ASSERT(nthreads > 1)((nthreads > 1) ? 0 : __kmp_debug_assert("nthreads > 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2478))
;
2479#endif
2480 KMP_DEBUG_ASSERT(*unfinished_threads >= 0)((*unfinished_threads >= 0) ? 0 : __kmp_debug_assert("*unfinished_threads >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2480))
;
2481
2482 while (1) { // Outer loop keeps trying to find tasks in case of single thread
2483 // getting tasks from target constructs
2484 while (1) { // Inner loop to find a task and execute it
2485 task = NULL__null;
2486 if (use_own_tasks) { // check on own queue first
2487 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained);
2488 }
2489 if ((task == NULL__null) && (nthreads > 1)) { // Steal a task
2490 int asleep = 1;
2491 use_own_tasks = 0;
2492 // Try to steal from the last place I stole from successfully.
2493 if (victim_tid == -2) { // haven't stolen anything yet
2494 victim_tid = threads_data[tid].td.td_deque_last_stolen;
2495 if (victim_tid !=
2496 -1) // if we have a last stolen from victim, get the thread
2497 other_thread = threads_data[victim_tid].td.td_thr;
2498 }
2499 if (victim_tid != -1) { // found last victim
2500 asleep = 0;
2501 } else if (!new_victim) { // no recent steals and we haven't already
2502 // used a new victim; select a random thread
2503 do { // Find a different thread to steal work from.
2504 // Pick a random thread. Initial plan was to cycle through all the
2505 // threads, and only return if we tried to steal from every thread,
2506 // and failed. Arch says that's not such a great idea.
2507 victim_tid = __kmp_get_random(thread) % (nthreads - 1);
2508 if (victim_tid >= tid) {
2509 ++victim_tid; // Adjusts random distribution to exclude self
2510 }
2511 // Found a potential victim
2512 other_thread = threads_data[victim_tid].td.td_thr;
2513 // There is a slight chance that __kmp_enable_tasking() did not wake
2514 // up all threads waiting at the barrier. If victim is sleeping,
2515 // then wake it up. Since we were going to pay the cache miss
2516 // penalty for referencing another thread's kmp_info_t struct
2517 // anyway,
2518 // the check shouldn't cost too much performance at this point. In
2519 // extra barrier mode, tasks do not sleep at the separate tasking
2520 // barrier, so this isn't a problem.
2521 asleep = 0;
2522 if ((__kmp_tasking_mode == tskm_task_teams) &&
2523 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647)) &&
2524 (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc))((void *)(const_cast<void *>(other_thread->th.th_sleep_loc
)))
!=
2525 NULL__null)) {
2526 asleep = 1;
2527 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(other_thread)((((other_thread) != __null) ? 0 : __kmp_debug_assert("(other_thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2527)), (other_thread)->th.th_info.ds.ds_gtid)
,
2528 other_thread->th.th_sleep_loc);
2529 // A sleeping thread should not have any tasks on it's queue.
2530 // There is a slight possibility that it resumes, steals a task
2531 // from another thread, which spawns more tasks, all in the time
2532 // that it takes this thread to check => don't write an assertion
2533 // that the victim's queue is empty. Try stealing from a
2534 // different thread.
2535 }
2536 } while (asleep);
2537 }
2538
2539 if (!asleep) {
2540 // We have a victim to try to steal from
2541 task = __kmp_steal_task(other_thread, gtid, task_team,
2542 unfinished_threads, thread_finished,
2543 is_constrained);
2544 }
2545 if (task != NULL__null) { // set last stolen to victim
2546 if (threads_data[tid].td.td_deque_last_stolen != victim_tid) {
2547 threads_data[tid].td.td_deque_last_stolen = victim_tid;
2548 // The pre-refactored code did not try more than 1 successful new
2549 // vicitm, unless the last one generated more local tasks;
2550 // new_victim keeps track of this
2551 new_victim = 1;
2552 }
2553 } else { // No tasks found; unset last_stolen
2554 KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1)if ((threads_data[tid].td.td_deque_last_stolen) != (-1)) (threads_data
[tid].td.td_deque_last_stolen) = (-1)
;
2555 victim_tid = -2; // no successful victim found
2556 }
2557 }
2558
2559 if (task == NULL__null) // break out of tasking loop
2560 break;
2561
2562// Found a task; execute it
2563#if USE_ITT_BUILD1 && USE_ITT_NOTIFY1
2564 if (__itt_sync_create_ptr__kmp_itt_sync_create_ptr__3_0 || KMP_ITT_DEBUG0) {
2565 if (itt_sync_obj == NULL__null) { // we are at fork barrier where we could not
2566 // get the object reliably
2567 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
2568 }
2569 __kmp_itt_task_starting(itt_sync_obj);
2570 }
2571#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
2572 __kmp_invoke_task(gtid, task, current_task);
2573#if USE_ITT_BUILD1
2574 if (itt_sync_obj != NULL__null)
2575 __kmp_itt_task_finished(itt_sync_obj);
2576#endif /* USE_ITT_BUILD */
2577 // If this thread is only partway through the barrier and the condition is
2578 // met, then return now, so that the barrier gather/release pattern can
2579 // proceed. If this thread is in the last spin loop in the barrier,
2580 // waiting to be released, we know that the termination condition will not
2581 // be satisified, so don't waste any cycles checking it.
2582 if (flag == NULL__null || (!final_spin && flag->done_check())) {
2583 KA_TRACE(if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
2584 15,if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
2585 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
2586 gtid))if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
;
2587 return TRUE(!0);
2588 }
2589 if (thread->th.th_task_team == NULL__null) {
2590 break;
2591 }
2592 // Yield before executing next task
2593 KMP_YIELD(__kmp_library == library_throughput){ __kmp_x86_pause(); __kmp_yield((__kmp_library == library_throughput
)); }
;
2594 // If execution of a stolen task results in more tasks being placed on our
2595 // run queue, reset use_own_tasks
2596 if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks)(threads_data[tid].td.td_deque_ntasks) != 0) {
2597 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d stolen task spawned "
"other tasks, restart\n", gtid); }
2598 "other tasks, restart\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d stolen task spawned "
"other tasks, restart\n", gtid); }
2599 gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d stolen task spawned "
"other tasks, restart\n", gtid); }
;
2600 use_own_tasks = 1;
2601 new_victim = 0;
2602 }
2603 }
2604
2605// The task source has been exhausted. If in final spin loop of barrier, check
2606// if termination condition is satisfied.
2607#if OMP_45_ENABLED(50 >= 45)
2608 // The work queue may be empty but there might be proxy tasks still
2609 // executing
2610 if (final_spin &&
2611 KMP_ATOMIC_LD_ACQ(&current_task->td_incomplete_child_tasks)(&current_task->td_incomplete_child_tasks)->load(std
::memory_order_acquire)
== 0)
2612#else
2613 if (final_spin)
2614#endif
2615 {
2616 // First, decrement the #unfinished threads, if that has not already been
2617 // done. This decrement might be to the spin location, and result in the
2618 // termination condition being satisfied.
2619 if (!*thread_finished) {
2620 kmp_int32 count;
2621
2622 count = KMP_ATOMIC_DEC(unfinished_threads)(unfinished_threads)->fetch_sub(1, std::memory_order_acq_rel
)
- 1;
2623 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d dec "
"unfinished_threads to %d task_team=%p\n", gtid, count, task_team
); }
2624 "unfinished_threads to %d task_team=%p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d dec "
"unfinished_threads to %d task_team=%p\n", gtid, count, task_team
); }
2625 gtid, count, task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d dec "
"unfinished_threads to %d task_team=%p\n", gtid, count, task_team
); }
;
2626 *thread_finished = TRUE(!0);
2627 }
2628
2629 // It is now unsafe to reference thread->th.th_team !!!
2630 // Decrementing task_team->tt.tt_unfinished_threads can allow the master
2631 // thread to pass through the barrier, where it might reset each thread's
2632 // th.th_team field for the next parallel region. If we can steal more
2633 // work, we know that this has not happened yet.
2634 if (flag != NULL__null && flag->done_check()) {
2635 KA_TRACE(if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
2636 15,if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
2637 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
2638 gtid))if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n"
, gtid); }
;
2639 return TRUE(!0);
2640 }
2641 }
2642
2643 // If this thread's task team is NULL, master has recognized that there are
2644 // no more tasks; bail out
2645 if (thread->th.th_task_team == NULL__null) {
2646 KA_TRACE(15,if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d no more tasks\n"
, gtid); }
2647 ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid))if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d no more tasks\n"
, gtid); }
;
2648 return FALSE0;
2649 }
2650
2651#if OMP_45_ENABLED(50 >= 45)
2652 // We could be getting tasks from target constructs; if this is the only
2653 // thread, keep trying to execute tasks from own queue
2654 if (nthreads == 1)
2655 use_own_tasks = 1;
2656 else
2657#endif
2658 {
2659 KA_TRACE(15,if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d can't find work\n"
, gtid); }
2660 ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid))if (kmp_a_debug >= 15) { __kmp_debug_printf ("__kmp_execute_tasks_template: T#%d can't find work\n"
, gtid); }
;
2661 return FALSE0;
2662 }
2663 }
2664}
2665
2666int __kmp_execute_tasks_32(
2667 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin,
2668 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), void *itt_sync_obj,
2669 kmp_int32 is_constrained) {
2670 return __kmp_execute_tasks_template(
2671 thread, gtid, flag, final_spin,
2672 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), itt_sync_obj, is_constrained);
2673}
2674
2675int __kmp_execute_tasks_64(
2676 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag, int final_spin,
2677 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), void *itt_sync_obj,
2678 kmp_int32 is_constrained) {
2679 return __kmp_execute_tasks_template(
2680 thread, gtid, flag, final_spin,
2681 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), itt_sync_obj, is_constrained);
2682}
2683
2684int __kmp_execute_tasks_oncore(
2685 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin,
2686 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), void *itt_sync_obj,
2687 kmp_int32 is_constrained) {
2688 return __kmp_execute_tasks_template(
2689 thread, gtid, flag, final_spin,
2690 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), itt_sync_obj, is_constrained);
2691}
2692
2693// __kmp_enable_tasking: Allocate task team and resume threads sleeping at the
2694// next barrier so they can assist in executing enqueued tasks.
2695// First thread in allocates the task team atomically.
2696static void __kmp_enable_tasking(kmp_task_team_t *task_team,
2697 kmp_info_t *this_thr) {
2698 kmp_thread_data_t *threads_data;
2699 int nthreads, i, is_init_thread;
2700
2701 KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_enable_tasking(enter): T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2702)), (this_thr)->th.th_info.ds.ds_gtid)); }
2702 __kmp_gtid_from_thread(this_thr)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_enable_tasking(enter): T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2702)), (this_thr)->th.th_info.ds.ds_gtid)); }
;
2703
2704 KMP_DEBUG_ASSERT(task_team != NULL)((task_team != __null) ? 0 : __kmp_debug_assert("task_team != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2704))
;
2705 KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL)((this_thr->th.th_team != __null) ? 0 : __kmp_debug_assert
("this_thr->th.th_team != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2705))
;
2706
2707 nthreads = task_team->tt.tt_nproc;
2708 KMP_DEBUG_ASSERT(nthreads > 0)((nthreads > 0) ? 0 : __kmp_debug_assert("nthreads > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2708))
;
2709 KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc)((nthreads == this_thr->th.th_team->t.t_nproc) ? 0 : __kmp_debug_assert
("nthreads == this_thr->th.th_team->t.t_nproc", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2709))
;
2710
2711 // Allocate or increase the size of threads_data if necessary
2712 is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team);
2713
2714 if (!is_init_thread) {
2715 // Some other thread already set up the array.
2716 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2719)), (this_thr)->th.th_info.ds.ds_gtid)); }
2717 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2719)), (this_thr)->th.th_info.ds.ds_gtid)); }
2718 ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2719)), (this_thr)->th.th_info.ds.ds_gtid)); }
2719 __kmp_gtid_from_thread(this_thr)))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2719)), (this_thr)->th.th_info.ds.ds_gtid)); }
;
2720 return;
2721 }
2722 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data)((void *)(task_team->tt.tt_threads_data));
2723 KMP_DEBUG_ASSERT(threads_data != NULL)((threads_data != __null) ? 0 : __kmp_debug_assert("threads_data != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2723))
;
2724
2725 if ((__kmp_tasking_mode == tskm_task_teams) &&
2726 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647))) {
2727 // Release any threads sleeping at the barrier, so that they can steal
2728 // tasks and execute them. In extra barrier mode, tasks do not sleep
2729 // at the separate tasking barrier, so this isn't a problem.
2730 for (i = 0; i < nthreads; i++) {
2731 volatile void *sleep_loc;
2732 kmp_info_t *thread = threads_data[i].td.td_thr;
2733
2734 if (i == this_thr->th.th_info.ds.ds_tid) {
2735 continue;
2736 }
2737 // Since we haven't locked the thread's suspend mutex lock at this
2738 // point, there is a small window where a thread might be putting
2739 // itself to sleep, but hasn't set the th_sleep_loc field yet.
2740 // To work around this, __kmp_execute_tasks_template() periodically checks
2741 // see if other threads are sleeping (using the same random mechanism that
2742 // is used for task stealing) and awakens them if they are.
2743 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))((void *)(const_cast<void *>(thread->th.th_sleep_loc
)))
) !=
2744 NULL__null) {
2745 KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n",if (kmp_f_debug >= 50) { __kmp_debug_printf ("__kmp_enable_tasking: T#%d waking up thread T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2746)), (this_thr)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2747)), (thread)->th.th_info.ds.ds_gtid)); }
2746 __kmp_gtid_from_thread(this_thr),if (kmp_f_debug >= 50) { __kmp_debug_printf ("__kmp_enable_tasking: T#%d waking up thread T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2746)), (this_thr)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2747)), (thread)->th.th_info.ds.ds_gtid)); }
2747 __kmp_gtid_from_thread(thread)))if (kmp_f_debug >= 50) { __kmp_debug_printf ("__kmp_enable_tasking: T#%d waking up thread T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2746)), (this_thr)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2747)), (thread)->th.th_info.ds.ds_gtid)); }
;
2748 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread)((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2748)), (thread)->th.th_info.ds.ds_gtid)
, sleep_loc);
2749 } else {
2750 KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",if (kmp_f_debug >= 50) { __kmp_debug_printf ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2751)), (this_thr)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2752)), (thread)->th.th_info.ds.ds_gtid)); }
2751 __kmp_gtid_from_thread(this_thr),if (kmp_f_debug >= 50) { __kmp_debug_printf ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2751)), (this_thr)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2752)), (thread)->th.th_info.ds.ds_gtid)); }
2752 __kmp_gtid_from_thread(thread)))if (kmp_f_debug >= 50) { __kmp_debug_printf ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2751)), (this_thr)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2752)), (thread)->th.th_info.ds.ds_gtid)); }
;
2753 }
2754 }
2755 }
2756
2757 KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_enable_tasking(exit): T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2758)), (this_thr)->th.th_info.ds.ds_gtid)); }
2758 __kmp_gtid_from_thread(this_thr)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_enable_tasking(exit): T#%d\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2758)), (this_thr)->th.th_info.ds.ds_gtid)); }
;
2759}
2760
2761/* // TODO: Check the comment consistency
2762 * Utility routines for "task teams". A task team (kmp_task_t) is kind of
2763 * like a shadow of the kmp_team_t data struct, with a different lifetime.
2764 * After a child * thread checks into a barrier and calls __kmp_release() from
2765 * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no
2766 * longer assume that the kmp_team_t structure is intact (at any moment, the
2767 * master thread may exit the barrier code and free the team data structure,
2768 * and return the threads to the thread pool).
2769 *
2770 * This does not work with the the tasking code, as the thread is still
2771 * expected to participate in the execution of any tasks that may have been
2772 * spawned my a member of the team, and the thread still needs access to all
2773 * to each thread in the team, so that it can steal work from it.
2774 *
2775 * Enter the existence of the kmp_task_team_t struct. It employs a reference
2776 * counting mechanims, and is allocated by the master thread before calling
2777 * __kmp_<barrier_kind>_release, and then is release by the last thread to
2778 * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes
2779 * of the kmp_task_team_t structs for consecutive barriers can overlap
2780 * (and will, unless the master thread is the last thread to exit the barrier
2781 * release phase, which is not typical).
2782 *
2783 * The existence of such a struct is useful outside the context of tasking,
2784 * but for now, I'm trying to keep it specific to the OMP_30_ENABLED macro,
2785 * so that any performance differences show up when comparing the 2.5 vs. 3.0
2786 * libraries.
2787 *
2788 * We currently use the existence of the threads array as an indicator that
2789 * tasks were spawned since the last barrier. If the structure is to be
2790 * useful outside the context of tasking, then this will have to change, but
2791 * not settting the field minimizes the performance impact of tasking on
2792 * barriers, when no explicit tasks were spawned (pushed, actually).
2793 */
2794
2795static kmp_task_team_t *__kmp_free_task_teams =
2796 NULL__null; // Free list for task_team data structures
2797// Lock for task team data structures
2798kmp_bootstrap_lock_t __kmp_task_team_lock =
2799 KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock){ { { true } , &((__kmp_task_team_lock)), __null, { 0U },
{ 0U }, { 0 }, { -1 } } }
;
2800
2801// __kmp_alloc_task_deque:
2802// Allocates a task deque for a particular thread, and initialize the necessary
2803// data structures relating to the deque. This only happens once per thread
2804// per task team since task teams are recycled. No lock is needed during
2805// allocation since each thread allocates its own deque.
2806static void __kmp_alloc_task_deque(kmp_info_t *thread,
2807 kmp_thread_data_t *thread_data) {
2808 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock);
2809 KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL)((thread_data->td.td_deque == __null) ? 0 : __kmp_debug_assert
("thread_data->td.td_deque == __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2809))
;
2810
2811 // Initialize last stolen task field to "none"
2812 thread_data->td.td_deque_last_stolen = -1;
2813
2814 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0)(((thread_data->td.td_deque_ntasks) == 0) ? 0 : __kmp_debug_assert
("(thread_data->td.td_deque_ntasks) == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2814))
;
2815 KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0)((thread_data->td.td_deque_head == 0) ? 0 : __kmp_debug_assert
("thread_data->td.td_deque_head == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2815))
;
2816 KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0)((thread_data->td.td_deque_tail == 0) ? 0 : __kmp_debug_assert
("thread_data->td.td_deque_tail == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2816))
;
2817
2818 KE_TRACE(if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2821)), (thread)->th.th_info.ds.ds_gtid), (1 << 8)
, thread_data); }
2819 10,if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2821)), (thread)->th.th_info.ds.ds_gtid), (1 << 8)
, thread_data); }
2820 ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2821)), (thread)->th.th_info.ds.ds_gtid), (1 << 8)
, thread_data); }
2821 __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2821)), (thread)->th.th_info.ds.ds_gtid), (1 << 8)
, thread_data); }
;
2822 // Allocate space for task deque, and zero the deque
2823 // Cannot use __kmp_thread_calloc() because threads not around for
2824 // kmp_reap_task_team( ).
2825 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate(___kmp_allocate(((1 << 8) * sizeof(kmp_taskdata_t *)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2826)
2826 INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *))___kmp_allocate(((1 << 8) * sizeof(kmp_taskdata_t *)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2826)
;
2827 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE(1 << 8);
2828}
2829
2830// __kmp_realloc_task_deque:
2831// Re-allocates a task deque for a particular thread, copies the content from
2832// the old deque and adjusts the necessary data structures relating to the
2833// deque. This operation must be done with a the deque_lock being held
2834static void __kmp_realloc_task_deque(kmp_info_t *thread,
2835 kmp_thread_data_t *thread_data) {
2836 kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td)((thread_data->td).td_deque_size);
2837 kmp_int32 new_size = 2 * size;
2838
2839 KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "
"%d] for thread_data %p\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2841)), (thread)->th.th_info.ds.ds_gtid), size, new_size
, thread_data); }
2840 "%d] for thread_data %p\n",if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "
"%d] for thread_data %p\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2841)), (thread)->th.th_info.ds.ds_gtid), size, new_size
, thread_data); }
2841 __kmp_gtid_from_thread(thread), size, new_size, thread_data))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "
"%d] for thread_data %p\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2841)), (thread)->th.th_info.ds.ds_gtid), size, new_size
, thread_data); }
;
2842
2843 kmp_taskdata_t **new_deque =
2844 (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *))___kmp_allocate((new_size * sizeof(kmp_taskdata_t *)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2844)
;
2845
2846 int i, j;
2847 for (i = thread_data->td.td_deque_head, j = 0; j < size;
2848 i = (i + 1) & TASK_DEQUE_MASK(thread_data->td)((thread_data->td).td_deque_size - 1), j++)
2849 new_deque[j] = thread_data->td.td_deque[i];
2850
2851 __kmp_free(thread_data->td.td_deque)___kmp_free((thread_data->td.td_deque), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2851)
;
2852
2853 thread_data->td.td_deque_head = 0;
2854 thread_data->td.td_deque_tail = size;
2855 thread_data->td.td_deque = new_deque;
2856 thread_data->td.td_deque_size = new_size;
2857}
2858
2859// __kmp_free_task_deque:
2860// Deallocates a task deque for a particular thread. Happens at library
2861// deallocation so don't need to reset all thread data fields.
2862static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) {
2863 if (thread_data->td.td_deque != NULL__null) {
2864 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2865 TCW_4(thread_data->td.td_deque_ntasks, 0)(thread_data->td.td_deque_ntasks) = (0);
2866 __kmp_free(thread_data->td.td_deque)___kmp_free((thread_data->td.td_deque), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2866)
;
2867 thread_data->td.td_deque = NULL__null;
2868 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2869 }
2870
2871#ifdef BUILD_TIED_TASK_STACK
2872 // GEH: Figure out what to do here for td_susp_tied_tasks
2873 if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) {
2874 __kmp_free_task_stack(__kmp_thread_from_gtid(gtid)((((gtid) >= 0) ? 0 : __kmp_debug_assert("(gtid) >= 0",
"/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2874)), __kmp_threads[(gtid)])
, thread_data);
2875 }
2876#endif // BUILD_TIED_TASK_STACK
2877}
2878
2879// __kmp_realloc_task_threads_data:
2880// Allocates a threads_data array for a task team, either by allocating an
2881// initial array or enlarging an existing array. Only the first thread to get
2882// the lock allocs or enlarges the array and re-initializes the array eleemnts.
2883// That thread returns "TRUE", the rest return "FALSE".
2884// Assumes that the new array size is given by task_team -> tt.tt_nproc.
2885// The current size is given by task_team -> tt.tt_max_threads.
2886static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
2887 kmp_task_team_t *task_team) {
2888 kmp_thread_data_t **threads_data_p;
2889 kmp_int32 nthreads, maxthreads;
2890 int is_init_thread = FALSE0;
2891
2892 if (TCR_4(task_team->tt.tt_found_tasks)(task_team->tt.tt_found_tasks)) {
2893 // Already reallocated and initialized.
2894 return FALSE0;
2895 }
2896
2897 threads_data_p = &task_team->tt.tt_threads_data;
2898 nthreads = task_team->tt.tt_nproc;
2899 maxthreads = task_team->tt.tt_max_threads;
2900
2901 // All threads must lock when they encounter the first task of the implicit
2902 // task region to make sure threads_data fields are (re)initialized before
2903 // used.
2904 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
2905
2906 if (!TCR_4(task_team->tt.tt_found_tasks)(task_team->tt.tt_found_tasks)) {
2907 // first thread to enable tasking
2908 kmp_team_t *team = thread->th.th_team;
2909 int i;
2910
2911 is_init_thread = TRUE(!0);
2912 if (maxthreads < nthreads) {
2913
2914 if (*threads_data_p != NULL__null) {
2915 kmp_thread_data_t *old_data = *threads_data_p;
2916 kmp_thread_data_t *new_data = NULL__null;
2917
2918 KE_TRACE(if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d reallocating "
"threads data for task_team %p, new_size = %d, old_size = %d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2922)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
, maxthreads); }
2919 10,if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d reallocating "
"threads data for task_team %p, new_size = %d, old_size = %d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2922)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
, maxthreads); }
2920 ("__kmp_realloc_task_threads_data: T#%d reallocating "if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d reallocating "
"threads data for task_team %p, new_size = %d, old_size = %d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2922)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
, maxthreads); }
2921 "threads data for task_team %p, new_size = %d, old_size = %d\n",if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d reallocating "
"threads data for task_team %p, new_size = %d, old_size = %d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2922)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
, maxthreads); }
2922 __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d reallocating "
"threads data for task_team %p, new_size = %d, old_size = %d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2922)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
, maxthreads); }
;
2923 // Reallocate threads_data to have more elements than current array
2924 // Cannot use __kmp_thread_realloc() because threads not around for
2925 // kmp_reap_task_team( ). Note all new array entries are initialized
2926 // to zero by __kmp_allocate().
2927 new_data = (kmp_thread_data_t *)__kmp_allocate(___kmp_allocate((nthreads * sizeof(kmp_thread_data_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2928)
2928 nthreads * sizeof(kmp_thread_data_t))___kmp_allocate((nthreads * sizeof(kmp_thread_data_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2928)
;
2929 // copy old data to new data
2930 KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t),memcpy((void *)new_data, (void *)old_data, maxthreads * sizeof
(kmp_thread_data_t))
2931 (void *)old_data, maxthreads * sizeof(kmp_thread_data_t))memcpy((void *)new_data, (void *)old_data, maxthreads * sizeof
(kmp_thread_data_t))
;
2932
2933#ifdef BUILD_TIED_TASK_STACK
2934 // GEH: Figure out if this is the right thing to do
2935 for (i = maxthreads; i < nthreads; i++) {
2936 kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
2937 __kmp_init_task_stack(__kmp_gtid_from_thread(thread)((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2937)), (thread)->th.th_info.ds.ds_gtid)
, thread_data);
2938 }
2939#endif // BUILD_TIED_TASK_STACK
2940 // Install the new data and free the old data
2941 (*threads_data_p) = new_data;
2942 __kmp_free(old_data)___kmp_free((old_data), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2942)
;
2943 } else {
2944 KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating "if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d allocating "
"threads data for task_team %p, size = %d\n", ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2946)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
); }
2945 "threads data for task_team %p, size = %d\n",if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d allocating "
"threads data for task_team %p, size = %d\n", ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2946)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
); }
2946 __kmp_gtid_from_thread(thread), task_team, nthreads))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_realloc_task_threads_data: T#%d allocating "
"threads data for task_team %p, size = %d\n", ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2946)), (thread)->th.th_info.ds.ds_gtid), task_team, nthreads
); }
;
2947 // Make the initial allocate for threads_data array, and zero entries
2948 // Cannot use __kmp_thread_calloc() because threads not around for
2949 // kmp_reap_task_team( ).
2950 ANNOTATE_IGNORE_WRITES_BEGIN();
2951 *threads_data_p = (kmp_thread_data_t *)__kmp_allocate(___kmp_allocate((nthreads * sizeof(kmp_thread_data_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2952)
2952 nthreads * sizeof(kmp_thread_data_t))___kmp_allocate((nthreads * sizeof(kmp_thread_data_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2952)
;
2953 ANNOTATE_IGNORE_WRITES_END();
2954#ifdef BUILD_TIED_TASK_STACK
2955 // GEH: Figure out if this is the right thing to do
2956 for (i = 0; i < nthreads; i++) {
2957 kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
2958 __kmp_init_task_stack(__kmp_gtid_from_thread(thread)((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2958)), (thread)->th.th_info.ds.ds_gtid)
, thread_data);
2959 }
2960#endif // BUILD_TIED_TASK_STACK
2961 }
2962 task_team->tt.tt_max_threads = nthreads;
2963 } else {
2964 // If array has (more than) enough elements, go ahead and use it
2965 KMP_DEBUG_ASSERT(*threads_data_p != NULL)((*threads_data_p != __null) ? 0 : __kmp_debug_assert("*threads_data_p != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2965))
;
2966 }
2967
2968 // initialize threads_data pointers back to thread_info structures
2969 for (i = 0; i < nthreads; i++) {
2970 kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
2971 thread_data->td.td_thr = team->t.t_threads[i];
2972
2973 if (thread_data->td.td_deque_last_stolen >= nthreads) {
2974 // The last stolen field survives across teams / barrier, and the number
2975 // of threads may have changed. It's possible (likely?) that a new
2976 // parallel region will exhibit the same behavior as previous region.
2977 thread_data->td.td_deque_last_stolen = -1;
2978 }
2979 }
2980
2981 KMP_MB();
2982 TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE)(task_team->tt.tt_found_tasks) = ((!0));
2983 }
2984
2985 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
2986 return is_init_thread;
2987}
2988
2989// __kmp_free_task_threads_data:
2990// Deallocates a threads_data array for a task team, including any attached
2991// tasking deques. Only occurs at library shutdown.
2992static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) {
2993 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
2994 if (task_team->tt.tt_threads_data != NULL__null) {
2995 int i;
2996 for (i = 0; i < task_team->tt.tt_max_threads; i++) {
2997 __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]);
2998 }
2999 __kmp_free(task_team->tt.tt_threads_data)___kmp_free((task_team->tt.tt_threads_data), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 2999)
;
3000 task_team->tt.tt_threads_data = NULL__null;
3001 }
3002 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
3003}
3004
3005// __kmp_allocate_task_team:
3006// Allocates a task team associated with a specific team, taking it from
3007// the global task team free list if possible. Also initializes data
3008// structures.
3009static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread,
3010 kmp_team_t *team) {
3011 kmp_task_team_t *task_team = NULL__null;
3012 int nthreads;
3013
3014 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d entering; team = %p\n"
, (thread ? ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3015)), (thread)->th.th_info.ds.ds_gtid) : -1), team); }
3015 (thread ? __kmp_gtid_from_thread(thread) : -1), team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d entering; team = %p\n"
, (thread ? ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3015)), (thread)->th.th_info.ds.ds_gtid) : -1), team); }
;
3016
3017 if (TCR_PTR(__kmp_free_task_teams)((void *)(__kmp_free_task_teams)) != NULL__null) {
3018 // Take a task team from the task team pool
3019 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3020 if (__kmp_free_task_teams != NULL__null) {
3021 task_team = __kmp_free_task_teams;
3022 TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next)((__kmp_free_task_teams)) = ((task_team->tt.tt_next));
3023 task_team->tt.tt_next = NULL__null;
3024 }
3025 __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3026 }
3027
3028 if (task_team == NULL__null) {
3029 KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating "if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d allocating "
"task team for team %p\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3031)), (thread)->th.th_info.ds.ds_gtid), team); }
3030 "task team for team %p\n",if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d allocating "
"task team for team %p\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3031)), (thread)->th.th_info.ds.ds_gtid), team); }
3031 __kmp_gtid_from_thread(thread), team))if (kmp_e_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d allocating "
"task team for team %p\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3031)), (thread)->th.th_info.ds.ds_gtid), team); }
;
3032 // Allocate a new task team if one is not available.
3033 // Cannot use __kmp_thread_malloc() because threads not around for
3034 // kmp_reap_task_team( ).
3035 task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t))___kmp_allocate((sizeof(kmp_task_team_t)), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3035)
;
3036 __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock);
3037 // AC: __kmp_allocate zeroes returned memory
3038 // task_team -> tt.tt_threads_data = NULL;
3039 // task_team -> tt.tt_max_threads = 0;
3040 // task_team -> tt.tt_next = NULL;
3041 }
3042
3043 TCW_4(task_team->tt.tt_found_tasks, FALSE)(task_team->tt.tt_found_tasks) = (0);
3044#if OMP_45_ENABLED(50 >= 45)
3045 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE)(task_team->tt.tt_found_proxy_tasks) = (0);
3046#endif
3047 task_team->tt.tt_nproc = nthreads = team->t.t_nproc;
3048
3049 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads)(&task_team->tt.tt_unfinished_threads)->store(nthreads
, std::memory_order_release)
;
3050 TCW_4(task_team->tt.tt_active, TRUE)(task_team->tt.tt_active) = ((!0));
3051
3052 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
"unfinished_threads init'd to %d\n", (thread ? ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3054)), (thread)->th.th_info.ds.ds_gtid) : -1), task_team
, (&task_team->tt.tt_unfinished_threads)->load(std::
memory_order_relaxed)); }
3053 "unfinished_threads init'd to %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
"unfinished_threads init'd to %d\n", (thread ? ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3054)), (thread)->th.th_info.ds.ds_gtid) : -1), task_team
, (&task_team->tt.tt_unfinished_threads)->load(std::
memory_order_relaxed)); }
3054 (thread ? __kmp_gtid_from_thread(thread) : -1), task_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
"unfinished_threads init'd to %d\n", (thread ? ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3054)), (thread)->th.th_info.ds.ds_gtid) : -1), task_team
, (&task_team->tt.tt_unfinished_threads)->load(std::
memory_order_relaxed)); }
3055 KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads)))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
"unfinished_threads init'd to %d\n", (thread ? ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3054)), (thread)->th.th_info.ds.ds_gtid) : -1), task_team
, (&task_team->tt.tt_unfinished_threads)->load(std::
memory_order_relaxed)); }
;
3056 return task_team;
3057}
3058
3059// __kmp_free_task_team:
3060// Frees the task team associated with a specific thread, and adds it
3061// to the global task team free list.
3062void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) {
3063 KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_team: T#%d task_team = %p\n"
, thread ? ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3064)), (thread)->th.th_info.ds.ds_gtid) : -1, task_team
); }
3064 thread ? __kmp_gtid_from_thread(thread) : -1, task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_task_team: T#%d task_team = %p\n"
, thread ? ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3064)), (thread)->th.th_info.ds.ds_gtid) : -1, task_team
); }
;
3065
3066 // Put task team back on free list
3067 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3068
3069 KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL)((task_team->tt.tt_next == __null) ? 0 : __kmp_debug_assert
("task_team->tt.tt_next == __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3069))
;
3070 task_team->tt.tt_next = __kmp_free_task_teams;
3071 TCW_PTR(__kmp_free_task_teams, task_team)((__kmp_free_task_teams)) = ((task_team));
3072
3073 __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3074}
3075
3076// __kmp_reap_task_teams:
3077// Free all the task teams on the task team free list.
3078// Should only be done during library shutdown.
3079// Cannot do anything that needs a thread structure or gtid since they are
3080// already gone.
3081void __kmp_reap_task_teams(void) {
3082 kmp_task_team_t *task_team;
3083
3084 if (TCR_PTR(__kmp_free_task_teams)((void *)(__kmp_free_task_teams)) != NULL__null) {
3085 // Free all task_teams on the free list
3086 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3087 while ((task_team = __kmp_free_task_teams) != NULL__null) {
3088 __kmp_free_task_teams = task_team->tt.tt_next;
3089 task_team->tt.tt_next = NULL__null;
3090
3091 // Free threads_data if necessary
3092 if (task_team->tt.tt_threads_data != NULL__null) {
3093 __kmp_free_task_threads_data(task_team);
3094 }
3095 __kmp_free(task_team)___kmp_free((task_team), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3095)
;
3096 }
3097 __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3098 }
3099}
3100
3101// __kmp_wait_to_unref_task_teams:
3102// Some threads could still be in the fork barrier release code, possibly
3103// trying to steal tasks. Wait for each thread to unreference its task team.
3104void __kmp_wait_to_unref_task_teams(void) {
3105 kmp_info_t *thread;
3106 kmp_uint32 spins;
3107 int done;
3108
3109 KMP_INIT_YIELD(spins){ (spins) = __kmp_yield_init; };
3110
3111 for (;;) {
3112 done = TRUE(!0);
3113
3114 // TODO: GEH - this may be is wrong because some sync would be necessary
3115 // in case threads are added to the pool during the traversal. Need to
3116 // verify that lock for thread pool is held when calling this routine.
3117 for (thread = CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool); thread != NULL__null;
3118 thread = thread->th.th_next_pool) {
3119#if KMP_OS_WINDOWS0
3120 DWORD exit_val;
3121#endif
3122 if (TCR_PTR(thread->th.th_task_team)((void *)(thread->th.th_task_team)) == NULL__null) {
3123 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3124)), (thread)->th.th_info.ds.ds_gtid)); }
3124 __kmp_gtid_from_thread(thread)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3124)), (thread)->th.th_info.ds.ds_gtid)); }
;
3125 continue;
3126 }
3127#if KMP_OS_WINDOWS0
3128 // TODO: GEH - add this check for Linux* OS / OS X* as well?
3129 if (!__kmp_is_thread_alive(thread, &exit_val)) {
3130 thread->th.th_task_team = NULL__null;
3131 continue;
3132 }
3133#endif
3134
3135 done = FALSE0; // Because th_task_team pointer is not NULL for this thread
3136
3137 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: Waiting for T#%d to "
"unreference task_team\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3139)), (thread)->th.th_info.ds.ds_gtid)); }
3138 "unreference task_team\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: Waiting for T#%d to "
"unreference task_team\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3139)), (thread)->th.th_info.ds.ds_gtid)); }
3139 __kmp_gtid_from_thread(thread)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: Waiting for T#%d to "
"unreference task_team\n", ((((thread) != __null) ? 0 : __kmp_debug_assert
("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3139)), (thread)->th.th_info.ds.ds_gtid)); }
;
3140
3141 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647)) {
3142 volatile void *sleep_loc;
3143 // If the thread is sleeping, awaken it.
3144 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))((void *)(const_cast<void *>(thread->th.th_sleep_loc
)))
) !=
3145 NULL__null) {
3146 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid)); }
3147 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid)); }
3148 ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid)); }
3149 __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread)))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n"
, ((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid), ((((thread) !=
__null) ? 0 : __kmp_debug_assert("(thread) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3149)), (thread)->th.th_info.ds.ds_gtid)); }
;
3150 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread)((((thread) != __null) ? 0 : __kmp_debug_assert("(thread) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3150)), (thread)->th.th_info.ds.ds_gtid)
, sleep_loc);
3151 }
3152 }
3153 }
3154 if (done) {
3155 break;
3156 }
3157
3158 // If we are oversubscribed, or have waited a bit (and library mode is
3159 // throughput), yield. Pause is in the following code.
3160 KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc){ __kmp_x86_pause(); __kmp_yield(((__kmp_nth) > __kmp_avail_proc
)); }
;
3161 KMP_YIELD_SPIN(spins){ __kmp_x86_pause(); (spins) -= 2; if (!(spins)) { __kmp_yield
(1); (spins) = __kmp_yield_next; } }
; // Yields only if KMP_LIBRARY=throughput
3162 }
3163}
3164
3165// __kmp_task_team_setup: Create a task_team for the current team, but use
3166// an already created, unused one if it already exists.
3167void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
3168 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec)((__kmp_tasking_mode != tskm_immediate_exec) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode != tskm_immediate_exec", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3168))
;
3169
3170 // If this task_team hasn't been created yet, allocate it. It will be used in
3171 // the region after the next.
3172 // If it exists, it is the current task team and shouldn't be touched yet as
3173 // it may still be in use.
3174 if (team->t.t_task_team[this_thr->th.th_task_state] == NULL__null &&
3175 (always || team->t.t_nproc > 1)) {
3176 team->t.t_task_team[this_thr->th.th_task_state] =
3177 __kmp_allocate_task_team(this_thr, team);
3178 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created new task_team %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created new task_team %p "
"for team %d at parity=%d\n", ((((this_thr) != __null) ? 0 :
__kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3180)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[this_thr->th.th_task_state], ((team != __null) ? team->
t.t_id : -1), this_thr->th.th_task_state); }
3179 "for team %d at parity=%d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created new task_team %p "
"for team %d at parity=%d\n", ((((this_thr) != __null) ? 0 :
__kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3180)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[this_thr->th.th_task_state], ((team != __null) ? team->
t.t_id : -1), this_thr->th.th_task_state); }
3180 __kmp_gtid_from_thread(this_thr),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created new task_team %p "
"for team %d at parity=%d\n", ((((this_thr) != __null) ? 0 :
__kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3180)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[this_thr->th.th_task_state], ((team != __null) ? team->
t.t_id : -1), this_thr->th.th_task_state); }
3181 team->t.t_task_team[this_thr->th.th_task_state],if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created new task_team %p "
"for team %d at parity=%d\n", ((((this_thr) != __null) ? 0 :
__kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3180)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[this_thr->th.th_task_state], ((team != __null) ? team->
t.t_id : -1), this_thr->th.th_task_state); }
3182 ((team != NULL) ? team->t.t_id : -1),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created new task_team %p "
"for team %d at parity=%d\n", ((((this_thr) != __null) ? 0 :
__kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3180)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[this_thr->th.th_task_state], ((team != __null) ? team->
t.t_id : -1), this_thr->th.th_task_state); }
3183 this_thr->th.th_task_state))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created new task_team %p "
"for team %d at parity=%d\n", ((((this_thr) != __null) ? 0 :
__kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3180)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[this_thr->th.th_task_state], ((team != __null) ? team->
t.t_id : -1), this_thr->th.th_task_state); }
;
3184 }
3185
3186 // After threads exit the release, they will call sync, and then point to this
3187 // other task_team; make sure it is allocated and properly initialized. As
3188 // threads spin in the barrier release phase, they will continue to use the
3189 // previous task_team struct(above), until they receive the signal to stop
3190 // checking for tasks (they can't safely reference the kmp_team_t struct,
3191 // which could be reallocated by the master thread). No task teams are formed
3192 // for serialized teams.
3193 if (team->t.t_nproc > 1) {
3194 int other_team = 1 - this_thr->th.th_task_state;
3195 if (team->t.t_task_team[other_team] == NULL__null) { // setup other team as well
3196 team->t.t_task_team[other_team] =
3197 __kmp_allocate_task_team(this_thr, team);
3198 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created second new "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created second new "
"task_team %p for team %d at parity=%d\n", ((((this_thr) != __null
) ? 0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3200)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3199 "task_team %p for team %d at parity=%d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created second new "
"task_team %p for team %d at parity=%d\n", ((((this_thr) != __null
) ? 0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3200)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3200 __kmp_gtid_from_thread(this_thr),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created second new "
"task_team %p for team %d at parity=%d\n", ((((this_thr) != __null
) ? 0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3200)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3201 team->t.t_task_team[other_team],if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created second new "
"task_team %p for team %d at parity=%d\n", ((((this_thr) != __null
) ? 0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3200)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3202 ((team != NULL) ? team->t.t_id : -1), other_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d created second new "
"task_team %p for team %d at parity=%d\n", ((((this_thr) != __null
) ? 0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3200)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
;
3203 } else { // Leave the old task team struct in place for the upcoming region;
3204 // adjust as needed
3205 kmp_task_team_t *task_team = team->t.t_task_team[other_team];
3206 if (!task_team->tt.tt_active ||
3207 team->t.t_nproc != task_team->tt.tt_nproc) {
3208 TCW_4(task_team->tt.tt_nproc, team->t.t_nproc)(task_team->tt.tt_nproc) = (team->t.t_nproc);
3209 TCW_4(task_team->tt.tt_found_tasks, FALSE)(task_team->tt.tt_found_tasks) = (0);
3210#if OMP_45_ENABLED(50 >= 45)
3211 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE)(task_team->tt.tt_found_proxy_tasks) = (0);
3212#endif
3213 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads,(&task_team->tt.tt_unfinished_threads)->store(team->
t.t_nproc, std::memory_order_release)
3214 team->t.t_nproc)(&task_team->tt.tt_unfinished_threads)->store(team->
t.t_nproc, std::memory_order_release)
;
3215 TCW_4(task_team->tt.tt_active, TRUE)(task_team->tt.tt_active) = ((!0));
3216 }
3217 // if team size has changed, the first thread to enable tasking will
3218 // realloc threads_data if necessary
3219 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d reset next task_team "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d reset next task_team "
"%p for team %d at parity=%d\n", ((((this_thr) != __null) ? 0
: __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3221)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3220 "%p for team %d at parity=%d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d reset next task_team "
"%p for team %d at parity=%d\n", ((((this_thr) != __null) ? 0
: __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3221)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3221 __kmp_gtid_from_thread(this_thr),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d reset next task_team "
"%p for team %d at parity=%d\n", ((((this_thr) != __null) ? 0
: __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3221)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3222 team->t.t_task_team[other_team],if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d reset next task_team "
"%p for team %d at parity=%d\n", ((((this_thr) != __null) ? 0
: __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3221)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
3223 ((team != NULL) ? team->t.t_id : -1), other_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_setup: Master T#%d reset next task_team "
"%p for team %d at parity=%d\n", ((((this_thr) != __null) ? 0
: __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3221)), (this_thr)->th.th_info.ds.ds_gtid), team->t.t_task_team
[other_team], ((team != __null) ? team->t.t_id : -1), other_team
); }
;
3224 }
3225 }
3226}
3227
3228// __kmp_task_team_sync: Propagation of task team data from team to threads
3229// which happens just after the release phase of a team barrier. This may be
3230// called by any thread, but only for teams with # threads > 1.
3231void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
3232 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec)((__kmp_tasking_mode != tskm_immediate_exec) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode != tskm_immediate_exec", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3232))
;
3233
3234 // Toggle the th_task_state field, to switch which task_team this thread
3235 // refers to
3236 this_thr->th.th_task_state = 1 - this_thr->th.th_task_state;
3237 // It is now safe to propagate the task team pointer from the team struct to
3238 // the current thread.
3239 TCW_PTR(this_thr->th.th_task_team,((this_thr->th.th_task_team)) = ((team->t.t_task_team[this_thr
->th.th_task_state]))
3240 team->t.t_task_team[this_thr->th.th_task_state])((this_thr->th.th_task_team)) = ((team->t.t_task_team[this_thr
->th.th_task_state]))
;
3241 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
"%p from Team #%d (parity=%d)\n", ((((this_thr) != __null) ?
0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3244)), (this_thr)->th.th_info.ds.ds_gtid), this_thr->
th.th_task_team, ((team != __null) ? team->t.t_id : -1), this_thr
->th.th_task_state); }
3242 ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
"%p from Team #%d (parity=%d)\n", ((((this_thr) != __null) ?
0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3244)), (this_thr)->th.th_info.ds.ds_gtid), this_thr->
th.th_task_team, ((team != __null) ? team->t.t_id : -1), this_thr
->th.th_task_state); }
3243 "%p from Team #%d (parity=%d)\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
"%p from Team #%d (parity=%d)\n", ((((this_thr) != __null) ?
0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3244)), (this_thr)->th.th_info.ds.ds_gtid), this_thr->
th.th_task_team, ((team != __null) ? team->t.t_id : -1), this_thr
->th.th_task_state); }
3244 __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
"%p from Team #%d (parity=%d)\n", ((((this_thr) != __null) ?
0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3244)), (this_thr)->th.th_info.ds.ds_gtid), this_thr->
th.th_task_team, ((team != __null) ? team->t.t_id : -1), this_thr
->th.th_task_state); }
3245 ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
"%p from Team #%d (parity=%d)\n", ((((this_thr) != __null) ?
0 : __kmp_debug_assert("(this_thr) != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3244)), (this_thr)->th.th_info.ds.ds_gtid), this_thr->
th.th_task_team, ((team != __null) ? team->t.t_id : -1), this_thr
->th.th_task_state); }
;
3246}
3247
3248// __kmp_task_team_wait: Master thread waits for outstanding tasks after the
3249// barrier gather phase. Only called by master thread if #threads in team > 1 or
3250// if proxy tasks were created.
3251//
3252// wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off
3253// by passing in 0 optionally as the last argument. When wait is zero, master
3254// thread does not wait for unfinished_threads to reach 0.
3255void __kmp_task_team_wait(
3256 kmp_info_t *this_thr,
3257 kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), void *itt_sync_obj, int wait) {
3258 kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state];
3259
3260 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec)((__kmp_tasking_mode != tskm_immediate_exec) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode != tskm_immediate_exec", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3260))
;
3261 KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team)((task_team == this_thr->th.th_task_team) ? 0 : __kmp_debug_assert
("task_team == this_thr->th.th_task_team", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3261))
;
3262
3263 if ((task_team != NULL__null) && KMP_TASKING_ENABLED(task_team)(((task_team)->tt.tt_found_tasks) == (!0))) {
3264 if (wait) {
3265 KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d waiting for all tasks "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d waiting for all tasks "
"(for unfinished_threads to reach 0) on task_team = %p\n", (
(((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3267)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
3266 "(for unfinished_threads to reach 0) on task_team = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d waiting for all tasks "
"(for unfinished_threads to reach 0) on task_team = %p\n", (
(((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3267)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
3267 __kmp_gtid_from_thread(this_thr), task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d waiting for all tasks "
"(for unfinished_threads to reach 0) on task_team = %p\n", (
(((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3267)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
;
3268 // Worker threads may have dropped through to release phase, but could
3269 // still be executing tasks. Wait here for tasks to complete. To avoid
3270 // memory contention, only master thread checks termination condition.
3271 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *,reinterpret_cast<std::atomic<kmp_uint32> *>(&
task_team->tt.tt_unfinished_threads)
3272 &task_team->tt.tt_unfinished_threads)reinterpret_cast<std::atomic<kmp_uint32> *>(&
task_team->tt.tt_unfinished_threads)
,
3273 0U);
3274 flag.wait(this_thr, TRUE(!0) USE_ITT_BUILD_ARG(itt_sync_obj), itt_sync_obj);
3275 }
3276 // Deactivate the old task team, so that the worker threads will stop
3277 // referencing it while spinning.
3278 KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
"setting active to false, setting local and team's pointer to NULL\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3282)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
3279 20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
"setting active to false, setting local and team's pointer to NULL\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3282)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
3280 ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
"setting active to false, setting local and team's pointer to NULL\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3282)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
3281 "setting active to false, setting local and team's pointer to NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
"setting active to false, setting local and team's pointer to NULL\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3282)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
3282 __kmp_gtid_from_thread(this_thr), task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
"setting active to false, setting local and team's pointer to NULL\n"
, ((((this_thr) != __null) ? 0 : __kmp_debug_assert("(this_thr) != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3282)), (this_thr)->th.th_info.ds.ds_gtid), task_team); }
;
3283#if OMP_45_ENABLED(50 >= 45)
3284 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 ||((task_team->tt.tt_nproc > 1 || task_team->tt.tt_found_proxy_tasks
== (!0)) ? 0 : __kmp_debug_assert("task_team->tt.tt_nproc > 1 || task_team->tt.tt_found_proxy_tasks == (!0)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3285))
3285 task_team->tt.tt_found_proxy_tasks == TRUE)((task_team->tt.tt_nproc > 1 || task_team->tt.tt_found_proxy_tasks
== (!0)) ? 0 : __kmp_debug_assert("task_team->tt.tt_nproc > 1 || task_team->tt.tt_found_proxy_tasks == (!0)"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3285))
;
3286 TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE)(task_team->tt.tt_found_proxy_tasks) = (0);
3287#else
3288 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1)((task_team->tt.tt_nproc > 1) ? 0 : __kmp_debug_assert(
"task_team->tt.tt_nproc > 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3288))
;
3289#endif
3290 KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0)if ((task_team->tt.tt_untied_task_encountered) != (0)) (task_team
->tt.tt_untied_task_encountered) = (0)
;
3291 TCW_SYNC_4(task_team->tt.tt_active, FALSE)(task_team->tt.tt_active) = (0);
3292 KMP_MB();
3293
3294 TCW_PTR(this_thr->th.th_task_team, NULL)((this_thr->th.th_task_team)) = ((__null));
3295 }
3296}
3297
3298// __kmp_tasking_barrier:
3299// This routine may only called when __kmp_tasking_mode == tskm_extra_barrier.
3300// Internal function to execute all tasks prior to a regular barrier or a join
3301// barrier. It is a full barrier itself, which unfortunately turns regular
3302// barriers into double barriers and join barriers into 1 1/2 barriers.
3303void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) {
3304 std::atomic<kmp_uint32> *spin = RCAST(reinterpret_cast<std::atomic<kmp_uint32> *>(&
team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads
)
3305 std::atomic<kmp_uint32> *,reinterpret_cast<std::atomic<kmp_uint32> *>(&
team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads
)
3306 &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads)reinterpret_cast<std::atomic<kmp_uint32> *>(&
team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads
)
;
3307 int flag = FALSE0;
3308 KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier)((__kmp_tasking_mode == tskm_extra_barrier) ? 0 : __kmp_debug_assert
("__kmp_tasking_mode == tskm_extra_barrier", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3308))
;
3309
3310#if USE_ITT_BUILD1
3311 KMP_FSYNC_SPIN_INIT(spin, NULL)int sync_iters = 0; if (__kmp_itt_fsync_prepare_ptr__3_0) { if
(spin == __null) { spin = __null; } } __asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
::"i"(0x4376) : "%ebx")
;
3312#endif /* USE_ITT_BUILD */
3313 kmp_flag_32 spin_flag(spin, 0U);
3314 while (!spin_flag.execute_tasks(thread, gtid, TRUE(!0),
3315 &flag USE_ITT_BUILD_ARG(NULL), __null, 0)) {
3316#if USE_ITT_BUILD1
3317 // TODO: What about itt_sync_obj??
3318 KMP_FSYNC_SPIN_PREPARE(RCAST(void *, spin))do { if (__kmp_itt_fsync_prepare_ptr__3_0 && sync_iters
< __kmp_itt_prepare_delay) { ++sync_iters; if (sync_iters
>= __kmp_itt_prepare_delay) { (!__kmp_itt_fsync_prepare_ptr__3_0
) ? (void)0 : __kmp_itt_fsync_prepare_ptr__3_0((void *)((void
*)reinterpret_cast<void *>(spin))); } } } while (0)
;
3319#endif /* USE_ITT_BUILD */
3320
3321 if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
3322 if (__kmp_global.g.g_abort)
3323 __kmp_abort_thread();
3324 break;
3325 }
3326 KMP_YIELD(TRUE){ __kmp_x86_pause(); __kmp_yield(((!0))); }; // GH: We always yield here
3327 }
3328#if USE_ITT_BUILD1
3329 KMP_FSYNC_SPIN_ACQUIRED(RCAST(void *, spin))do { __asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
::"i"(0x4377) : "%ebx"); if (sync_iters >= __kmp_itt_prepare_delay
) { (!__kmp_itt_fsync_acquired_ptr__3_0) ? (void)0 : __kmp_itt_fsync_acquired_ptr__3_0
((void *)((void *)reinterpret_cast<void *>(spin))); } }
while (0)
;
3330#endif /* USE_ITT_BUILD */
3331}
3332
3333#if OMP_45_ENABLED(50 >= 45)
3334
3335// __kmp_give_task puts a task into a given thread queue if:
3336// - the queue for that thread was created
3337// - there's space in that queue
3338// Because of this, __kmp_push_task needs to check if there's space after
3339// getting the lock
3340static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task,
3341 kmp_int32 pass) {
3342 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
3343 kmp_task_team_t *task_team = taskdata->td_task_team;
3344
3345 KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_give_task: trying to give task %p to thread %d.\n"
, taskdata, tid); }
3346 taskdata, tid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_give_task: trying to give task %p to thread %d.\n"
, taskdata, tid); }
;
3347
3348 // If task_team is NULL something went really bad...
3349 KMP_DEBUG_ASSERT(task_team != NULL)((task_team != __null) ? 0 : __kmp_debug_assert("task_team != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3349))
;
3350
3351 bool result = false;
3352 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
3353
3354 if (thread_data->td.td_deque == NULL__null) {
3355 // There's no queue in this thread, go find another one
3356 // We're guaranteed that at least one thread has a queue
3357 KA_TRACE(30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: thread %d has no queue while giving task %p.\n"
, tid, taskdata); }
3358 ("__kmp_give_task: thread %d has no queue while giving task %p.\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: thread %d has no queue while giving task %p.\n"
, tid, taskdata); }
3359 tid, taskdata))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: thread %d has no queue while giving task %p.\n"
, tid, taskdata); }
;
3360 return result;
3361 }
3362
3363 if (TCR_4(thread_data->td.td_deque_ntasks)(thread_data->td.td_deque_ntasks) >=
3364 TASK_DEQUE_SIZE(thread_data->td)((thread_data->td).td_deque_size)) {
3365 KA_TRACE(if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: queue is full while giving task %p to thread %d.\n"
, taskdata, tid); }
3366 30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: queue is full while giving task %p to thread %d.\n"
, taskdata, tid); }
3367 ("__kmp_give_task: queue is full while giving task %p to thread %d.\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: queue is full while giving task %p to thread %d.\n"
, taskdata, tid); }
3368 taskdata, tid))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: queue is full while giving task %p to thread %d.\n"
, taskdata, tid); }
;
3369
3370 // if this deque is bigger than the pass ratio give a chance to another
3371 // thread
3372 if (TASK_DEQUE_SIZE(thread_data->td)((thread_data->td).td_deque_size) / INITIAL_TASK_DEQUE_SIZE(1 << 8) >= pass)
3373 return result;
3374
3375 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3376 __kmp_realloc_task_deque(thread, thread_data);
3377
3378 } else {
3379
3380 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3381
3382 if (TCR_4(thread_data->td.td_deque_ntasks)(thread_data->td.td_deque_ntasks) >=
3383 TASK_DEQUE_SIZE(thread_data->td)((thread_data->td).td_deque_size)) {
3384 KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to "if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: queue is full while giving task %p to "
"thread %d.\n", taskdata, tid); }
3385 "thread %d.\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: queue is full while giving task %p to "
"thread %d.\n", taskdata, tid); }
3386 taskdata, tid))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: queue is full while giving task %p to "
"thread %d.\n", taskdata, tid); }
;
3387
3388 // if this deque is bigger than the pass ratio give a chance to another
3389 // thread
3390 if (TASK_DEQUE_SIZE(thread_data->td)((thread_data->td).td_deque_size) / INITIAL_TASK_DEQUE_SIZE(1 << 8) >= pass)
3391 goto release_and_exit;
3392
3393 __kmp_realloc_task_deque(thread, thread_data);
3394 }
3395 }
3396
3397 // lock is held here, and there is space in the deque
3398
3399 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata;
3400 // Wrap index.
3401 thread_data->td.td_deque_tail =
3402 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td)((thread_data->td).td_deque_size - 1);
3403 TCW_4(thread_data->td.td_deque_ntasks,(thread_data->td.td_deque_ntasks) = ((thread_data->td.td_deque_ntasks
) + 1)
3404 TCR_4(thread_data->td.td_deque_ntasks) + 1)(thread_data->td.td_deque_ntasks) = ((thread_data->td.td_deque_ntasks
) + 1)
;
3405
3406 result = true;
3407 KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: successfully gave task %p to thread %d.\n"
, taskdata, tid); }
3408 taskdata, tid))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_give_task: successfully gave task %p to thread %d.\n"
, taskdata, tid); }
;
3409
3410release_and_exit:
3411 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3412
3413 return result;
3414}
3415
3416/* The finish of the proxy tasks is divided in two pieces:
3417 - the top half is the one that can be done from a thread outside the team
3418 - the bottom half must be run from a them within the team
3419
3420 In order to run the bottom half the task gets queued back into one of the
3421 threads of the team. Once the td_incomplete_child_task counter of the parent
3422 is decremented the threads can leave the barriers. So, the bottom half needs
3423 to be queued before the counter is decremented. The top half is therefore
3424 divided in two parts:
3425 - things that can be run before queuing the bottom half
3426 - things that must be run after queuing the bottom half
3427
3428 This creates a second race as the bottom half can free the task before the
3429 second top half is executed. To avoid this we use the
3430 td_incomplete_child_task of the proxy task to synchronize the top and bottom
3431 half. */
3432static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
3433 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT)((taskdata->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3433))
;
3434 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY)((taskdata->td_flags.proxy == 1) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.proxy == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3434))
;
3435 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0)((taskdata->td_flags.complete == 0) ? 0 : __kmp_debug_assert
("taskdata->td_flags.complete == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3435))
;
3436 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0)((taskdata->td_flags.freed == 0) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.freed == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3436))
;
3437
3438 taskdata->td_flags.complete = 1; // mark the task as completed
3439
3440 if (taskdata->td_taskgroup)
3441 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count)(&taskdata->td_taskgroup->count)->fetch_sub(1, std
::memory_order_acq_rel)
;
3442
3443 // Create an imaginary children for this task so the bottom half cannot
3444 // release the task before we have completed the second top half
3445 KMP_ATOMIC_INC(&taskdata->td_incomplete_child_tasks)(&taskdata->td_incomplete_child_tasks)->fetch_add(1
, std::memory_order_acq_rel)
;
3446}
3447
3448static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
3449 kmp_int32 children = 0;
3450
3451 // Predecrement simulated by "- 1" calculation
3452 children =
3453 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks)(&taskdata->td_parent->td_incomplete_child_tasks)->
fetch_sub(1, std::memory_order_acq_rel)
- 1;
3454 KMP_DEBUG_ASSERT(children >= 0)((children >= 0) ? 0 : __kmp_debug_assert("children >= 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3454))
;
3455
3456 // Remove the imaginary children
3457 KMP_ATOMIC_DEC(&taskdata->td_incomplete_child_tasks)(&taskdata->td_incomplete_child_tasks)->fetch_sub(1
, std::memory_order_acq_rel)
;
3458}
3459
3460static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) {
3461 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask)(((kmp_taskdata_t *)ptask) - 1);
3462 kmp_info_t *thread = __kmp_threads[gtid];
3463
3464 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY)((taskdata->td_flags.proxy == 1) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.proxy == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3464))
;
3465 KMP_DEBUG_ASSERT(taskdata->td_flags.complete ==((taskdata->td_flags.complete == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.complete == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3466))
3466 1)((taskdata->td_flags.complete == 1) ? 0 : __kmp_debug_assert
("taskdata->td_flags.complete == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3466))
; // top half must run before bottom half
3467
3468 // We need to wait to make sure the top half is finished
3469 // Spinning here should be ok as this should happen quickly
3470 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks)(&taskdata->td_incomplete_child_tasks)->load(std::memory_order_acquire
)
> 0)
3471 ;
3472
3473 __kmp_release_deps(gtid, taskdata);
3474 __kmp_free_task_and_ancestors(gtid, taskdata, thread);
3475}
3476
3477/*!
3478@ingroup TASKING
3479@param gtid Global Thread ID of encountering thread
3480@param ptask Task which execution is completed
3481
3482Execute the completation of a proxy task from a thread of that is part of the
3483team. Run first and bottom halves directly.
3484*/
3485void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) {
3486 KMP_DEBUG_ASSERT(ptask != NULL)((ptask != __null) ? 0 : __kmp_debug_assert("ptask != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3486))
;
3487 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask)(((kmp_taskdata_t *)ptask) - 1);
3488 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n"
, gtid, taskdata); }
3489 10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n"
, gtid, taskdata); }
3490 gtid, taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n"
, gtid, taskdata); }
;
3491
3492 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY)((taskdata->td_flags.proxy == 1) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.proxy == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3492))
;
3493
3494 __kmp_first_top_half_finish_proxy(taskdata);
3495 __kmp_second_top_half_finish_proxy(taskdata);
3496 __kmp_bottom_half_finish_proxy(gtid, ptask);
3497
3498 KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n"
, gtid, taskdata); }
3499 ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n"
, gtid, taskdata); }
3500 gtid, taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n"
, gtid, taskdata); }
;
3501}
3502
3503/*!
3504@ingroup TASKING
3505@param ptask Task which execution is completed
3506
3507Execute the completation of a proxy task from a thread that could not belong to
3508the team.
3509*/
3510void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) {
3511 KMP_DEBUG_ASSERT(ptask != NULL)((ptask != __null) ? 0 : __kmp_debug_assert("ptask != __null"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3511))
;
3512 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask)(((kmp_taskdata_t *)ptask) - 1);
3513
3514 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n"
, taskdata); }
3515 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n"
, taskdata); }
3516 ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n"
, taskdata); }
3517 taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n"
, taskdata); }
;
3518
3519 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY)((taskdata->td_flags.proxy == 1) ? 0 : __kmp_debug_assert(
"taskdata->td_flags.proxy == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3519))
;
3520
3521 __kmp_first_top_half_finish_proxy(taskdata);
3522
3523 // Enqueue task to complete bottom half completion from a thread within the
3524 // corresponding team
3525 kmp_team_t *team = taskdata->td_team;
3526 kmp_int32 nthreads = team->t.t_nproc;
3527 kmp_info_t *thread;
3528
3529 // This should be similar to start_k = __kmp_get_random( thread ) % nthreads
3530 // but we cannot use __kmp_get_random here
3531 kmp_int32 start_k = 0;
3532 kmp_int32 pass = 1;
3533 kmp_int32 k = start_k;
3534
3535 do {
3536 // For now we're just linearly trying to find a thread
3537 thread = team->t.t_threads[k];
3538 k = (k + 1) % nthreads;
3539
3540 // we did a full pass through all the threads
3541 if (k == start_k)
3542 pass = pass << 1;
3543
3544 } while (!__kmp_give_task(thread, k, ptask, pass));
3545
3546 __kmp_second_top_half_finish_proxy(taskdata);
3547
3548 KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n"
, taskdata); }
3549 10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n"
, taskdata); }
3550 ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n"
, taskdata); }
3551 taskdata))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n"
, taskdata); }
;
3552}
3553
3554// __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task
3555// for taskloop
3556//
3557// thread: allocating thread
3558// task_src: pointer to source task to be duplicated
3559// returns: a pointer to the allocated kmp_task_t structure (task).
3560kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
3561 kmp_task_t *task;
3562 kmp_taskdata_t *taskdata;
3563 kmp_taskdata_t *taskdata_src;
3564 kmp_taskdata_t *parent_task = thread->th.th_current_task;
3565 size_t shareds_offset;
3566 size_t task_size;
3567
3568 KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n"
, thread, task_src); }
3569 task_src))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n"
, thread, task_src); }
;
3570 taskdata_src = KMP_TASK_TO_TASKDATA(task_src)(((kmp_taskdata_t *)task_src) - 1);
3571 KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy ==((taskdata_src->td_flags.proxy == 0) ? 0 : __kmp_debug_assert
("taskdata_src->td_flags.proxy == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3572))
3572 TASK_FULL)((taskdata_src->td_flags.proxy == 0) ? 0 : __kmp_debug_assert
("taskdata_src->td_flags.proxy == 0", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3572))
; // it should not be proxy task
3573 KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT)((taskdata_src->td_flags.tasktype == 1) ? 0 : __kmp_debug_assert
("taskdata_src->td_flags.tasktype == 1", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3573))
;
3574 task_size = taskdata_src->td_size_alloc;
3575
3576 // Allocate a kmp_taskdata_t block and a kmp_task_t block.
3577 KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n"
, thread, task_size); }
3578 task_size))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n"
, thread, task_size); }
;
3579#if USE_FAST_MEMORY3
3580 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size)___kmp_fast_allocate((thread), (task_size), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3580)
;
3581#else
3582 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size)___kmp_thread_malloc((thread), (task_size), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3582)
;
3583#endif /* USE_FAST_MEMORY */
3584 KMP_MEMCPYmemcpy(taskdata, taskdata_src, task_size);
3585
3586 task = KMP_TASKDATA_TO_TASK(taskdata)(kmp_task_t *)(taskdata + 1);
3587
3588 // Initialize new task (only specific fields not affected by memcpy)
3589 taskdata->td_task_id = KMP_GEN_TASK_ID()(~0);
3590 if (task->shareds != NULL__null) { // need setup shareds pointer
3591 shareds_offset = (char *)task_src->shareds - (char *)taskdata_src;
3592 task->shareds = &((char *)taskdata)[shareds_offset];
3593 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==(((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1
)) == 0) ? 0 : __kmp_debug_assert("(((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3594))
3594 0)(((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1
)) == 0) ? 0 : __kmp_debug_assert("(((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3594))
;
3595 }
3596 taskdata->td_alloc_thread = thread;
3597 taskdata->td_parent = parent_task;
3598 taskdata->td_taskgroup =
3599 parent_task
3600 ->td_taskgroup; // task inherits the taskgroup from the parent task
3601
3602 // Only need to keep track of child task counts if team parallel and tasking
3603 // not serialized
3604 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
3605 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks)(&parent_task->td_incomplete_child_tasks)->fetch_add
(1, std::memory_order_acq_rel)
;
3606 if (parent_task->td_taskgroup)
3607 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count)(&parent_task->td_taskgroup->count)->fetch_add(1
, std::memory_order_acq_rel)
;
3608 // Only need to keep track of allocated child tasks for explicit tasks since
3609 // implicit not deallocated
3610 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT1)
3611 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks)(&taskdata->td_parent->td_allocated_child_tasks)->
fetch_add(1, std::memory_order_acq_rel)
;
3612 }
3613
3614 KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n"
, thread, taskdata, taskdata->td_parent); }
3615 ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n"
, thread, taskdata, taskdata->td_parent); }
3616 thread, taskdata, taskdata->td_parent))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n"
, thread, taskdata, taskdata->td_parent); }
;
3617#if OMPT_SUPPORT1
3618 if (UNLIKELY(ompt_enabled.enabled)__builtin_expect(!!(ompt_enabled.enabled), 0))
3619 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid);
3620#endif
3621 return task;
3622}
3623
3624// Routine optionally generated by the compiler for setting the lastprivate flag
3625// and calling needed constructors for private/firstprivate objects
3626// (used to form taskloop tasks from pattern task)
3627// Parameters: dest task, src task, lastprivate flag.
3628typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
3629
3630KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8)static_assert(sizeof(long) == 4 || sizeof(long) == 8, "Build condition error"
)
;
3631
3632// class to encapsulate manipulating loop bounds in a taskloop task.
3633// this abstracts away the Intel vs GOMP taskloop interface for setting/getting
3634// the loop bound variables.
3635class kmp_taskloop_bounds_t {
3636 kmp_task_t *task;
3637 const kmp_taskdata_t *taskdata;
3638 size_t lower_offset;
3639 size_t upper_offset;
3640
3641public:
3642 kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub)
3643 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1)),
3644 lower_offset((char *)lb - (char *)task),
3645 upper_offset((char *)ub - (char *)task) {
3646 KMP_DEBUG_ASSERT((char *)lb > (char *)_task)(((char *)lb > (char *)_task) ? 0 : __kmp_debug_assert("(char *)lb > (char *)_task"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3646))
;
3647 KMP_DEBUG_ASSERT((char *)ub > (char *)_task)(((char *)ub > (char *)_task) ? 0 : __kmp_debug_assert("(char *)ub > (char *)_task"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3647))
;
3648 }
3649 kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds)
3650 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)(((kmp_taskdata_t *)_task) - 1)),
3651 lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {}
3652 size_t get_lower_offset() const { return lower_offset; }
3653 size_t get_upper_offset() const { return upper_offset; }
3654 kmp_uint64 get_lb() const {
3655 kmp_int64 retval;
3656#if defined(KMP_GOMP_COMPAT)
3657 // Intel task just returns the lower bound normally
3658 if (!taskdata->td_flags.native) {
3659 retval = *(kmp_int64 *)((char *)task + lower_offset);
3660 } else {
3661 // GOMP task has to take into account the sizeof(long)
3662 if (taskdata->td_size_loop_bounds == 4) {
3663 kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds)reinterpret_cast<kmp_int32 *>(task->shareds);
3664 retval = (kmp_int64)*lb;
3665 } else {
3666 kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds)reinterpret_cast<kmp_int64 *>(task->shareds);
3667 retval = (kmp_int64)*lb;
3668 }
3669 }
3670#else
3671 retval = *(kmp_int64 *)((char *)task + lower_offset);
3672#endif // defined(KMP_GOMP_COMPAT)
3673 return retval;
3674 }
3675 kmp_uint64 get_ub() const {
3676 kmp_int64 retval;
3677#if defined(KMP_GOMP_COMPAT)
3678 // Intel task just returns the upper bound normally
3679 if (!taskdata->td_flags.native) {
3680 retval = *(kmp_int64 *)((char *)task + upper_offset);
3681 } else {
3682 // GOMP task has to take into account the sizeof(long)
3683 if (taskdata->td_size_loop_bounds == 4) {
3684 kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds)reinterpret_cast<kmp_int32 *>(task->shareds) + 1;
3685 retval = (kmp_int64)*ub;
3686 } else {
3687 kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds)reinterpret_cast<kmp_int64 *>(task->shareds) + 1;
3688 retval = (kmp_int64)*ub;
3689 }
3690 }
3691#else
3692 retval = *(kmp_int64 *)((char *)task + upper_offset);
3693#endif // defined(KMP_GOMP_COMPAT)
3694 return retval;
3695 }
3696 void set_lb(kmp_uint64 lb) {
3697#if defined(KMP_GOMP_COMPAT)
3698 // Intel task just sets the lower bound normally
3699 if (!taskdata->td_flags.native) {
3700 *(kmp_uint64 *)((char *)task + lower_offset) = lb;
3701 } else {
3702 // GOMP task has to take into account the sizeof(long)
3703 if (taskdata->td_size_loop_bounds == 4) {
3704 kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds)reinterpret_cast<kmp_uint32 *>(task->shareds);
3705 *lower = (kmp_uint32)lb;
3706 } else {
3707 kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds)reinterpret_cast<kmp_uint64 *>(task->shareds);
3708 *lower = (kmp_uint64)lb;
3709 }
3710 }
3711#else
3712 *(kmp_uint64 *)((char *)task + lower_offset) = lb;
3713#endif // defined(KMP_GOMP_COMPAT)
3714 }
3715 void set_ub(kmp_uint64 ub) {
3716#if defined(KMP_GOMP_COMPAT)
3717 // Intel task just sets the upper bound normally
3718 if (!taskdata->td_flags.native) {
3719 *(kmp_uint64 *)((char *)task + upper_offset) = ub;
3720 } else {
3721 // GOMP task has to take into account the sizeof(long)
3722 if (taskdata->td_size_loop_bounds == 4) {
3723 kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds)reinterpret_cast<kmp_uint32 *>(task->shareds) + 1;
3724 *upper = (kmp_uint32)ub;
3725 } else {
3726 kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds)reinterpret_cast<kmp_uint64 *>(task->shareds) + 1;
3727 *upper = (kmp_uint64)ub;
3728 }
3729 }
3730#else
3731 *(kmp_uint64 *)((char *)task + upper_offset) = ub;
3732#endif // defined(KMP_GOMP_COMPAT)
3733 }
3734};
3735
3736// __kmp_taskloop_linear: Start tasks of the taskloop linearly
3737//
3738// loc Source location information
3739// gtid Global thread ID
3740// task Pattern task, exposes the loop iteration range
3741// lb Pointer to loop lower bound in task structure
3742// ub Pointer to loop upper bound in task structure
3743// st Loop stride
3744// ub_glob Global upper bound (used for lastprivate check)
3745// num_tasks Number of tasks to execute
3746// grainsize Number of loop iterations per task
3747// extras Number of chunks with grainsize+1 iterations
3748// tc Iterations count
3749// task_dup Tasks duplication routine
3750// codeptr_ra Return address for OMPT events
3751void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
3752 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
3753 kmp_uint64 ub_glob, kmp_uint64 num_tasks,
3754 kmp_uint64 grainsize, kmp_uint64 extras,
3755 kmp_uint64 tc,
3756#if OMPT_SUPPORT1
3757 void *codeptr_ra,
3758#endif
3759 void *task_dup) {
3760 KMP_COUNT_BLOCK(OMP_TASKLOOP)((void)0);
3761 KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling)((void)0);
3762 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
3763 // compiler provides global bounds here
3764 kmp_taskloop_bounds_t task_bounds(task, lb, ub);
3765 kmp_uint64 lower = task_bounds.get_lb();
3766 kmp_uint64 upper = task_bounds.get_ub();
3767 kmp_uint64 i;
3768 kmp_info_t *thread = __kmp_threads[gtid];
3769 kmp_taskdata_t *current_task = thread->th.th_current_task;
3770 kmp_task_t *next_task;
3771 kmp_int32 lastpriv = 0;
3772
3773 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras)((tc == num_tasks * grainsize + extras) ? 0 : __kmp_debug_assert
("tc == num_tasks * grainsize + extras", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3773))
;
3774 KMP_DEBUG_ASSERT(num_tasks > extras)((num_tasks > extras) ? 0 : __kmp_debug_assert("num_tasks > extras"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3774))
;
3775 KMP_DEBUG_ASSERT(num_tasks > 0)((num_tasks > 0) ? 0 : __kmp_debug_assert("num_tasks > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3775))
;
3776 KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
"extras %lld, i=%lld,%lld(%d)%lld, dup %p\n", gtid, num_tasks
, grainsize, extras, lower, upper, ub_glob, st, task_dup); }
3777 "extras %lld, i=%lld,%lld(%d)%lld, dup %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
"extras %lld, i=%lld,%lld(%d)%lld, dup %p\n", gtid, num_tasks
, grainsize, extras, lower, upper, ub_glob, st, task_dup); }
3778 gtid, num_tasks, grainsize, extras, lower, upper, ub_glob, st,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
"extras %lld, i=%lld,%lld(%d)%lld, dup %p\n", gtid, num_tasks
, grainsize, extras, lower, upper, ub_glob, st, task_dup); }
3779 task_dup))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
"extras %lld, i=%lld,%lld(%d)%lld, dup %p\n", gtid, num_tasks
, grainsize, extras, lower, upper, ub_glob, st, task_dup); }
;
3780
3781 // Launch num_tasks tasks, assign grainsize iterations each task
3782 for (i = 0; i < num_tasks; ++i) {
21
Loop condition is true. Entering loop body
3783 kmp_uint64 chunk_minus_1;
3784 if (extras == 0) {
22
Assuming 'extras' is not equal to 0
23
Taking false branch
3785 chunk_minus_1 = grainsize - 1;
3786 } else {
3787 chunk_minus_1 = grainsize;
3788 --extras; // first extras iterations get bigger chunk (grainsize+1)
3789 }
3790 upper = lower + st * chunk_minus_1;
3791 if (i == num_tasks - 1) {
24
Assuming the condition is false
25
Taking false branch
3792 // schedule the last task, set lastprivate flag if needed
3793 if (st == 1) { // most common case
3794 KMP_DEBUG_ASSERT(upper == *ub)((upper == *ub) ? 0 : __kmp_debug_assert("upper == *ub", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3794))
;
3795 if (upper == ub_glob)
3796 lastpriv = 1;
3797 } else if (st > 0) { // positive loop stride
3798 KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper)(((kmp_uint64)st > *ub - upper) ? 0 : __kmp_debug_assert("(kmp_uint64)st > *ub - upper"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3798))
;
3799 if ((kmp_uint64)st > ub_glob - upper)
3800 lastpriv = 1;
3801 } else { // negative loop stride
3802 KMP_DEBUG_ASSERT(upper + st < *ub)((upper + st < *ub) ? 0 : __kmp_debug_assert("upper + st < *ub"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3802))
;
3803 if (upper - ub_glob < (kmp_uint64)(-st))
3804 lastpriv = 1;
3805 }
3806 }
3807 next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
3808 kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task)(((kmp_taskdata_t *)next_task) - 1);
3809 kmp_taskloop_bounds_t next_task_bounds =
3810 kmp_taskloop_bounds_t(next_task, task_bounds);
3811
3812 // adjust task-specific bounds
3813 next_task_bounds.set_lb(lower);
3814 if (next_taskdata->td_flags.native) {
26
Taking false branch
3815 next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1));
3816 } else {
3817 next_task_bounds.set_ub(upper);
3818 }
3819 if (ptask_dup != NULL__null) // set lastprivate flag, construct fistprivates, etc.
27
Assuming 'ptask_dup' is equal to NULL
28
Taking false branch
3820 ptask_dup(next_task, task, lastpriv);
3821 KA_TRACE(40,if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
"upper %lld stride %lld, (offsets %p %p)\n", gtid, i, next_task
, lower, upper, st, next_task_bounds.get_lower_offset(), next_task_bounds
.get_upper_offset()); }
3822 ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
"upper %lld stride %lld, (offsets %p %p)\n", gtid, i, next_task
, lower, upper, st, next_task_bounds.get_lower_offset(), next_task_bounds
.get_upper_offset()); }
3823 "upper %lld stride %lld, (offsets %p %p)\n",if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
"upper %lld stride %lld, (offsets %p %p)\n", gtid, i, next_task
, lower, upper, st, next_task_bounds.get_lower_offset(), next_task_bounds
.get_upper_offset()); }
3824 gtid, i, next_task, lower, upper, st,if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
"upper %lld stride %lld, (offsets %p %p)\n", gtid, i, next_task
, lower, upper, st, next_task_bounds.get_lower_offset(), next_task_bounds
.get_upper_offset()); }
3825 next_task_bounds.get_lower_offset(),if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
"upper %lld stride %lld, (offsets %p %p)\n", gtid, i, next_task
, lower, upper, st, next_task_bounds.get_lower_offset(), next_task_bounds
.get_upper_offset()); }
3826 next_task_bounds.get_upper_offset()))if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
"upper %lld stride %lld, (offsets %p %p)\n", gtid, i, next_task
, lower, upper, st, next_task_bounds.get_lower_offset(), next_task_bounds
.get_upper_offset()); }
;
3827#if OMPT_SUPPORT1
3828 __kmp_omp_taskloop_task(NULL__null, gtid, next_task,
29
Calling '__kmp_omp_taskloop_task'
3829 codeptr_ra); // schedule new task
3830#else
3831 __kmp_omp_task(gtid, next_task, true); // schedule new task
3832#endif
3833 lower = upper + st; // adjust lower bound for the next iteration
3834 }
3835 // free the pattern task and exit
3836 __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
3837 // do not execute the pattern task, just do internal bookkeeping
3838 __kmp_task_finish<false>(gtid, task, current_task);
3839}
3840
3841// Structure to keep taskloop parameters for auxiliary task
3842// kept in the shareds of the task structure.
3843typedef struct __taskloop_params {
3844 kmp_task_t *task;
3845 kmp_uint64 *lb;
3846 kmp_uint64 *ub;
3847 void *task_dup;
3848 kmp_int64 st;
3849 kmp_uint64 ub_glob;
3850 kmp_uint64 num_tasks;
3851 kmp_uint64 grainsize;
3852 kmp_uint64 extras;
3853 kmp_uint64 tc;
3854 kmp_uint64 num_t_min;
3855#if OMPT_SUPPORT1
3856 void *codeptr_ra;
3857#endif
3858} __taskloop_params_t;
3859
3860void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *,
3861 kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64,
3862 kmp_uint64, kmp_uint64, kmp_uint64, kmp_uint64,
3863#if OMPT_SUPPORT1
3864 void *,
3865#endif
3866 void *);
3867
3868// Execute part of the the taskloop submitted as a task.
3869int __kmp_taskloop_task(int gtid, void *ptask) {
3870 __taskloop_params_t *p =
3871 (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds;
3872 kmp_task_t *task = p->task;
3873 kmp_uint64 *lb = p->lb;
3874 kmp_uint64 *ub = p->ub;
3875 void *task_dup = p->task_dup;
3876 // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
3877 kmp_int64 st = p->st;
3878 kmp_uint64 ub_glob = p->ub_glob;
3879 kmp_uint64 num_tasks = p->num_tasks;
3880 kmp_uint64 grainsize = p->grainsize;
3881 kmp_uint64 extras = p->extras;
3882 kmp_uint64 tc = p->tc;
3883 kmp_uint64 num_t_min = p->num_t_min;
3884#if OMPT_SUPPORT1
3885 void *codeptr_ra = p->codeptr_ra;
3886#endif
3887#if KMP_DEBUG1
3888 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
3889 KMP_DEBUG_ASSERT(task != NULL)((task != __null) ? 0 : __kmp_debug_assert("task != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3889))
;
3890 KA_TRACE(20, ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
3891 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
3892 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
3893 task_dup))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
;
3894#endif
3895 KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min)((num_tasks * 2 + 1 > num_t_min) ? 0 : __kmp_debug_assert(
"num_tasks * 2 + 1 > num_t_min", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3895))
;
3896 if (num_tasks > num_t_min)
3897 __kmp_taskloop_recur(NULL__null, gtid, task, lb, ub, st, ub_glob, num_tasks,
3898 grainsize, extras, tc, num_t_min,
3899#if OMPT_SUPPORT1
3900 codeptr_ra,
3901#endif
3902 task_dup);
3903 else
3904 __kmp_taskloop_linear(NULL__null, gtid, task, lb, ub, st, ub_glob, num_tasks,
3905 grainsize, extras, tc,
3906#if OMPT_SUPPORT1
3907 codeptr_ra,
3908#endif
3909 task_dup);
3910
3911 KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid))if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmp_taskloop_task(exit): T#%d\n"
, gtid); }
;
3912 return 0;
3913}
3914
3915// Schedule part of the the taskloop as a task,
3916// execute the rest of the the taskloop.
3917//
3918// loc Source location information
3919// gtid Global thread ID
3920// task Pattern task, exposes the loop iteration range
3921// lb Pointer to loop lower bound in task structure
3922// ub Pointer to loop upper bound in task structure
3923// st Loop stride
3924// ub_glob Global upper bound (used for lastprivate check)
3925// num_tasks Number of tasks to execute
3926// grainsize Number of loop iterations per task
3927// extras Number of chunks with grainsize+1 iterations
3928// tc Iterations count
3929// num_t_min Threashold to launch tasks recursively
3930// task_dup Tasks duplication routine
3931// codeptr_ra Return address for OMPT events
3932void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
3933 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
3934 kmp_uint64 ub_glob, kmp_uint64 num_tasks,
3935 kmp_uint64 grainsize, kmp_uint64 extras,
3936 kmp_uint64 tc, kmp_uint64 num_t_min,
3937#if OMPT_SUPPORT1
3938 void *codeptr_ra,
3939#endif
3940 void *task_dup) {
3941#if KMP_DEBUG1
3942 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
3943 KMP_DEBUG_ASSERT(task != NULL)((task != __null) ? 0 : __kmp_debug_assert("task != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3943))
;
3944 KMP_DEBUG_ASSERT(num_tasks > num_t_min)((num_tasks > num_t_min) ? 0 : __kmp_debug_assert("num_tasks > num_t_min"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3944))
;
3945 KA_TRACE(20, ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
3946 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
3947 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
3948 task_dup))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
" %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", gtid, taskdata
, num_tasks, grainsize, extras, *lb, *ub, st, task_dup); }
;
3949#endif
3950 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
3951 kmp_uint64 lower = *lb;
3952 kmp_uint64 upper = *ub;
3953 kmp_info_t *thread = __kmp_threads[gtid];
3954 // kmp_taskdata_t *current_task = thread->th.th_current_task;
3955 kmp_task_t *next_task;
3956 kmp_int32 lastpriv = 0;
3957 size_t lower_offset =
3958 (char *)lb - (char *)task; // remember offset of lb in the task structure
3959 size_t upper_offset =
3960 (char *)ub - (char *)task; // remember offset of ub in the task structure
3961
3962 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras)((tc == num_tasks * grainsize + extras) ? 0 : __kmp_debug_assert
("tc == num_tasks * grainsize + extras", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3962))
;
3963 KMP_DEBUG_ASSERT(num_tasks > extras)((num_tasks > extras) ? 0 : __kmp_debug_assert("num_tasks > extras"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3963))
;
3964 KMP_DEBUG_ASSERT(num_tasks > 0)((num_tasks > 0) ? 0 : __kmp_debug_assert("num_tasks > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 3964))
;
3965
3966 // split the loop in two halves
3967 kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1;
3968 kmp_uint64 gr_size0 = grainsize;
3969 kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute
3970 kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task
3971 if (n_tsk0 <= extras) {
3972 gr_size0++; // integrate extras into grainsize
3973 ext0 = 0; // no extra iters in 1st half
3974 ext1 = extras - n_tsk0; // remaining extras
3975 tc0 = gr_size0 * n_tsk0;
3976 tc1 = tc - tc0;
3977 } else { // n_tsk0 > extras
3978 ext1 = 0; // no extra iters in 2nd half
3979 ext0 = extras;
3980 tc1 = grainsize * n_tsk1;
3981 tc0 = tc - tc1;
3982 }
3983 ub0 = lower + st * (tc0 - 1);
3984 lb1 = ub0 + st;
3985
3986 // create pattern task for 2nd half of the loop
3987 next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
3988 // adjust lower bound (upper bound is not changed) for the 2nd half
3989 *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
3990 if (ptask_dup != NULL__null) // construct fistprivates, etc.
3991 ptask_dup(next_task, task, 0);
3992 *ub = ub0; // adjust upper bound for the 1st half
3993
3994 // create auxiliary task for 2nd half of the loop
3995 kmp_task_t *new_task =
3996 __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *),
3997 sizeof(__taskloop_params_t), &__kmp_taskloop_task);
3998 __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds;
3999 p->task = next_task;
4000 p->lb = (kmp_uint64 *)((char *)next_task + lower_offset);
4001 p->ub = (kmp_uint64 *)((char *)next_task + upper_offset);
4002 p->task_dup = task_dup;
4003 p->st = st;
4004 p->ub_glob = ub_glob;
4005 p->num_tasks = n_tsk1;
4006 p->grainsize = grainsize;
4007 p->extras = ext1;
4008 p->tc = tc1;
4009 p->num_t_min = num_t_min;
4010#if OMPT_SUPPORT1
4011 p->codeptr_ra = codeptr_ra;
4012#endif
4013
4014#if OMPT_SUPPORT1
4015 // schedule new task with correct return address for OMPT events
4016 __kmp_omp_taskloop_task(NULL__null, gtid, new_task, codeptr_ra);
4017#else
4018 __kmp_omp_task(NULL__null, gtid, new_task); // schedule new task
4019#endif
4020
4021 // execute the 1st half of current subrange
4022 if (n_tsk0 > num_t_min)
4023 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0,
4024 ext0, tc0, num_t_min,
4025#if OMPT_SUPPORT1
4026 codeptr_ra,
4027#endif
4028 task_dup);
4029 else
4030 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
4031 gr_size0, ext0, tc0,
4032#if OMPT_SUPPORT1
4033 codeptr_ra,
4034#endif
4035 task_dup);
4036
4037 KA_TRACE(40, ("__kmpc_taskloop_recur(exit): T#%d\n", gtid))if (kmp_a_debug >= 40) { __kmp_debug_printf ("__kmpc_taskloop_recur(exit): T#%d\n"
, gtid); }
;
4038}
4039
4040/*!
4041@ingroup TASKING
4042@param loc Source location information
4043@param gtid Global thread ID
4044@param task Task structure
4045@param if_val Value of the if clause
4046@param lb Pointer to loop lower bound in task structure
4047@param ub Pointer to loop upper bound in task structure
4048@param st Loop stride
4049@param nogroup Flag, 1 if nogroup clause specified, 0 otherwise
4050@param sched Schedule specified 0/1/2 for none/grainsize/num_tasks
4051@param grainsize Schedule value if specified
4052@param task_dup Tasks duplication routine
4053
4054Execute the taskloop construct.
4055*/
4056void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
4057 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup,
4058 int sched, kmp_uint64 grainsize, void *task_dup) {
4059 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task)(((kmp_taskdata_t *)task) - 1);
4060 KMP_DEBUG_ASSERT(task != NULL)((task != __null) ? 0 : __kmp_debug_assert("task != __null", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 4060))
;
4061
4062 if (nogroup == 0) {
1
Assuming 'nogroup' is not equal to 0
2
Taking false branch
4063#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
4064 OMPT_STORE_RETURN_ADDRESS(gtid)if (ompt_enabled.enabled && gtid >= 0 && __kmp_threads
[gtid] && !__kmp_threads[gtid]->th.ompt_thread_info
.return_address) __kmp_threads[gtid]->th.ompt_thread_info.
return_address = __builtin_return_address(0)
;
4065#endif
4066 __kmpc_taskgroup(loc, gtid);
4067 }
4068
4069 // =========================================================================
4070 // calculate loop parameters
4071 kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4072 kmp_uint64 tc;
4073 // compiler provides global bounds here
4074 kmp_uint64 lower = task_bounds.get_lb();
4075 kmp_uint64 upper = task_bounds.get_ub();
4076 kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag
4077 kmp_uint64 num_tasks = 0, extras = 0;
4078 kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks;
4079 kmp_info_t *thread = __kmp_threads[gtid];
4080 kmp_taskdata_t *current_task = thread->th.th_current_task;
4081
4082 KA_TRACE(20, ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
"grain %llu(%d), dup %p\n", gtid, taskdata, lower, upper, st
, grainsize, sched, task_dup); }
4083 "grain %llu(%d), dup %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
"grain %llu(%d), dup %p\n", gtid, taskdata, lower, upper, st
, grainsize, sched, task_dup); }
4084 gtid, taskdata, lower, upper, st, grainsize, sched, task_dup))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
"grain %llu(%d), dup %p\n", gtid, taskdata, lower, upper, st
, grainsize, sched, task_dup); }
;
4085
4086 // compute trip count
4087 if (st == 1) { // most common case
3
Assuming 'st' is not equal to 1
4
Taking false branch
4088 tc = upper - lower + 1;
4089 } else if (st < 0) {
5
Assuming 'st' is >= 0
6
Taking false branch
4090 tc = (lower - upper) / (-st) + 1;
4091 } else { // st > 0
4092 tc = (upper - lower) / st + 1;
4093 }
4094 if (tc == 0) {
7
Assuming 'tc' is not equal to 0
8
Taking false branch
4095 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d zero-trip loop\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop(exit): T#%d zero-trip loop\n"
, gtid); }
;
4096 // free the pattern task and exit
4097 __kmp_task_start(gtid, task, current_task);
4098 // do not execute anything for zero-trip loop
4099 __kmp_task_finish<false>(gtid, task, current_task);
4100 return;
4101 }
4102
4103#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
4104 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL__null);
4105 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
4106 if (ompt_enabled.ompt_callback_work) {
9
Assuming the condition is false
10
Taking false branch
4107 ompt_callbacks.ompt_callback(ompt_callback_work)ompt_callback_work_callback(
4108 ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
4109 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0));
4110 }
4111#endif
4112
4113 if (num_tasks_min == 0)
11
Assuming 'num_tasks_min' is not equal to 0
12
Taking false branch
4114 // TODO: can we choose better default heuristic?
4115 num_tasks_min =
4116 KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE)((thread->th.th_team_nproc * 10) < ((1 << 8)) ? (
thread->th.th_team_nproc * 10) : ((1 << 8)))
;
4117
4118 // compute num_tasks/grainsize based on the input provided
4119 switch (sched) {
13
Control jumps to 'case 1:' at line 4134
4120 case 0: // no schedule clause specified, we can choose the default
4121 // let's try to schedule (team_size*10) tasks
4122 grainsize = thread->th.th_team_nproc * 10;
4123 case 2: // num_tasks provided
4124 if (grainsize > tc) {
4125 num_tasks = tc; // too big num_tasks requested, adjust values
4126 grainsize = 1;
4127 extras = 0;
4128 } else {
4129 num_tasks = grainsize;
4130 grainsize = tc / num_tasks;
4131 extras = tc % num_tasks;
4132 }
4133 break;
4134 case 1: // grainsize provided
4135 if (grainsize > tc) {
14
Assuming 'grainsize' is <= 'tc'
15
Taking false branch
4136 num_tasks = 1; // too big grainsize requested, adjust values
4137 grainsize = tc;
4138 extras = 0;
4139 } else {
4140 num_tasks = tc / grainsize;
4141 // adjust grainsize for balanced distribution of iterations
4142 grainsize = tc / num_tasks;
4143 extras = tc % num_tasks;
4144 }
4145 break;
16
Execution continues on line 4149
4146 default:
4147 KMP_ASSERT2(0, "unknown scheduling of taskloop")((0) ? 0 : __kmp_debug_assert(("unknown scheduling of taskloop"
), "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 4147))
;
4148 }
4149 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras)((tc == num_tasks * grainsize + extras) ? 0 : __kmp_debug_assert
("tc == num_tasks * grainsize + extras", "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 4149))
;
4150 KMP_DEBUG_ASSERT(num_tasks > extras)((num_tasks > extras) ? 0 : __kmp_debug_assert("num_tasks > extras"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 4150))
;
4151 KMP_DEBUG_ASSERT(num_tasks > 0)((num_tasks > 0) ? 0 : __kmp_debug_assert("num_tasks > 0"
, "/build/llvm-toolchain-snapshot-7~svn338205/projects/openmp/runtime/src/kmp_tasking.cpp"
, 4151))
;
4152 // =========================================================================
4153
4154 // check if clause value first
4155 // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native)
4156 if (if_val == 0) { // if(0) specified, mark task as serial
17
Assuming 'if_val' is not equal to 0
18
Taking false branch
4157 taskdata->td_flags.task_serial = 1;
4158 taskdata->td_flags.tiedness = TASK_TIED1; // AC: serial task cannot be untied
4159 // always start serial tasks linearly
4160 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4161 grainsize, extras, tc,
4162#if OMPT_SUPPORT1
4163 OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0),
4164#endif
4165 task_dup);
4166 // !taskdata->td_flags.native => currently force linear spawning of tasks
4167 // for GOMP_taskloop
4168 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) {
19
Assuming 'num_tasks' is <= 'num_tasks_min'
4169 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
"(%lld), grain %llu, extras %llu\n", gtid, tc, num_tasks, num_tasks_min
, grainsize, extras); }
4170 "(%lld), grain %llu, extras %llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
"(%lld), grain %llu, extras %llu\n", gtid, tc, num_tasks, num_tasks_min
, grainsize, extras); }
4171 gtid, tc, num_tasks, num_tasks_min, grainsize, extras))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
"(%lld), grain %llu, extras %llu\n", gtid, tc, num_tasks, num_tasks_min
, grainsize, extras); }
;
4172 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4173 grainsize, extras, tc, num_tasks_min,
4174#if OMPT_SUPPORT1
4175 OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0),
4176#endif
4177 task_dup);
4178 } else {
4179 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu"if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
"(%lld), grain %llu, extras %llu\n", gtid, tc, num_tasks, num_tasks_min
, grainsize, extras); }
4180 "(%lld), grain %llu, extras %llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
"(%lld), grain %llu, extras %llu\n", gtid, tc, num_tasks, num_tasks_min
, grainsize, extras); }
4181 gtid, tc, num_tasks, num_tasks_min, grainsize, extras))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
"(%lld), grain %llu, extras %llu\n", gtid, tc, num_tasks, num_tasks_min
, grainsize, extras); }
;
4182 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
20
Calling '__kmp_taskloop_linear'
4183 grainsize, extras, tc,
4184#if OMPT_SUPPORT1
4185 OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0),
4186#endif
4187 task_dup);
4188 }
4189
4190#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
4191 if (ompt_enabled.ompt_callback_work) {
4192 ompt_callbacks.ompt_callback(ompt_callback_work)ompt_callback_work_callback(
4193 ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data),
4194 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)__builtin_return_address(0));
4195 }
4196#endif
4197
4198 if (nogroup == 0) {
4199#if OMPT_SUPPORT1 && OMPT_OPTIONAL1
4200 OMPT_STORE_RETURN_ADDRESS(gtid)if (ompt_enabled.enabled && gtid >= 0 && __kmp_threads
[gtid] && !__kmp_threads[gtid]->th.ompt_thread_info
.return_address) __kmp_threads[gtid]->th.ompt_thread_info.
return_address = __builtin_return_address(0)
;
4201#endif
4202 __kmpc_end_taskgroup(loc, gtid);
4203 }
4204 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_taskloop(exit): T#%d\n"
, gtid); }
;
4205}
4206
4207#endif