LLVM OpenMP* Runtime Library
 All Classes Functions Variables Typedefs Enumerations Enumerator Groups Pages
z_Windows_NT_util.cpp
1 /*
2  * z_Windows_NT_util.cpp -- platform specific routines.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_wait_release.h"
19 
20 /* This code is related to NtQuerySystemInformation() function. This function
21  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
22  number of running threads in the system. */
23 
24 #include <ntsecapi.h> // UNICODE_STRING
25 #include <ntstatus.h>
26 
27 enum SYSTEM_INFORMATION_CLASS {
28  SystemProcessInformation = 5
29 }; // SYSTEM_INFORMATION_CLASS
30 
31 struct CLIENT_ID {
32  HANDLE UniqueProcess;
33  HANDLE UniqueThread;
34 }; // struct CLIENT_ID
35 
36 enum THREAD_STATE {
37  StateInitialized,
38  StateReady,
39  StateRunning,
40  StateStandby,
41  StateTerminated,
42  StateWait,
43  StateTransition,
44  StateUnknown
45 }; // enum THREAD_STATE
46 
47 struct VM_COUNTERS {
48  SIZE_T PeakVirtualSize;
49  SIZE_T VirtualSize;
50  ULONG PageFaultCount;
51  SIZE_T PeakWorkingSetSize;
52  SIZE_T WorkingSetSize;
53  SIZE_T QuotaPeakPagedPoolUsage;
54  SIZE_T QuotaPagedPoolUsage;
55  SIZE_T QuotaPeakNonPagedPoolUsage;
56  SIZE_T QuotaNonPagedPoolUsage;
57  SIZE_T PagefileUsage;
58  SIZE_T PeakPagefileUsage;
59  SIZE_T PrivatePageCount;
60 }; // struct VM_COUNTERS
61 
62 struct SYSTEM_THREAD {
63  LARGE_INTEGER KernelTime;
64  LARGE_INTEGER UserTime;
65  LARGE_INTEGER CreateTime;
66  ULONG WaitTime;
67  LPVOID StartAddress;
68  CLIENT_ID ClientId;
69  DWORD Priority;
70  LONG BasePriority;
71  ULONG ContextSwitchCount;
72  THREAD_STATE State;
73  ULONG WaitReason;
74 }; // SYSTEM_THREAD
75 
76 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
77 #if KMP_ARCH_X86
78 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
80 #else
81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
83 #endif
84 
85 struct SYSTEM_PROCESS_INFORMATION {
86  ULONG NextEntryOffset;
87  ULONG NumberOfThreads;
88  LARGE_INTEGER Reserved[3];
89  LARGE_INTEGER CreateTime;
90  LARGE_INTEGER UserTime;
91  LARGE_INTEGER KernelTime;
92  UNICODE_STRING ImageName;
93  DWORD BasePriority;
94  HANDLE ProcessId;
95  HANDLE ParentProcessId;
96  ULONG HandleCount;
97  ULONG Reserved2[2];
98  VM_COUNTERS VMCounters;
99  IO_COUNTERS IOCounters;
100  SYSTEM_THREAD Threads[1];
101 }; // SYSTEM_PROCESS_INFORMATION
102 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
103 
104 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
107 #if KMP_ARCH_X86
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
113 #else
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
119 #endif
120 
121 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
122  PVOID, ULONG, PULONG);
123 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
124 
125 HMODULE ntdll = NULL;
126 
127 /* End of NtQuerySystemInformation()-related code */
128 
129 static HMODULE kernel32 = NULL;
130 
131 #if KMP_HANDLE_SIGNALS
132 typedef void (*sig_func_t)(int);
133 static sig_func_t __kmp_sighldrs[NSIG];
134 static int __kmp_siginstalled[NSIG];
135 #endif
136 
137 #if KMP_USE_MONITOR
138 static HANDLE __kmp_monitor_ev;
139 #endif
140 static kmp_int64 __kmp_win32_time;
141 double __kmp_win32_tick;
142 
143 int __kmp_init_runtime = FALSE;
144 CRITICAL_SECTION __kmp_win32_section;
145 
146 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
147  InitializeCriticalSection(&mx->cs);
148 #if USE_ITT_BUILD
149  __kmp_itt_system_object_created(&mx->cs, "Critical Section");
150 #endif /* USE_ITT_BUILD */
151 }
152 
153 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
154  DeleteCriticalSection(&mx->cs);
155 }
156 
157 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
158  EnterCriticalSection(&mx->cs);
159 }
160 
161 int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
162  return TryEnterCriticalSection(&mx->cs);
163 }
164 
165 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
166  LeaveCriticalSection(&mx->cs);
167 }
168 
169 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
170  cv->waiters_count_ = 0;
171  cv->wait_generation_count_ = 0;
172  cv->release_count_ = 0;
173 
174  /* Initialize the critical section */
175  __kmp_win32_mutex_init(&cv->waiters_count_lock_);
176 
177  /* Create a manual-reset event. */
178  cv->event_ = CreateEvent(NULL, // no security
179  TRUE, // manual-reset
180  FALSE, // non-signaled initially
181  NULL); // unnamed
182 #if USE_ITT_BUILD
183  __kmp_itt_system_object_created(cv->event_, "Event");
184 #endif /* USE_ITT_BUILD */
185 }
186 
187 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
188  __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
189  __kmp_free_handle(cv->event_);
190  memset(cv, '\0', sizeof(*cv));
191 }
192 
193 /* TODO associate cv with a team instead of a thread so as to optimize
194  the case where we wake up a whole team */
195 
196 template <class C>
197 static void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
198  kmp_info_t *th, C *flag) {
199  int my_generation;
200  int last_waiter;
201 
202  /* Avoid race conditions */
203  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
204 
205  /* Increment count of waiters */
206  cv->waiters_count_++;
207 
208  /* Store current generation in our activation record. */
209  my_generation = cv->wait_generation_count_;
210 
211  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
212  __kmp_win32_mutex_unlock(mx);
213 
214  for (;;) {
215  int wait_done = 0;
216  DWORD res, timeout = 5000; // just tried to quess an appropriate number
217  /* Wait until the event is signaled */
218  res = WaitForSingleObject(cv->event_, timeout);
219 
220  if (res == WAIT_OBJECT_0) {
221  // event signaled
222  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
223  /* Exit the loop when the <cv->event_> is signaled and there are still
224  waiting threads from this <wait_generation> that haven't been released
225  from this wait yet. */
226  wait_done = (cv->release_count_ > 0) &&
227  (cv->wait_generation_count_ != my_generation);
228  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
229  } else if (res == WAIT_TIMEOUT || res == WAIT_FAILED) {
230  // check if the flag and cv counters are in consistent state
231  // as MS sent us debug dump whith inconsistent state of data
232  __kmp_win32_mutex_lock(mx);
233  typename C::flag_t old_f = flag->set_sleeping();
234  if (!flag->done_check_val(old_f & ~KMP_BARRIER_SLEEP_STATE)) {
235  __kmp_win32_mutex_unlock(mx);
236  continue;
237  }
238  // condition fulfilled, exiting
239  old_f = flag->unset_sleeping();
240  KMP_DEBUG_ASSERT(old_f & KMP_BARRIER_SLEEP_STATE);
241  TCW_PTR(th->th.th_sleep_loc, NULL);
242  KF_TRACE(50, ("__kmp_win32_cond_wait: exiting, condition "
243  "fulfilled: flag's loc(%p): %u => %u\n",
244  flag->get(), old_f, *(flag->get())));
245 
246  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
247  KMP_DEBUG_ASSERT(cv->waiters_count_ > 0);
248  cv->release_count_ = cv->waiters_count_;
249  cv->wait_generation_count_++;
250  wait_done = 1;
251  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
252 
253  __kmp_win32_mutex_unlock(mx);
254  }
255  /* there used to be a semicolon after the if statement, it looked like a
256  bug, so i removed it */
257  if (wait_done)
258  break;
259  }
260 
261  __kmp_win32_mutex_lock(mx);
262  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
263 
264  cv->waiters_count_--;
265  cv->release_count_--;
266 
267  last_waiter = (cv->release_count_ == 0);
268 
269  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
270 
271  if (last_waiter) {
272  /* We're the last waiter to be notified, so reset the manual event. */
273  ResetEvent(cv->event_);
274  }
275 }
276 
277 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
278  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
279 
280  if (cv->waiters_count_ > 0) {
281  SetEvent(cv->event_);
282  /* Release all the threads in this generation. */
283 
284  cv->release_count_ = cv->waiters_count_;
285 
286  /* Start a new generation. */
287  cv->wait_generation_count_++;
288  }
289 
290  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
291 }
292 
293 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
294  __kmp_win32_cond_broadcast(cv);
295 }
296 
297 void __kmp_enable(int new_state) {
298  if (__kmp_init_runtime)
299  LeaveCriticalSection(&__kmp_win32_section);
300 }
301 
302 void __kmp_disable(int *old_state) {
303  *old_state = 0;
304 
305  if (__kmp_init_runtime)
306  EnterCriticalSection(&__kmp_win32_section);
307 }
308 
309 void __kmp_suspend_initialize(void) { /* do nothing */
310 }
311 
312 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
313  int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init);
314  int new_value = TRUE;
315  // Return if already initialized
316  if (old_value == new_value)
317  return;
318  // Wait, then return if being initialized
319  if (old_value == -1 ||
320  !__kmp_atomic_compare_store(&th->th.th_suspend_init, old_value, -1)) {
321  while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init) != new_value) {
322  KMP_CPU_PAUSE();
323  }
324  } else {
325  // Claim to be the initializer and do initializations
326  __kmp_win32_cond_init(&th->th.th_suspend_cv);
327  __kmp_win32_mutex_init(&th->th.th_suspend_mx);
328  KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, new_value);
329  }
330 }
331 
332 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
333  if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init)) {
334  /* this means we have initialize the suspension pthread objects for this
335  thread in this instance of the process */
336  __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
337  __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
338  KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, FALSE);
339  }
340 }
341 
342 int __kmp_try_suspend_mx(kmp_info_t *th) {
343  return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
344 }
345 
346 void __kmp_lock_suspend_mx(kmp_info_t *th) {
347  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
348 }
349 
350 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
351  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
352 }
353 
354 /* This routine puts the calling thread to sleep after setting the
355  sleep bit for the indicated flag variable to true. */
356 template <class C>
357 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
358  kmp_info_t *th = __kmp_threads[th_gtid];
359  int status;
360  typename C::flag_t old_spin;
361 
362  KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
363  th_gtid, flag->get()));
364 
365  __kmp_suspend_initialize_thread(th);
366  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
367 
368  KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
369  " loc(%p)\n",
370  th_gtid, flag->get()));
371 
372  /* TODO: shouldn't this use release semantics to ensure that
373  __kmp_suspend_initialize_thread gets called first? */
374  old_spin = flag->set_sleeping();
375 #if OMP_50_ENABLED
376  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
377  __kmp_pause_status != kmp_soft_paused) {
378  flag->unset_sleeping();
379  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
380  return;
381  }
382 #endif
383 
384  KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
385  " loc(%p)==%d\n",
386  th_gtid, flag->get(), *(flag->get())));
387 
388  if (flag->done_check_val(old_spin)) {
389  old_spin = flag->unset_sleeping();
390  KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
391  "for flag's loc(%p)\n",
392  th_gtid, flag->get()));
393  } else {
394 #ifdef DEBUG_SUSPEND
395  __kmp_suspend_count++;
396 #endif
397  /* Encapsulate in a loop as the documentation states that this may "with
398  low probability" return when the condition variable has not been signaled
399  or broadcast */
400  int deactivated = FALSE;
401  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
402  while (flag->is_sleeping()) {
403  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
404  "kmp_win32_cond_wait()\n",
405  th_gtid));
406  // Mark the thread as no longer active (only in the first iteration of the
407  // loop).
408  if (!deactivated) {
409  th->th.th_active = FALSE;
410  if (th->th.th_active_in_pool) {
411  th->th.th_active_in_pool = FALSE;
412  KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
413  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
414  }
415  deactivated = TRUE;
416  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
417  flag);
418  } else {
419  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
420  flag);
421  }
422 
423 #ifdef KMP_DEBUG
424  if (flag->is_sleeping()) {
425  KF_TRACE(100,
426  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
427  }
428 #endif /* KMP_DEBUG */
429 
430  } // while
431 
432  // Mark the thread as active again (if it was previous marked as inactive)
433  if (deactivated) {
434  th->th.th_active = TRUE;
435  if (TCR_4(th->th.th_in_pool)) {
436  KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
437  th->th.th_active_in_pool = TRUE;
438  }
439  }
440  }
441 
442  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
443 
444  KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
445 }
446 
447 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
448  __kmp_suspend_template(th_gtid, flag);
449 }
450 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
451  __kmp_suspend_template(th_gtid, flag);
452 }
453 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
454  __kmp_suspend_template(th_gtid, flag);
455 }
456 
457 /* This routine signals the thread specified by target_gtid to wake up
458  after setting the sleep bit indicated by the flag argument to FALSE */
459 template <class C>
460 static inline void __kmp_resume_template(int target_gtid, C *flag) {
461  kmp_info_t *th = __kmp_threads[target_gtid];
462  int status;
463 
464 #ifdef KMP_DEBUG
465  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
466 #endif
467 
468  KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
469  gtid, target_gtid));
470 
471  __kmp_suspend_initialize_thread(th);
472  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
473 
474  if (!flag) { // coming from __kmp_null_resume_wrapper
475  flag = (C *)th->th.th_sleep_loc;
476  }
477 
478  // First, check if the flag is null or its type has changed. If so, someone
479  // else woke it up.
480  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
481  // simply shows what
482  // flag was cast to
483  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
484  "awake: flag's loc(%p)\n",
485  gtid, target_gtid, NULL));
486  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
487  return;
488  } else {
489  typename C::flag_t old_spin = flag->unset_sleeping();
490  if (!flag->is_sleeping_val(old_spin)) {
491  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
492  "awake: flag's loc(%p): %u => %u\n",
493  gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
494  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
495  return;
496  }
497  }
498  TCW_PTR(th->th.th_sleep_loc, NULL);
499  KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
500  "bit for flag's loc(%p)\n",
501  gtid, target_gtid, flag->get()));
502 
503  __kmp_win32_cond_signal(&th->th.th_suspend_cv);
504  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
505 
506  KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
507  " for T#%d\n",
508  gtid, target_gtid));
509 }
510 
511 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
512  __kmp_resume_template(target_gtid, flag);
513 }
514 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
515  __kmp_resume_template(target_gtid, flag);
516 }
517 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
518  __kmp_resume_template(target_gtid, flag);
519 }
520 
521 void __kmp_yield() { Sleep(0); }
522 
523 void __kmp_gtid_set_specific(int gtid) {
524  if (__kmp_init_gtid) {
525  KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
526  __kmp_gtid_threadprivate_key));
527  if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
528  KMP_FATAL(TLSSetValueFailed);
529  } else {
530  KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
531  }
532 }
533 
534 int __kmp_gtid_get_specific() {
535  int gtid;
536  if (!__kmp_init_gtid) {
537  KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
538  "KMP_GTID_SHUTDOWN\n"));
539  return KMP_GTID_SHUTDOWN;
540  }
541  gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
542  if (gtid == 0) {
543  gtid = KMP_GTID_DNE;
544  } else {
545  gtid--;
546  }
547  KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
548  __kmp_gtid_threadprivate_key, gtid));
549  return gtid;
550 }
551 
552 void __kmp_affinity_bind_thread(int proc) {
553  if (__kmp_num_proc_groups > 1) {
554  // Form the GROUP_AFFINITY struct directly, rather than filling
555  // out a bit vector and calling __kmp_set_system_affinity().
556  GROUP_AFFINITY ga;
557  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
558  sizeof(DWORD_PTR))));
559  ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
560  ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
561  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
562 
563  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
564  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
565  DWORD error = GetLastError();
566  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
567  kmp_msg_t err_code = KMP_ERR(error);
568  __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
569  __kmp_msg_null);
570  if (__kmp_generate_warnings == kmp_warnings_off) {
571  __kmp_str_free(&err_code.str);
572  }
573  }
574  }
575  } else {
576  kmp_affin_mask_t *mask;
577  KMP_CPU_ALLOC_ON_STACK(mask);
578  KMP_CPU_ZERO(mask);
579  KMP_CPU_SET(proc, mask);
580  __kmp_set_system_affinity(mask, TRUE);
581  KMP_CPU_FREE_FROM_STACK(mask);
582  }
583 }
584 
585 void __kmp_affinity_determine_capable(const char *env_var) {
586 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
587 
588 #if KMP_GROUP_AFFINITY
589  KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
590 #else
591  KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
592 #endif
593 
594  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
595  "Windows* OS affinity interface functional (mask size = "
596  "%" KMP_SIZE_T_SPEC ").\n",
597  __kmp_affin_mask_size));
598 }
599 
600 double __kmp_read_cpu_time(void) {
601  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
602  int status;
603  double cpu_time;
604 
605  cpu_time = 0;
606 
607  status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
608  &KernelTime, &UserTime);
609 
610  if (status) {
611  double sec = 0;
612 
613  sec += KernelTime.dwHighDateTime;
614  sec += UserTime.dwHighDateTime;
615 
616  /* Shift left by 32 bits */
617  sec *= (double)(1 << 16) * (double)(1 << 16);
618 
619  sec += KernelTime.dwLowDateTime;
620  sec += UserTime.dwLowDateTime;
621 
622  cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
623  }
624 
625  return cpu_time;
626 }
627 
628 int __kmp_read_system_info(struct kmp_sys_info *info) {
629  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
630  info->minflt = 0; /* the number of page faults serviced without any I/O */
631  info->majflt = 0; /* the number of page faults serviced that required I/O */
632  info->nswap = 0; // the number of times a process was "swapped" out of memory
633  info->inblock = 0; // the number of times the file system had to perform input
634  info->oublock = 0; // number of times the file system had to perform output
635  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
636  info->nivcsw = 0; /* the number of times a context switch was forced */
637 
638  return 1;
639 }
640 
641 void __kmp_runtime_initialize(void) {
642  SYSTEM_INFO info;
643  kmp_str_buf_t path;
644  UINT path_size;
645 
646  if (__kmp_init_runtime) {
647  return;
648  }
649 
650 #if KMP_DYNAMIC_LIB
651  /* Pin dynamic library for the lifetime of application */
652  {
653  // First, turn off error message boxes
654  UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
655  HMODULE h;
656  BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
657  GET_MODULE_HANDLE_EX_FLAG_PIN,
658  (LPCTSTR)&__kmp_serial_initialize, &h);
659  KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
660  SetErrorMode(err_mode); // Restore error mode
661  KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
662  }
663 #endif
664 
665  InitializeCriticalSection(&__kmp_win32_section);
666 #if USE_ITT_BUILD
667  __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
668 #endif /* USE_ITT_BUILD */
669  __kmp_initialize_system_tick();
670 
671 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
672  if (!__kmp_cpuinfo.initialized) {
673  __kmp_query_cpuid(&__kmp_cpuinfo);
674  }
675 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
676 
677 /* Set up minimum number of threads to switch to TLS gtid */
678 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
679  // Windows* OS, static library.
680  /* New thread may use stack space previously used by another thread,
681  currently terminated. On Windows* OS, in case of static linking, we do not
682  know the moment of thread termination, and our structures (__kmp_threads
683  and __kmp_root arrays) are still keep info about dead threads. This leads
684  to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
685  (by searching through stack addresses of all known threads) for
686  unregistered foreign tread.
687 
688  Setting __kmp_tls_gtid_min to 0 workarounds this problem:
689  __kmp_get_global_thread_id() does not search through stacks, but get gtid
690  from TLS immediately.
691  --ln
692  */
693  __kmp_tls_gtid_min = 0;
694 #else
695  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
696 #endif
697 
698  /* for the static library */
699  if (!__kmp_gtid_threadprivate_key) {
700  __kmp_gtid_threadprivate_key = TlsAlloc();
701  if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
702  KMP_FATAL(TLSOutOfIndexes);
703  }
704  }
705 
706  // Load ntdll.dll.
707  /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
708  (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
709  have to specify full path to the library. */
710  __kmp_str_buf_init(&path);
711  path_size = GetSystemDirectory(path.str, path.size);
712  KMP_DEBUG_ASSERT(path_size > 0);
713  if (path_size >= path.size) {
714  // Buffer is too short. Expand the buffer and try again.
715  __kmp_str_buf_reserve(&path, path_size);
716  path_size = GetSystemDirectory(path.str, path.size);
717  KMP_DEBUG_ASSERT(path_size > 0);
718  }
719  if (path_size > 0 && path_size < path.size) {
720  // Now we have system directory name in the buffer.
721  // Append backslash and name of dll to form full path,
722  path.used = path_size;
723  __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
724 
725  // Now load ntdll using full path.
726  ntdll = GetModuleHandle(path.str);
727  }
728 
729  KMP_DEBUG_ASSERT(ntdll != NULL);
730  if (ntdll != NULL) {
731  NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
732  ntdll, "NtQuerySystemInformation");
733  }
734  KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
735 
736 #if KMP_GROUP_AFFINITY
737  // Load kernel32.dll.
738  // Same caveat - must use full system path name.
739  if (path_size > 0 && path_size < path.size) {
740  // Truncate the buffer back to just the system path length,
741  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
742  path.used = path_size;
743  __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
744 
745  // Load kernel32.dll using full path.
746  kernel32 = GetModuleHandle(path.str);
747  KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
748 
749  // Load the function pointers to kernel32.dll routines
750  // that may or may not exist on this system.
751  if (kernel32 != NULL) {
752  __kmp_GetActiveProcessorCount =
753  (kmp_GetActiveProcessorCount_t)GetProcAddress(
754  kernel32, "GetActiveProcessorCount");
755  __kmp_GetActiveProcessorGroupCount =
756  (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
757  kernel32, "GetActiveProcessorGroupCount");
758  __kmp_GetThreadGroupAffinity =
759  (kmp_GetThreadGroupAffinity_t)GetProcAddress(
760  kernel32, "GetThreadGroupAffinity");
761  __kmp_SetThreadGroupAffinity =
762  (kmp_SetThreadGroupAffinity_t)GetProcAddress(
763  kernel32, "SetThreadGroupAffinity");
764 
765  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
766  " = %p\n",
767  __kmp_GetActiveProcessorCount));
768  KA_TRACE(10, ("__kmp_runtime_initialize: "
769  "__kmp_GetActiveProcessorGroupCount = %p\n",
770  __kmp_GetActiveProcessorGroupCount));
771  KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
772  " = %p\n",
773  __kmp_GetThreadGroupAffinity));
774  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
775  " = %p\n",
776  __kmp_SetThreadGroupAffinity));
777  KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
778  sizeof(kmp_affin_mask_t)));
779 
780  // See if group affinity is supported on this system.
781  // If so, calculate the #groups and #procs.
782  //
783  // Group affinity was introduced with Windows* 7 OS and
784  // Windows* Server 2008 R2 OS.
785  if ((__kmp_GetActiveProcessorCount != NULL) &&
786  (__kmp_GetActiveProcessorGroupCount != NULL) &&
787  (__kmp_GetThreadGroupAffinity != NULL) &&
788  (__kmp_SetThreadGroupAffinity != NULL) &&
789  ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
790  1)) {
791  // Calculate the total number of active OS procs.
792  int i;
793 
794  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
795  " detected\n",
796  __kmp_num_proc_groups));
797 
798  __kmp_xproc = 0;
799 
800  for (i = 0; i < __kmp_num_proc_groups; i++) {
801  DWORD size = __kmp_GetActiveProcessorCount(i);
802  __kmp_xproc += size;
803  KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
804  i, size));
805  }
806  } else {
807  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
808  " detected\n",
809  __kmp_num_proc_groups));
810  }
811  }
812  }
813  if (__kmp_num_proc_groups <= 1) {
814  GetSystemInfo(&info);
815  __kmp_xproc = info.dwNumberOfProcessors;
816  }
817 #else
818  GetSystemInfo(&info);
819  __kmp_xproc = info.dwNumberOfProcessors;
820 #endif /* KMP_GROUP_AFFINITY */
821 
822  // If the OS said there were 0 procs, take a guess and use a value of 2.
823  // This is done for Linux* OS, also. Do we need error / warning?
824  if (__kmp_xproc <= 0) {
825  __kmp_xproc = 2;
826  }
827 
828  KA_TRACE(5,
829  ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
830 
831  __kmp_str_buf_free(&path);
832 
833 #if USE_ITT_BUILD
834  __kmp_itt_initialize();
835 #endif /* USE_ITT_BUILD */
836 
837  __kmp_init_runtime = TRUE;
838 } // __kmp_runtime_initialize
839 
840 void __kmp_runtime_destroy(void) {
841  if (!__kmp_init_runtime) {
842  return;
843  }
844 
845 #if USE_ITT_BUILD
846  __kmp_itt_destroy();
847 #endif /* USE_ITT_BUILD */
848 
849  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
850  /* due to the KX_TRACE() commands */
851  KA_TRACE(40, ("__kmp_runtime_destroy\n"));
852 
853  if (__kmp_gtid_threadprivate_key) {
854  TlsFree(__kmp_gtid_threadprivate_key);
855  __kmp_gtid_threadprivate_key = 0;
856  }
857 
858  __kmp_affinity_uninitialize();
859  DeleteCriticalSection(&__kmp_win32_section);
860 
861  ntdll = NULL;
862  NtQuerySystemInformation = NULL;
863 
864 #if KMP_ARCH_X86_64
865  kernel32 = NULL;
866  __kmp_GetActiveProcessorCount = NULL;
867  __kmp_GetActiveProcessorGroupCount = NULL;
868  __kmp_GetThreadGroupAffinity = NULL;
869  __kmp_SetThreadGroupAffinity = NULL;
870 #endif // KMP_ARCH_X86_64
871 
872  __kmp_init_runtime = FALSE;
873 }
874 
875 void __kmp_terminate_thread(int gtid) {
876  kmp_info_t *th = __kmp_threads[gtid];
877 
878  if (!th)
879  return;
880 
881  KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
882 
883  if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
884  /* It's OK, the thread may have exited already */
885  }
886  __kmp_free_handle(th->th.th_info.ds.ds_thread);
887 }
888 
889 void __kmp_clear_system_time(void) {
890  BOOL status;
891  LARGE_INTEGER time;
892  status = QueryPerformanceCounter(&time);
893  __kmp_win32_time = (kmp_int64)time.QuadPart;
894 }
895 
896 void __kmp_initialize_system_tick(void) {
897  {
898  BOOL status;
899  LARGE_INTEGER freq;
900 
901  status = QueryPerformanceFrequency(&freq);
902  if (!status) {
903  DWORD error = GetLastError();
904  __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
905  KMP_ERR(error), __kmp_msg_null);
906 
907  } else {
908  __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
909  }
910  }
911 }
912 
913 /* Calculate the elapsed wall clock time for the user */
914 
915 void __kmp_elapsed(double *t) {
916  BOOL status;
917  LARGE_INTEGER now;
918  status = QueryPerformanceCounter(&now);
919  *t = ((double)now.QuadPart) * __kmp_win32_tick;
920 }
921 
922 /* Calculate the elapsed wall clock tick for the user */
923 
924 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
925 
926 void __kmp_read_system_time(double *delta) {
927  if (delta != NULL) {
928  BOOL status;
929  LARGE_INTEGER now;
930 
931  status = QueryPerformanceCounter(&now);
932 
933  *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
934  __kmp_win32_tick;
935  }
936 }
937 
938 /* Return the current time stamp in nsec */
939 kmp_uint64 __kmp_now_nsec() {
940  LARGE_INTEGER now;
941  QueryPerformanceCounter(&now);
942  return 1e9 * __kmp_win32_tick * now.QuadPart;
943 }
944 
945 extern "C"
946 void *__stdcall __kmp_launch_worker(void *arg) {
947  volatile void *stack_data;
948  void *exit_val;
949  void *padding = 0;
950  kmp_info_t *this_thr = (kmp_info_t *)arg;
951  int gtid;
952 
953  gtid = this_thr->th.th_info.ds.ds_gtid;
954  __kmp_gtid_set_specific(gtid);
955 #ifdef KMP_TDATA_GTID
956 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
957  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
958  "reference: http://support.microsoft.com/kb/118816"
959 //__kmp_gtid = gtid;
960 #endif
961 
962 #if USE_ITT_BUILD
963  __kmp_itt_thread_name(gtid);
964 #endif /* USE_ITT_BUILD */
965 
966  __kmp_affinity_set_init_mask(gtid, FALSE);
967 
968 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
969  // Set FP control regs to be a copy of the parallel initialization thread's.
970  __kmp_clear_x87_fpu_status_word();
971  __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
972  __kmp_load_mxcsr(&__kmp_init_mxcsr);
973 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
974 
975  if (__kmp_stkoffset > 0 && gtid > 0) {
976  padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
977  }
978 
979  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
980  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
981  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
982 
983  if (TCR_4(__kmp_gtid_mode) <
984  2) { // check stack only if it is used to get gtid
985  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
986  KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
987  __kmp_check_stack_overlap(this_thr);
988  }
989  KMP_MB();
990  exit_val = __kmp_launch_thread(this_thr);
991  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
992  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
993  KMP_MB();
994  return exit_val;
995 }
996 
997 #if KMP_USE_MONITOR
998 /* The monitor thread controls all of the threads in the complex */
999 
1000 void *__stdcall __kmp_launch_monitor(void *arg) {
1001  DWORD wait_status;
1002  kmp_thread_t monitor;
1003  int status;
1004  int interval;
1005  kmp_info_t *this_thr = (kmp_info_t *)arg;
1006 
1007  KMP_DEBUG_ASSERT(__kmp_init_monitor);
1008  TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
1009  // TODO: hide "2" in enum (like {true,false,started})
1010  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1011  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
1012 
1013  KMP_MB(); /* Flush all pending memory write invalidates. */
1014  KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
1015 
1016  monitor = GetCurrentThread();
1017 
1018  /* set thread priority */
1019  status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
1020  if (!status) {
1021  DWORD error = GetLastError();
1022  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1023  }
1024 
1025  /* register us as monitor */
1026  __kmp_gtid_set_specific(KMP_GTID_MONITOR);
1027 #ifdef KMP_TDATA_GTID
1028 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1029  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1030  "reference: http://support.microsoft.com/kb/118816"
1031 //__kmp_gtid = KMP_GTID_MONITOR;
1032 #endif
1033 
1034 #if USE_ITT_BUILD
1035  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
1036 // monitor thread.
1037 #endif /* USE_ITT_BUILD */
1038 
1039  KMP_MB(); /* Flush all pending memory write invalidates. */
1040 
1041  interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
1042 
1043  while (!TCR_4(__kmp_global.g.g_done)) {
1044  /* This thread monitors the state of the system */
1045 
1046  KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
1047 
1048  wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
1049 
1050  if (wait_status == WAIT_TIMEOUT) {
1051  TCW_4(__kmp_global.g.g_time.dt.t_value,
1052  TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1053  }
1054 
1055  KMP_MB(); /* Flush all pending memory write invalidates. */
1056  }
1057 
1058  KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1059 
1060  status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1061  if (!status) {
1062  DWORD error = GetLastError();
1063  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1064  }
1065 
1066  if (__kmp_global.g.g_abort != 0) {
1067  /* now we need to terminate the worker threads */
1068  /* the value of t_abort is the signal we caught */
1069  int gtid;
1070 
1071  KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1072  (__kmp_global.g.g_abort)));
1073 
1074  /* terminate the OpenMP worker threads */
1075  /* TODO this is not valid for sibling threads!!
1076  * the uber master might not be 0 anymore.. */
1077  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1078  __kmp_terminate_thread(gtid);
1079 
1080  __kmp_cleanup();
1081 
1082  Sleep(0);
1083 
1084  KA_TRACE(10,
1085  ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1086 
1087  if (__kmp_global.g.g_abort > 0) {
1088  raise(__kmp_global.g.g_abort);
1089  }
1090  }
1091 
1092  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1093 
1094  KMP_MB();
1095  return arg;
1096 }
1097 #endif
1098 
1099 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1100  kmp_thread_t handle;
1101  DWORD idThread;
1102 
1103  KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1104 
1105  th->th.th_info.ds.ds_gtid = gtid;
1106 
1107  if (KMP_UBER_GTID(gtid)) {
1108  int stack_data;
1109 
1110  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1111  other threads to use. Is it appropriate to just use GetCurrentThread?
1112  When should we close this handle? When unregistering the root? */
1113  {
1114  BOOL rc;
1115  rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1116  GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1117  FALSE, DUPLICATE_SAME_ACCESS);
1118  KMP_ASSERT(rc);
1119  KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1120  "handle = %" KMP_UINTPTR_SPEC "\n",
1121  (LPVOID)th, th->th.th_info.ds.ds_thread));
1122  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1123  }
1124  if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1125  /* we will dynamically update the stack range if gtid_mode == 1 */
1126  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1127  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1128  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1129  __kmp_check_stack_overlap(th);
1130  }
1131  } else {
1132  KMP_MB(); /* Flush all pending memory write invalidates. */
1133 
1134  /* Set stack size for this thread now. */
1135  KA_TRACE(10,
1136  ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1137  stack_size));
1138 
1139  stack_size += gtid * __kmp_stkoffset;
1140 
1141  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1142  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1143 
1144  KA_TRACE(10,
1145  ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1146  " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1147  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1148  (LPVOID)th, &idThread));
1149 
1150  handle = CreateThread(
1151  NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1152  (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1153 
1154  KA_TRACE(10,
1155  ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1156  " bytes, &__kmp_launch_worker = %p, th = %p, "
1157  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1158  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1159  (LPVOID)th, idThread, handle));
1160 
1161  if (handle == 0) {
1162  DWORD error = GetLastError();
1163  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1164  } else {
1165  th->th.th_info.ds.ds_thread = handle;
1166  }
1167 
1168  KMP_MB(); /* Flush all pending memory write invalidates. */
1169  }
1170 
1171  KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1172 }
1173 
1174 int __kmp_still_running(kmp_info_t *th) {
1175  return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1176 }
1177 
1178 #if KMP_USE_MONITOR
1179 void __kmp_create_monitor(kmp_info_t *th) {
1180  kmp_thread_t handle;
1181  DWORD idThread;
1182  int ideal, new_ideal;
1183 
1184  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1185  // We don't need monitor thread in case of MAX_BLOCKTIME
1186  KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1187  "MAX blocktime\n"));
1188  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1189  th->th.th_info.ds.ds_gtid = 0;
1190  TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1191  return;
1192  }
1193  KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1194 
1195  KMP_MB(); /* Flush all pending memory write invalidates. */
1196 
1197  __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1198  if (__kmp_monitor_ev == NULL) {
1199  DWORD error = GetLastError();
1200  __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1201  }
1202 #if USE_ITT_BUILD
1203  __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1204 #endif /* USE_ITT_BUILD */
1205 
1206  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1207  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1208 
1209  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1210  // to automatically expand stacksize based on CreateThread error code.
1211  if (__kmp_monitor_stksize == 0) {
1212  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1213  }
1214  if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1215  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1216  }
1217 
1218  KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1219  (int)__kmp_monitor_stksize));
1220 
1221  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1222 
1223  handle =
1224  CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1225  (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1226  STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1227  if (handle == 0) {
1228  DWORD error = GetLastError();
1229  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1230  } else
1231  th->th.th_info.ds.ds_thread = handle;
1232 
1233  KMP_MB(); /* Flush all pending memory write invalidates. */
1234 
1235  KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1236  (void *)th->th.th_info.ds.ds_thread));
1237 }
1238 #endif
1239 
1240 /* Check to see if thread is still alive.
1241  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1242  with a exit_val = code. Because of this we can not rely on exit_val having
1243  any particular value. So this routine may return STILL_ALIVE in exit_val
1244  even after the thread is dead. */
1245 
1246 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1247  DWORD rc;
1248  rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1249  if (rc == 0) {
1250  DWORD error = GetLastError();
1251  __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
1252  __kmp_msg_null);
1253  }
1254  return (*exit_val == STILL_ACTIVE);
1255 }
1256 
1257 void __kmp_exit_thread(int exit_status) {
1258  ExitThread(exit_status);
1259 } // __kmp_exit_thread
1260 
1261 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1262 static void __kmp_reap_common(kmp_info_t *th) {
1263  DWORD exit_val;
1264 
1265  KMP_MB(); /* Flush all pending memory write invalidates. */
1266 
1267  KA_TRACE(
1268  10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1269 
1270  /* 2006-10-19:
1271  There are two opposite situations:
1272  1. Windows* OS keep thread alive after it resets ds_alive flag and
1273  exits from thread function. (For example, see C70770/Q394281 "unloading of
1274  dll based on OMP is very slow".)
1275  2. Windows* OS may kill thread before it resets ds_alive flag.
1276 
1277  Right solution seems to be waiting for *either* thread termination *or*
1278  ds_alive resetting. */
1279  {
1280  // TODO: This code is very similar to KMP_WAIT. Need to generalize
1281  // KMP_WAIT to cover this usage also.
1282  void *obj = NULL;
1283  kmp_uint32 spins;
1284 #if USE_ITT_BUILD
1285  KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1286 #endif /* USE_ITT_BUILD */
1287  KMP_INIT_YIELD(spins);
1288  do {
1289 #if USE_ITT_BUILD
1290  KMP_FSYNC_SPIN_PREPARE(obj);
1291 #endif /* USE_ITT_BUILD */
1292  __kmp_is_thread_alive(th, &exit_val);
1293  KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
1294  } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1295 #if USE_ITT_BUILD
1296  if (exit_val == STILL_ACTIVE) {
1297  KMP_FSYNC_CANCEL(obj);
1298  } else {
1299  KMP_FSYNC_SPIN_ACQUIRED(obj);
1300  }
1301 #endif /* USE_ITT_BUILD */
1302  }
1303 
1304  __kmp_free_handle(th->th.th_info.ds.ds_thread);
1305 
1306  /* NOTE: The ExitProcess(code) system call causes all threads to Terminate
1307  with a exit_val = code. Because of this we can not rely on exit_val having
1308  any particular value. */
1309  if (exit_val == STILL_ACTIVE) {
1310  KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1311  } else if ((void *)exit_val != (void *)th) {
1312  KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1313  }
1314 
1315  KA_TRACE(10,
1316  ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1317  "\n",
1318  th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1319 
1320  th->th.th_info.ds.ds_thread = 0;
1321  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1322  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1323  th->th.th_info.ds.ds_thread_id = 0;
1324 
1325  KMP_MB(); /* Flush all pending memory write invalidates. */
1326 }
1327 
1328 #if KMP_USE_MONITOR
1329 void __kmp_reap_monitor(kmp_info_t *th) {
1330  int status;
1331 
1332  KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1333  (void *)th->th.th_info.ds.ds_thread));
1334 
1335  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1336  // If both tid and gtid are 0, it means the monitor did not ever start.
1337  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1338  KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1339  if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1340  KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1341  return;
1342  }
1343 
1344  KMP_MB(); /* Flush all pending memory write invalidates. */
1345 
1346  status = SetEvent(__kmp_monitor_ev);
1347  if (status == FALSE) {
1348  DWORD error = GetLastError();
1349  __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1350  }
1351  KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1352  th->th.th_info.ds.ds_gtid));
1353  __kmp_reap_common(th);
1354 
1355  __kmp_free_handle(__kmp_monitor_ev);
1356 
1357  KMP_MB(); /* Flush all pending memory write invalidates. */
1358 }
1359 #endif
1360 
1361 void __kmp_reap_worker(kmp_info_t *th) {
1362  KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1363  th->th.th_info.ds.ds_gtid));
1364  __kmp_reap_common(th);
1365 }
1366 
1367 #if KMP_HANDLE_SIGNALS
1368 
1369 static void __kmp_team_handler(int signo) {
1370  if (__kmp_global.g.g_abort == 0) {
1371  // Stage 1 signal handler, let's shut down all of the threads.
1372  if (__kmp_debug_buf) {
1373  __kmp_dump_debug_buffer();
1374  }
1375  KMP_MB(); // Flush all pending memory write invalidates.
1376  TCW_4(__kmp_global.g.g_abort, signo);
1377  KMP_MB(); // Flush all pending memory write invalidates.
1378  TCW_4(__kmp_global.g.g_done, TRUE);
1379  KMP_MB(); // Flush all pending memory write invalidates.
1380  }
1381 } // __kmp_team_handler
1382 
1383 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1384  sig_func_t old = signal(signum, handler);
1385  if (old == SIG_ERR) {
1386  int error = errno;
1387  __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1388  __kmp_msg_null);
1389  }
1390  return old;
1391 }
1392 
1393 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1394  int parallel_init) {
1395  sig_func_t old;
1396  KMP_MB(); /* Flush all pending memory write invalidates. */
1397  KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1398  if (parallel_init) {
1399  old = __kmp_signal(sig, handler);
1400  // SIG_DFL on Windows* OS in NULL or 0.
1401  if (old == __kmp_sighldrs[sig]) {
1402  __kmp_siginstalled[sig] = 1;
1403  } else { // Restore/keep user's handler if one previously installed.
1404  old = __kmp_signal(sig, old);
1405  }
1406  } else {
1407  // Save initial/system signal handlers to see if user handlers installed.
1408  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1409  // called once with parallel_init == TRUE.
1410  old = __kmp_signal(sig, SIG_DFL);
1411  __kmp_sighldrs[sig] = old;
1412  __kmp_signal(sig, old);
1413  }
1414  KMP_MB(); /* Flush all pending memory write invalidates. */
1415 } // __kmp_install_one_handler
1416 
1417 static void __kmp_remove_one_handler(int sig) {
1418  if (__kmp_siginstalled[sig]) {
1419  sig_func_t old;
1420  KMP_MB(); // Flush all pending memory write invalidates.
1421  KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1422  old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1423  if (old != __kmp_team_handler) {
1424  KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1425  "restoring: sig=%d\n",
1426  sig));
1427  old = __kmp_signal(sig, old);
1428  }
1429  __kmp_sighldrs[sig] = NULL;
1430  __kmp_siginstalled[sig] = 0;
1431  KMP_MB(); // Flush all pending memory write invalidates.
1432  }
1433 } // __kmp_remove_one_handler
1434 
1435 void __kmp_install_signals(int parallel_init) {
1436  KB_TRACE(10, ("__kmp_install_signals: called\n"));
1437  if (!__kmp_handle_signals) {
1438  KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1439  "handlers not installed\n"));
1440  return;
1441  }
1442  __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1443  __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1444  __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1445  __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1446  __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1447  __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1448 } // __kmp_install_signals
1449 
1450 void __kmp_remove_signals(void) {
1451  int sig;
1452  KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1453  for (sig = 1; sig < NSIG; ++sig) {
1454  __kmp_remove_one_handler(sig);
1455  }
1456 } // __kmp_remove_signals
1457 
1458 #endif // KMP_HANDLE_SIGNALS
1459 
1460 /* Put the thread to sleep for a time period */
1461 void __kmp_thread_sleep(int millis) {
1462  DWORD status;
1463 
1464  status = SleepEx((DWORD)millis, FALSE);
1465  if (status) {
1466  DWORD error = GetLastError();
1467  __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1468  __kmp_msg_null);
1469  }
1470 }
1471 
1472 // Determine whether the given address is mapped into the current address space.
1473 int __kmp_is_address_mapped(void *addr) {
1474  DWORD status;
1475  MEMORY_BASIC_INFORMATION lpBuffer;
1476  SIZE_T dwLength;
1477 
1478  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1479 
1480  status = VirtualQuery(addr, &lpBuffer, dwLength);
1481 
1482  return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1483  ((lpBuffer.Protect == PAGE_NOACCESS) ||
1484  (lpBuffer.Protect == PAGE_EXECUTE)));
1485 }
1486 
1487 kmp_uint64 __kmp_hardware_timestamp(void) {
1488  kmp_uint64 r = 0;
1489 
1490  QueryPerformanceCounter((LARGE_INTEGER *)&r);
1491  return r;
1492 }
1493 
1494 /* Free handle and check the error code */
1495 void __kmp_free_handle(kmp_thread_t tHandle) {
1496  /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1497  * as HANDLE */
1498  BOOL rc;
1499  rc = CloseHandle(tHandle);
1500  if (!rc) {
1501  DWORD error = GetLastError();
1502  __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1503  }
1504 }
1505 
1506 int __kmp_get_load_balance(int max) {
1507  static ULONG glb_buff_size = 100 * 1024;
1508 
1509  // Saved count of the running threads for the thread balance algortihm
1510  static int glb_running_threads = 0;
1511  static double glb_call_time = 0; /* Thread balance algorithm call time */
1512 
1513  int running_threads = 0; // Number of running threads in the system.
1514  NTSTATUS status = 0;
1515  ULONG buff_size = 0;
1516  ULONG info_size = 0;
1517  void *buffer = NULL;
1518  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1519  int first_time = 1;
1520 
1521  double call_time = 0.0; // start, finish;
1522 
1523  __kmp_elapsed(&call_time);
1524 
1525  if (glb_call_time &&
1526  (call_time - glb_call_time < __kmp_load_balance_interval)) {
1527  running_threads = glb_running_threads;
1528  goto finish;
1529  }
1530  glb_call_time = call_time;
1531 
1532  // Do not spend time on running algorithm if we have a permanent error.
1533  if (NtQuerySystemInformation == NULL) {
1534  running_threads = -1;
1535  goto finish;
1536  }
1537 
1538  if (max <= 0) {
1539  max = INT_MAX;
1540  }
1541 
1542  do {
1543 
1544  if (first_time) {
1545  buff_size = glb_buff_size;
1546  } else {
1547  buff_size = 2 * buff_size;
1548  }
1549 
1550  buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1551  if (buffer == NULL) {
1552  running_threads = -1;
1553  goto finish;
1554  }
1555  status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1556  buff_size, &info_size);
1557  first_time = 0;
1558 
1559  } while (status == STATUS_INFO_LENGTH_MISMATCH);
1560  glb_buff_size = buff_size;
1561 
1562 #define CHECK(cond) \
1563  { \
1564  KMP_DEBUG_ASSERT(cond); \
1565  if (!(cond)) { \
1566  running_threads = -1; \
1567  goto finish; \
1568  } \
1569  }
1570 
1571  CHECK(buff_size >= info_size);
1572  spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1573  for (;;) {
1574  ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1575  CHECK(0 <= offset &&
1576  offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1577  HANDLE pid = spi->ProcessId;
1578  ULONG num = spi->NumberOfThreads;
1579  CHECK(num >= 1);
1580  size_t spi_size =
1581  sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1582  CHECK(offset + spi_size <
1583  info_size); // Make sure process info record fits the buffer.
1584  if (spi->NextEntryOffset != 0) {
1585  CHECK(spi_size <=
1586  spi->NextEntryOffset); // And do not overlap with the next record.
1587  }
1588  // pid == 0 corresponds to the System Idle Process. It always has running
1589  // threads on all cores. So, we don't consider the running threads of this
1590  // process.
1591  if (pid != 0) {
1592  for (int i = 0; i < num; ++i) {
1593  THREAD_STATE state = spi->Threads[i].State;
1594  // Count threads that have Ready or Running state.
1595  // !!! TODO: Why comment does not match the code???
1596  if (state == StateRunning) {
1597  ++running_threads;
1598  // Stop counting running threads if the number is already greater than
1599  // the number of available cores
1600  if (running_threads >= max) {
1601  goto finish;
1602  }
1603  }
1604  }
1605  }
1606  if (spi->NextEntryOffset == 0) {
1607  break;
1608  }
1609  spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1610  }
1611 
1612 #undef CHECK
1613 
1614 finish: // Clean up and exit.
1615 
1616  if (buffer != NULL) {
1617  KMP_INTERNAL_FREE(buffer);
1618  }
1619 
1620  glb_running_threads = running_threads;
1621 
1622  return running_threads;
1623 } //__kmp_get_load_balance()