/build/source/openmp/runtime/src/kmp

Bug Summary

File:	build/source/openmp/runtime/src/kmp_runtime.cpp
Warning:	line 2569, column 34 Dereference of null pointer
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name kmp_runtime.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16 -I projects/openmp/runtime/src -I /build/source/openmp/runtime/src -I include -I /build/source/llvm/include -I /build/source/openmp/runtime/src/i18n -I /build/source/openmp/runtime/src/include -I /build/source/openmp/runtime/src/thirdparty/ittnotify -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D omp_EXPORTS -D _FORTIFY_SOURCE=2 -D NDEBUG -D _GNU_SOURCE -D _REENTRANT -D _FORTIFY_SOURCE=2 -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1673561342 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -Wno-extra -Wno-pedantic -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-frame-address -Wno-strict-aliasing -Wno-stringop-truncation -Wno-switch -Wno-uninitialized -Wno-cast-qual -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-01-13-042150-16221-1 -x c++ /build/source/openmp/runtime/src/kmp_runtime.cpp
1/*
* kmp_runtime.cpp -- KPTS runtime support library
*/

5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//

13#include "kmp.h"
14#include "kmp_affinity.h"
15#include "kmp_atomic.h"
16#include "kmp_environment.h"
17#include "kmp_error.h"
18#include "kmp_i18n.h"
19#include "kmp_io.h"
20#include "kmp_itt.h"
21#include "kmp_settings.h"
22#include "kmp_stats.h"
23#include "kmp_str.h"
24#include "kmp_wait_release.h"
25#include "kmp_wrapper_getpid.h"
26#include "kmp_dispatch.h"
27#if KMP_USE_HIER_SCHED0
28#include "kmp_dispatch_hier.h"
29#endif

31#if OMPT_SUPPORT1
32#include "ompt-specific.h"
33#endif
34#if OMPD_SUPPORT1
35#include "ompd-specific.h"
36#endif

38#if OMP_PROFILING_SUPPORT0
39#include "llvm/Support/TimeProfiler.h"
40static char *ProfileTraceFile = nullptr;
41#endif

43/* these are temporary issues to be dealt with */
44#define KMP_USE_PRCTL0 0

46#if KMP_OS_WINDOWS0
47#include <process.h>
48#endif

50#if KMP_OS_WINDOWS0
51// windows does not need include files as it doesn't use shared memory
52#else
53#include <sys/mman.h>
54#include <sys/stat.h>
55#include <fcntl.h>
56#define SHM_SIZE1024 1024
57#endif

59#if defined(KMP_GOMP_COMPAT)
60char const __kmp_version_alt_comp[] =
  KMP_VERSION_PREFIX"\x00@(#) " "LLVM OMP " "alternative compiler support: yes";
62#endif /* defined(KMP_GOMP_COMPAT) */

64char const __kmp_version_omp_api[] =
  KMP_VERSION_PREFIX"\x00@(#) " "LLVM OMP " "API version: 5.0 (201611)";

67#ifdef KMP_DEBUG1
68char const __kmp_version_lock[] =
  KMP_VERSION_PREFIX"\x00@(#) " "LLVM OMP " "lock type: run time selectable";
70#endif /* KMP_DEBUG */

72#define KMP_MIN(x, y)((x) < (y) ? (x) : (y)) ((x) < (y) ? (x) : (y))

74/* ------------------------------------------------------------------------ */

76#if KMP_USE_MONITOR
77kmp_info_t __kmp_monitor;
78#endif

80/* Forward declarations */

82void __kmp_cleanup(void);

84static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
                                int gtid);
86static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
                                kmp_internal_control_t *new_icvs,
                                ident_t *loc);
89#if KMP_AFFINITY_SUPPORTED1
90static void __kmp_partition_places(kmp_team_t *team,
                                 int update_master_only = 0);
92#endif
93static void __kmp_do_serial_initialize(void);
94void __kmp_fork_barrier(int gtid, int tid);
95void __kmp_join_barrier(int gtid);
96void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
                        kmp_internal_control_t *new_icvs, ident_t *loc);

99#ifdef USE_LOAD_BALANCE1
100static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
101#endif

103static int __kmp_expand_threads(int nNeed);
104#if KMP_OS_WINDOWS0
105static int __kmp_unregister_root_other_thread(int gtid);
106#endif
107static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
108kmp_info_t *__kmp_thread_pool_insert_pt = NULL__null;

110void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads,
                             int new_nthreads);
112void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads);

114/* Calculate the identifier of the current thread */
115/* fast (and somewhat portable) way to get unique identifier of executing
 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
117int __kmp_get_global_thread_id() {
int i;
kmp_info_t **other_threads;
size_t stack_data;
char *stack_addr;
size_t stack_size;
char *stack_base;

KA_TRACE(if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d  all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
    1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d  all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
    ("*** __kmp_get_global_thread_id: entering, nproc=%d  all_nproc=%d\n",if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d  all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); }
     __kmp_nth, __kmp_all_nth))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: entering, nproc=%d  all_nproc=%d\n"
, __kmp_nth, __kmp_all_nth); };

/* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
   a parallel region, made it return KMP_GTID_DNE to force serial_initialize
   by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
   __kmp_init_gtid for this to work. */

if (!TCR_4(__kmp_init_gtid)(__kmp_init_gtid))
  return KMP_GTID_DNE(-2);

138#ifdef KMP_TDATA_GTID1
if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
  KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using TDATA\n"
); };
  return __kmp_gtid;
}
143#endif
if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
  KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using keyed TLS\n"
); };
  return __kmp_gtid_get_specific();
}
KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: using internal alg.\n"
); };

stack_addr = (char *)&stack_data;
other_threads = __kmp_threads;

/* ATT: The code below is a source of potential bugs due to unsynchronized
   access to __kmp_threads array. For example:
   1. Current thread loads other_threads[i] to thr and checks it, it is
      non-NULL.
   2. Current thread is suspended by OS.
   3. Another thread unregisters and finishes (debug versions of free()
      may fill memory with something like 0xEF).
   4. Current thread is resumed.
   5. Current thread reads junk from *thr.
   TODO: Fix it.  --ln  */

for (i = 0; i < __kmp_threads_capacity; i++) {

  kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i])((void *)(other_threads[i]));
  if (!thr)
    continue;

  stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize)((void *)(thr->th.th_info.ds.ds_stacksize));
  stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase)((void *)(thr->th.th_info.ds.ds_stackbase));

  /* stack grows down -- search through all of the active threads */

  if (stack_addr <= stack_base) {
    size_t stack_diff = stack_base - stack_addr;

    if (stack_diff <= stack_size) {
      /* The only way we can be closer than the allocated */
      /* stack size is if we are running on this thread. */
      KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i)if (!(__kmp_gtid_get_specific() == i)) { __kmp_debug_assert("__kmp_gtid_get_specific() == i"
, "openmp/runtime/src/kmp_runtime.cpp", 181); };
      return i;
    }
  }
}

/* get specific to try and determine our gtid */
KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
 "thread, using TLS\n"); }
         ("*** __kmp_get_global_thread_id: internal alg. failed to find "if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
 "thread, using TLS\n"); }
          "thread, using TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id: internal alg. failed to find "
 "thread, using TLS\n"); };
i = __kmp_gtid_get_specific();

/*fprintf( stderr, "=== %d\n", i );  */ /* GROO */

/* if we havn't been assigned a gtid, then return code */
if (i < 0)
  return i;

/* dynamically updated stack window for uber threads to avoid get_specific
   call */
if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
  KMP_FATAL(StackOverflow, i)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_StackOverflow, i), __kmp_msg_null
);
}

stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
if (stack_addr > stack_base) {
  TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr)((other_threads[i]->th.th_info.ds.ds_stackbase)) = ((stack_addr
));
  TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
          other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
)
              stack_base)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((other_threads
[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base)
);
} else {
  TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
 - stack_addr))
          stack_base - stack_addr)((other_threads[i]->th.th_info.ds.ds_stacksize)) = ((stack_base
 - stack_addr));
}

/* Reprint stack bounds for ubermaster since they have been refined */
if (__kmp_storage_map) {
  char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
  char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
  __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
                               other_threads[i]->th.th_info.ds.ds_stacksize,
                               "th_%d stack (refinement)", i);
}
return i;
225}

227int __kmp_get_global_thread_id_reg() {
int gtid;

if (!__kmp_init_serial) {
  gtid = KMP_GTID_DNE(-2);
} else
233#ifdef KMP_TDATA_GTID1
    if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 3) {
  KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using TDATA\n"
); };
  gtid = __kmp_gtid;
} else
238#endif
    if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) >= 2) {
  KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"
); };
  gtid = __kmp_gtid_get_specific();
} else {
  KA_TRACE(1000,if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); }
           ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"))if (kmp_a_debug >= 1000) { __kmp_debug_printf ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"
); };
  gtid = __kmp_get_global_thread_id();
}

/* we must be a new uber master sibling thread */
if (gtid == KMP_GTID_DNE(-2)) {
  KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
 "Registering a new gtid.\n"); }
           ("__kmp_get_global_thread_id_reg: Encountered new root thread. "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
 "Registering a new gtid.\n"); }
            "Registering a new gtid.\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
 "Registering a new gtid.\n"); };
  __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
  if (!__kmp_init_serial) {
    __kmp_do_serial_initialize();
    gtid = __kmp_gtid_get_specific();
  } else {
    gtid = __kmp_register_root(FALSE0);
  }
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
}

KMP_DEBUG_ASSERT(gtid >= 0)if (!(gtid >= 0)) { __kmp_debug_assert("gtid >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 264); };

return gtid;
267}

269/* caller must hold forkjoin_lock */
270void __kmp_check_stack_overlap(kmp_info_t *th) {
int f;
char *stack_beg = NULL__null;
char *stack_end = NULL__null;
int gtid;

KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: called\n"
); };
if (__kmp_storage_map) {
  stack_end = (char *)th->th.th_info.ds.ds_stackbase;
  stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;

  gtid = __kmp_gtid_from_thread(th);

  if (gtid == KMP_GTID_MONITOR(-4)) {
    __kmp_print_storage_map_gtid(
        gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
        "th_%s stack (%s)", "mon",
        (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
  } else {
    __kmp_print_storage_map_gtid(
        gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
        "th_%d stack (%s)", gtid,
        (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
  }
}

/* No point in checking ubermaster threads since they use refinement and
 * cannot overlap */
gtid = __kmp_gtid_from_thread(th);
if (__kmp_env_checks == TRUE(!0) && !KMP_UBER_GTID(gtid)) {
  KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); }
           ("__kmp_check_stack_overlap: performing extensive checking\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: performing extensive checking\n"
); };
  if (stack_beg == NULL__null) {
    stack_end = (char *)th->th.th_info.ds.ds_stackbase;
    stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
  }

  for (f = 0; f < __kmp_threads_capacity; f++) {
    kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f])((void *)(__kmp_threads[f]));

    if (f_th && f_th != th) {
      char *other_stack_end =
          (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase)((void *)(f_th->th.th_info.ds.ds_stackbase));
      char *other_stack_beg =
          other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize));
      if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
          (stack_end > other_stack_beg && stack_end < other_stack_end)) {

        /* Print the other stack values before the abort */
        if (__kmp_storage_map)
          __kmp_print_storage_map_gtid(
              -1, other_stack_beg, other_stack_end,
              (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize)((void *)(f_th->th.th_info.ds.ds_stacksize)),
              "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));

        __kmp_fatal(KMP_MSG(StackOverlap)__kmp_msg_format(kmp_i18n_msg_StackOverlap), KMP_HNT(ChangeStackLimit)__kmp_msg_format(kmp_i18n_hnt_ChangeStackLimit),
                    __kmp_msg_null);
      }
    }
  }
}
KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_check_stack_overlap: returning\n"
); };
332}

334/* ------------------------------------------------------------------------ */

336void __kmp_infinite_loop(void) {
static int done = FALSE0;

while (!done) {
  KMP_YIELD(TRUE){ __kmp_x86_pause(); if (((!0)) && (((__kmp_use_yield
 == 1) || (__kmp_use_yield == 2 && (((__kmp_nth) >
 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc))))))) __kmp_yield
(); };
}
342}

344#define MAX_MESSAGE512 512

346void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
                                char const *format, ...) {
char buffer[MAX_MESSAGE512];
va_list ap;

va_start(ap, format)__builtin_va_start(ap, format);
KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
             p2, (unsigned long)size, format);
__kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
__kmp_vprintf(kmp_err, buffer, ap);
356#if KMP_PRINT_DATA_PLACEMENT
int node;
if (gtid >= 0) {
  if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
    if (__kmp_storage_map_verbose) {
      node = __kmp_get_host_node(p1);
      if (node < 0) /* doesn't work, so don't try this next time */
        __kmp_storage_map_verbose = FALSE0;
      else {
        char *last;
        int lastNode;
        int localProc = __kmp_get_cpu_from_gtid(gtid);

        const int page_size = KMP_GET_PAGE_SIZE()getpagesize();

        p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
        p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
        if (localProc >= 0)
          __kmp_printf_no_lock("  GTID %d localNode %d\n", gtid,
                               localProc >> 1);
        else
          __kmp_printf_no_lock("  GTID %d\n", gtid);
378#if KMP_USE_PRCTL0
        /* The more elaborate format is disabled for now because of the prctl
         * hanging bug. */
        do {
          last = p1;
          lastNode = node;
          /* This loop collates adjacent pages with the same host node. */
          do {
            (char *)p1 += page_size;
          } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
          __kmp_printf_no_lock("    %p-%p memNode %d\n", last, (char *)p1 - 1,
                               lastNode);
        } while (p1 <= p2);
391#else
        __kmp_printf_no_lock("    %p-%p memNode %d\n", p1,
                             (char *)p1 + (page_size - 1),
                             __kmp_get_host_node(p1));
        if (p1 < p2) {
          __kmp_printf_no_lock("    %p-%p memNode %d\n", p2,
                               (char *)p2 + (page_size - 1),
                               __kmp_get_host_node(p2));
        }
400#endif
      }
    }
  } else
    __kmp_printf_no_lock("  %s\n", KMP_I18N_STR(StorageMapWarning)__kmp_i18n_catgets(kmp_i18n_str_StorageMapWarning));
}
406#endif /* KMP_PRINT_DATA_PLACEMENT */
__kmp_release_bootstrap_lock(&__kmp_stdio_lock);
408}

410void __kmp_warn(char const *format, ...) {
char buffer[MAX_MESSAGE512];
va_list ap;

if (__kmp_generate_warnings == kmp_warnings_off) {
  return;
}

va_start(ap, format)__builtin_va_start(ap, format);

KMP_SNPRINTFsnprintf(buffer, sizeof(buffer), "OMP warning: %s\n", format);
__kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
__kmp_vprintf(kmp_err, buffer, ap);
__kmp_release_bootstrap_lock(&__kmp_stdio_lock);

va_end(ap)__builtin_va_end(ap);
426}

428void __kmp_abort_process() {
// Later threads may stall here, but that's ok because abort() will kill them.
__kmp_acquire_bootstrap_lock(&__kmp_exit_lock);

if (__kmp_debug_buf) {
  __kmp_dump_debug_buffer();
}

if (KMP_OS_WINDOWS0) {
  // Let other threads know of abnormal termination and prevent deadlock
  // if abort happened during library initialization or shutdown
  __kmp_global.g.g_abort = SIGABRT6;

  /* On Windows* OS by default abort() causes pop-up error box, which stalls
     nightly testing. Unfortunately, we cannot reliably suppress pop-up error
     boxes. _set_abort_behavior() works well, but this function is not
     available in VS7 (this is not problem for DLL, but it is a problem for
     static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
     help, at least in some versions of MS C RTL.

     It seems following sequence is the only way to simulate abort() and
     avoid pop-up error box. */
  raise(SIGABRT6);
  _exit(3); // Just in case, if signal ignored, exit anyway.
} else {
  __kmp_unregister_library();
  abort();
}

__kmp_infinite_loop();
__kmp_release_bootstrap_lock(&__kmp_exit_lock);

460} // __kmp_abort_process

462void __kmp_abort_thread(void) {
// TODO: Eliminate g_abort global variable and this function.
// In case of abort just call abort(), it will kill all the threads.
__kmp_infinite_loop();
466} // __kmp_abort_thread

468/* Print out the storage map for the major kmp_info_t thread data structures
 that are allocated together. */

471static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
__kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
                             gtid);

__kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
                             sizeof(kmp_desc_t), "th_%d.th_info", gtid);

__kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
                             sizeof(kmp_local_t), "th_%d.th_local", gtid);

__kmp_print_storage_map_gtid(
    gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
    sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);

__kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
                             &thr->th.th_bar[bs_plain_barrier + 1],
                             sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
                             gtid);

__kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
                             &thr->th.th_bar[bs_forkjoin_barrier + 1],
                             sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
                             gtid);

495#if KMP_FAST_REDUCTION_BARRIER1
__kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
                             &thr->th.th_bar[bs_reduction_barrier + 1],
                             sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
                             gtid);
500#endif // KMP_FAST_REDUCTION_BARRIER
501}

503/* Print out the storage map for the major kmp_team_t team data structures
 that are allocated together. */

506static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
                                       int team_id, int num_thr) {
int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
__kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
                             header, team_id);

__kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
                             &team->t.t_bar[bs_last_barrier],
                             sizeof(kmp_balign_team_t) * bs_last_barrier,
                             "%s_%d.t_bar", header, team_id);

__kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
                             &team->t.t_bar[bs_plain_barrier + 1],
                             sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
                             header, team_id);

__kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
                             &team->t.t_bar[bs_forkjoin_barrier + 1],
                             sizeof(kmp_balign_team_t),
                             "%s_%d.t_bar[forkjoin]", header, team_id);

527#if KMP_FAST_REDUCTION_BARRIER1
__kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
                             &team->t.t_bar[bs_reduction_barrier + 1],
                             sizeof(kmp_balign_team_t),
                             "%s_%d.t_bar[reduction]", header, team_id);
532#endif // KMP_FAST_REDUCTION_BARRIER

__kmp_print_storage_map_gtid(
    -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
    sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);

__kmp_print_storage_map_gtid(
    -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
    sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);

__kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
                             &team->t.t_disp_buffer[num_disp_buff],
                             sizeof(dispatch_shared_info_t) * num_disp_buff,
                             "%s_%d.t_disp_buffer", header, team_id);
546}

548static void __kmp_init_allocator() {
__kmp_init_memkind();
__kmp_init_target_mem();
551}
552static void __kmp_fini_allocator() { __kmp_fini_memkind(); }

554/* ------------------------------------------------------------------------ */

556#if KMP_DYNAMIC_LIB1
557#if KMP_OS_WINDOWS0

559BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
//__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );

switch (fdwReason) {

case DLL_PROCESS_ATTACH:
  KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_ATTACH\n"
); };

  return TRUE(!0);

case DLL_PROCESS_DETACH:
  KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: PROCESS_DETACH T#%d\n"
, __kmp_gtid_get_specific()); };

  // According to Windows* documentation for DllMain entry point:
  // for DLL_PROCESS_DETACH, lpReserved is used for telling the difference:
  //   lpReserved == NULL when FreeLibrary() is called,
  //   lpReserved != NULL when the process is terminated.
  // When FreeLibrary() is called, worker threads remain alive. So the
  // runtime's state is consistent and executing proper shutdown is OK.
  // When the process is terminated, worker threads have exited or been
  // forcefully terminated by the OS and only the shutdown thread remains.
  // This can leave the runtime in an inconsistent state.
  // Hence, only attempt proper cleanup when FreeLibrary() is called.
  // Otherwise, rely on OS to reclaim resources.
  if (lpReserved == NULL__null)
    __kmp_internal_end_library(__kmp_gtid_get_specific());

  return TRUE(!0);

case DLL_THREAD_ATTACH:
  KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_ATTACH\n"
); };

  /* if we want to register new siblings all the time here call
   * __kmp_get_gtid(); */
  return TRUE(!0);

case DLL_THREAD_DETACH:
  KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()))if (kmp_a_debug >= 10) { __kmp_debug_printf ("DllMain: THREAD_DETACH T#%d\n"
, __kmp_gtid_get_specific()); };

  __kmp_internal_end_thread(__kmp_gtid_get_specific());
  return TRUE(!0);
}

return TRUE(!0);
603}

605#endif /* KMP_OS_WINDOWS */
606#endif /* KMP_DYNAMIC_LIB */

608/* __kmp_parallel_deo -- Wait until it's our turn. */
609void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
int gtid = *gtid_ref;
611#ifdef BUILD_PARALLEL_ORDERED1
kmp_team_t *team = __kmp_team_from_gtid(gtid);
613#endif /* BUILD_PARALLEL_ORDERED */

if (__kmp_env_consistency_check) {
  if (__kmp_threads[gtid]->th.th_root->r.r_active)
617#if KMP_USE_DYNAMIC_LOCK1
    __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null, 0);
619#else
    __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL__null);
621#endif
}
623#ifdef BUILD_PARALLEL_ORDERED1
if (!team->t.t_serialized) {
  KMP_MB();
  KMP_WAIT__kmp_wait_4(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ__kmp_eq_4,
           NULL__null);
  KMP_MB();
}
630#endif /* BUILD_PARALLEL_ORDERED */
631}

633/* __kmp_parallel_dxo -- Signal the next task. */
634void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
int gtid = *gtid_ref;
636#ifdef BUILD_PARALLEL_ORDERED1
int tid = __kmp_tid_from_gtid(gtid);
kmp_team_t *team = __kmp_team_from_gtid(gtid);
639#endif /* BUILD_PARALLEL_ORDERED */

if (__kmp_env_consistency_check) {
  if (__kmp_threads[gtid]->th.th_root->r.r_active)
    __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
}
645#ifdef BUILD_PARALLEL_ORDERED1
if (!team->t.t_serialized) {
  KMP_MB(); /* Flush all pending memory write invalidates.  */

  /* use the tid of the next thread in this team */
  /* TODO replace with general release procedure */
  team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);

  KMP_MB(); /* Flush all pending memory write invalidates.  */
}
655#endif /* BUILD_PARALLEL_ORDERED */
656}

658/* ------------------------------------------------------------------------ */
659/* The BARRIER for a SINGLE process section is always explicit   */

661int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
int status;
kmp_info_t *th;
kmp_team_t *team;

if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
  __kmp_parallel_initialize();
__kmp_resume_if_soft_paused();

th = __kmp_threads[gtid];
team = th->th.th_team;
status = 0;

th->th.th_ident = id_ref;

if (team->t.t_serialized) {
  status = 1;
} else {
  kmp_int32 old_this = th->th.th_local.this_construct;

  ++th->th.th_local.this_construct;
  /* try to set team count to thread count--success means thread got the
     single block */
  /* TODO: Should this be acquire or release? */
  if (team->t.t_construct == old_this) {
    status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
                                            th->th.th_local.this_construct);
  }
689#if USE_ITT_BUILD1
  if (__itt_metadata_add_ptr__kmp_itt_metadata_add_ptr__3_0 && __kmp_forkjoin_frames_mode == 3 &&
      KMP_MASTER_GTID(gtid)(0 == __kmp_tid_from_gtid((gtid))) && th->th.th_teams_microtask == NULL__null &&
      team->t.t_active_level == 1) {
    // Only report metadata by primary thread of active team at level 1
    __kmp_itt_metadata_single(id_ref);
  }
696#endif /* USE_ITT_BUILD */
}

if (__kmp_env_consistency_check) {
  if (status && push_ws) {
    __kmp_push_workshare(gtid, ct_psingle, id_ref);
  } else {
    __kmp_check_workshare(gtid, ct_psingle, id_ref);
  }
}
706#if USE_ITT_BUILD1
if (status) {
  __kmp_itt_single_start(gtid);
}
710#endif /* USE_ITT_BUILD */
return status;
712}

714void __kmp_exit_single(int gtid) {
715#if USE_ITT_BUILD1
__kmp_itt_single_end(gtid);
717#endif /* USE_ITT_BUILD */
if (__kmp_env_consistency_check)
  __kmp_pop_workshare(gtid, ct_psingle, NULL__null);
720}

722/* determine if we can go parallel or must use a serialized parallel region and
* how many threads we can use
* set_nproc is the number of threads requested for the team
* returns 0 if we should serialize or only use one thread,
* otherwise the number of threads to use
* The forkjoin lock is held by the caller. */
728static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
                               int master_tid, int set_nthreads,
                               int enter_teams) {
int capacity;
int new_nthreads;
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 733); };
KMP_DEBUG_ASSERT(root && parent_team)if (!(root && parent_team)) { __kmp_debug_assert("root && parent_team"
, "openmp/runtime/src/kmp_runtime.cpp", 734); };
kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];

// If dyn-var is set, dynamically adjust the number of desired threads,
// according to the method specified by dynamic_mode.
new_nthreads = set_nthreads;
if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic)) {
  ;
}
743#ifdef USE_LOAD_BALANCE1
else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
  new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
  if (new_nthreads == 1) {
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
 "reservation to 1 thread\n", master_tid); }
                  "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
 "reservation to 1 thread\n", master_tid); }
                  master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
 "reservation to 1 thread\n", master_tid); };
    return 1;
  }
  if (new_nthreads < set_nthreads) {
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); }
                  "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); }
                  master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d load balance reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); };
  }
}
758#endif /* USE_LOAD_BALANCE */
else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
  new_nthreads = __kmp_avail_proc - __kmp_nth +
                 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
  if (new_nthreads <= 1) {
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
 "reservation to 1 thread\n", master_tid); }
                  "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
 "reservation to 1 thread\n", master_tid); }
                  master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
 "reservation to 1 thread\n", master_tid); };
    return 1;
  }
  if (new_nthreads < set_nthreads) {
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); }
                  "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); }
                  master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d thread limit reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); };
  } else {
    new_nthreads = set_nthreads;
  }
} else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
  if (set_nthreads > 2) {
    new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
    new_nthreads = (new_nthreads % set_nthreads) + 1;
    if (new_nthreads == 1) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
 "reservation to 1 thread\n", master_tid); }
                    "reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
 "reservation to 1 thread\n", master_tid); }
                    master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
 "reservation to 1 thread\n", master_tid); };
      return 1;
    }
    if (new_nthreads < set_nthreads) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); }
                    "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); }
                    master_tid, new_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d dynamic random reduced "
 "reservation to %d threads\n", master_tid, new_nthreads); };
    }
  }
} else {
  KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_runtime.cpp"
, 792); };
}

// Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
if (__kmp_nth + new_nthreads -
        (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
    __kmp_max_nth) {
  int tl_nthreads = __kmp_max_nth - __kmp_nth +
                    (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
  if (tl_nthreads <= 0) {
    tl_nthreads = 1;
  }

  // If dyn-var is false, emit a 1-time warning.
  if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic) && (!__kmp_reserve_warn)) {
    __kmp_reserve_warn = 1;
    __kmp_msg(kmp_ms_warning,
              KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
),
              KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
  }
  if (tl_nthreads == 1) {
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
 "reduced reservation to 1 thread\n", master_tid); }
                  "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
 "reduced reservation to 1 thread\n", master_tid); }
                  master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
 "reduced reservation to 1 thread\n", master_tid); };
    return 1;
  }
  KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
 "reservation to %d threads\n", master_tid, tl_nthreads); }
                "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
 "reservation to %d threads\n", master_tid, tl_nthreads); }
                master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
 "reservation to %d threads\n", master_tid, tl_nthreads); };
  new_nthreads = tl_nthreads;
}

// Respect OMP_THREAD_LIMIT
int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
if (cg_nthreads + new_nthreads -
        (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
    max_cg_threads) {
  int tl_nthreads = max_cg_threads - cg_nthreads +
                    (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
  if (tl_nthreads <= 0) {
    tl_nthreads = 1;
  }

  // If dyn-var is false, emit a 1-time warning.
  if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic) && (!__kmp_reserve_warn)) {
    __kmp_reserve_warn = 1;
    __kmp_msg(kmp_ms_warning,
              KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, tl_nthreads
),
              KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
  }
  if (tl_nthreads == 1) {
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
 "reduced reservation to 1 thread\n", master_tid); }
                  "reduced reservation to 1 thread\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
 "reduced reservation to 1 thread\n", master_tid); }
                  master_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
 "reduced reservation to 1 thread\n", master_tid); };
    return 1;
  }
  KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
 "reservation to %d threads\n", master_tid, tl_nthreads); }
                "reservation to %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
 "reservation to %d threads\n", master_tid, tl_nthreads); }
                master_tid, tl_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
 "reservation to %d threads\n", master_tid, tl_nthreads); };
  new_nthreads = tl_nthreads;
}

// Check if the threads array is large enough, or needs expanding.
// See comment in __kmp_register_root() about the adjustment if
// __kmp_threads[0] == NULL.
capacity = __kmp_threads_capacity;
if (TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
  --capacity;
}
// If it is not for initializing the hidden helper team, we need to take
// __kmp_hidden_helper_threads_num out of the capacity because it is included
// in __kmp_threads_capacity.
if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
  capacity -= __kmp_hidden_helper_threads_num;
}
if (__kmp_nth + new_nthreads -
        (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
    capacity) {
  // Expand the threads array.
  int slotsRequired = __kmp_nth + new_nthreads -
                      (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
                      capacity;
  int slotsAdded = __kmp_expand_threads(slotsRequired);
  if (slotsAdded < slotsRequired) {
    // The threads array was not expanded enough.
    new_nthreads -= (slotsRequired - slotsAdded);
    KMP_ASSERT(new_nthreads >= 1)if (!(new_nthreads >= 1)) { __kmp_debug_assert("new_nthreads >= 1"
, "openmp/runtime/src/kmp_runtime.cpp", 879); };

    // If dyn-var is false, emit a 1-time warning.
    if (!get__dynamic_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.dynamic) && (!__kmp_reserve_warn)) {
      __kmp_reserve_warn = 1;
      if (__kmp_tp_cached) {
        __kmp_msg(kmp_ms_warning,
                  KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
),
                  KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
),
                  KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
      } else {
        __kmp_msg(kmp_ms_warning,
                  KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, set_nthreads, new_nthreads
),
                  KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads), __kmp_msg_null);
      }
    }
  }
}

898#ifdef KMP_DEBUG1
if (new_nthreads == 1) {
  KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
 "dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
           ("__kmp_reserve_threads: T#%d serializing team after reclaiming "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
 "dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
            "dead roots and rechecking; requested %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
 "dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); }
            __kmp_get_gtid(), set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
 "dead roots and rechecking; requested %d threads\n", __kmp_get_global_thread_id
(), set_nthreads); };
} else {
  KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
 " %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
 set_nthreads); }
                " %d threads\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
 " %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
 set_nthreads); }
                __kmp_get_gtid(), new_nthreads, set_nthreads))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
 " %d threads\n", __kmp_get_global_thread_id(), new_nthreads,
 set_nthreads); };
}
909#endif // KMP_DEBUG
return new_nthreads;
911}

913/* Allocate threads from the thread pool and assign them to the new team. We are
 assured that there are enough threads available, because we checked on that
 earlier within critical section forkjoin */
916static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
                                  kmp_info_t *master_th, int master_gtid,
                                  int fork_teams_workers) {
int i;
int use_hot_team;

KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_fork_team_threads: new_nprocs = %d\n"
, team->t.t_nproc); };
KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid())if (!(master_gtid == __kmp_get_global_thread_id())) { __kmp_debug_assert
("master_gtid == __kmp_get_global_thread_id()", "openmp/runtime/src/kmp_runtime.cpp"
, 923); };
KMP_MB();

/* first, let's setup the primary thread */
master_th->th.th_info.ds.ds_tid = 0;
master_th->th.th_team = team;
master_th->th.th_team_nproc = team->t.t_nproc;
master_th->th.th_team_master = master_th;
master_th->th.th_team_serialized = FALSE0;
master_th->th.th_dispatch = &team->t.t_dispatch[0];

934/* make sure we are not the optimized hot team */
935#if KMP_NESTED_HOT_TEAMS1
use_hot_team = 0;
kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
if (hot_teams) { // hot teams array is not allocated if
  // KMP_HOT_TEAMS_MAX_LEVEL=0
  int level = team->t.t_active_level - 1; // index in array of hot teams
  if (master_th->th.th_teams_microtask) { // are we inside the teams?
    if (master_th->th.th_teams_size.nteams > 1) {
      ++level; // level was not increased in teams construct for
      // team_of_masters
    }
    if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
        master_th->th.th_teams_level == team->t.t_level) {
      ++level; // level was not increased in teams construct for
      // team_of_workers before the parallel
    } // team->t.t_level will be increased inside parallel
  }
  if (level < __kmp_hot_teams_max_level) {
    if (hot_teams[level].hot_team) {
      // hot team has already been allocated for given level
      KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team)if (!(hot_teams[level].hot_team == team)) { __kmp_debug_assert
("hot_teams[level].hot_team == team", "openmp/runtime/src/kmp_runtime.cpp"
, 955); };
      use_hot_team = 1; // the team is ready to use
    } else {
      use_hot_team = 0; // AC: threads are not allocated yet
      hot_teams[level].hot_team = team; // remember new hot team
      hot_teams[level].hot_team_nth = team->t.t_nproc;
    }
  } else {
    use_hot_team = 0;
  }
}
966#else
use_hot_team = team == root->r.r_hot_team;
968#endif
if (!use_hot_team) {

  /* install the primary thread */
  team->t.t_threads[0] = master_th;
  __kmp_initialize_info(master_th, team, 0, master_gtid);

  /* now, install the worker threads */
  for (i = 1; i < team->t.t_nproc; i++) {

    /* fork or reallocate a new thread and install it in team */
    kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
    team->t.t_threads[i] = thr;
    KMP_DEBUG_ASSERT(thr)if (!(thr)) { __kmp_debug_assert("thr", "openmp/runtime/src/kmp_runtime.cpp"
, 981); };
    KMP_DEBUG_ASSERT(thr->th.th_team == team)if (!(thr->th.th_team == team)) { __kmp_debug_assert("thr->th.th_team == team"
, "openmp/runtime/src/kmp_runtime.cpp", 982); };
    /* align team and thread arrived states */
    KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
 "T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  "T#%d(%d:%d) join =%llu, plain=%llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
 "T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  __kmp_gtid_from_tid(0, team), team->t.t_id, 0,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
 "T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  __kmp_gtid_from_tid(i, team), team->t.t_id, i,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
 "T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  team->t.t_bar[bs_forkjoin_barrier].b_arrived,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
 "T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  team->t.t_bar[bs_plain_barrier].b_arrived))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
 "T#%d(%d:%d) join =%llu, plain=%llu\n", __kmp_gtid_from_tid(
0, team), team->t.t_id, 0, __kmp_gtid_from_tid(i, team), team
->t.t_id, i, team->t.t_bar[bs_forkjoin_barrier].b_arrived
, team->t.t_bar[bs_plain_barrier].b_arrived); };
    thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
    thr->th.th_teams_level = master_th->th.th_teams_level;
    thr->th.th_teams_size = master_th->th.th_teams_size;
    { // Initialize threads' barrier data.
      int b;
      kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
      for (b = 0; b < bs_last_barrier; ++b) {
        balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
        KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 998); };
999#if USE_DEBUGGER0
        balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1001#endif
      }
    }
  }

1006#if KMP_AFFINITY_SUPPORTED1
  // Do not partition the places list for teams construct workers who
  // haven't actually been forked to do real work yet. This partitioning
  // will take place in the parallel region nested within the teams construct.
  if (!fork_teams_workers) {
    __kmp_partition_places(team);
  }
1013#endif
}

if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
  for (i = 0; i < team->t.t_nproc; i++) {
    kmp_info_t *thr = team->t.t_threads[i];
    if (thr->th.th_prev_num_threads != team->t.t_nproc ||
        thr->th.th_prev_level != team->t.t_level) {
      team->t.t_display_affinity = 1;
      break;
    }
  }
}

KMP_MB();
1028}

1030#if KMP_ARCH_X860 || KMP_ARCH_X86_641
1031// Propagate any changes to the floating point control registers out to the team
1032// We try to avoid unnecessary writes to the relevant cache line in the team
1033// structure, so we don't make changes unless they are needed.
1034inline static void propagateFPControl(kmp_team_t *team) {
if (__kmp_inherit_fp_control) {
  kmp_int16 x87_fpu_control_word;
  kmp_uint32 mxcsr;

  // Get primary thread's values of FPU control flags (both X87 and vector)
  __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
  __kmp_store_mxcsr(&mxcsr);
  mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;

  // There is no point looking at t_fp_control_saved here.
  // If it is TRUE, we still have to update the values if they are different
  // from those we now have. If it is FALSE we didn't save anything yet, but
  // our objective is the same. We have to ensure that the values in the team
  // are the same as those we have.
  // So, this code achieves what we need whether or not t_fp_control_saved is
  // true. By checking whether the value needs updating we avoid unnecessary
  // writes that would put the cache-line into a written state, causing all
  // threads in the team to have to read it again.
  KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word)if ((team->t.t_x87_fpu_control_word) != (x87_fpu_control_word
)) (team->t.t_x87_fpu_control_word) = (x87_fpu_control_word
);
  KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr)if ((team->t.t_mxcsr) != (mxcsr)) (team->t.t_mxcsr) = (
mxcsr);
  // Although we don't use this value, other code in the runtime wants to know
  // whether it should restore them. So we must ensure it is correct.
  KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE)if ((team->t.t_fp_control_saved) != ((!0))) (team->t.t_fp_control_saved
) = ((!0));
} else {
  // Similarly here. Don't write to this cache-line in the team structure
  // unless we have to.
  KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE)if ((team->t.t_fp_control_saved) != (0)) (team->t.t_fp_control_saved
) = (0);
}
1063}

1065// Do the opposite, setting the hardware registers to the updated values from
1066// the team.
1067inline static void updateHWFPControl(kmp_team_t *team) {
if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
  // Only reset the fp control regs if they have been changed in the team.
  // the parallel region that we are exiting.
  kmp_int16 x87_fpu_control_word;
  kmp_uint32 mxcsr;
  __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
  __kmp_store_mxcsr(&mxcsr);
  mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;

  if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
    __kmp_clear_x87_fpu_status_word();
    __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
  }

  if (team->t.t_mxcsr != mxcsr) {
    __kmp_load_mxcsr(&team->t.t_mxcsr);
  }
}
1086}
1087#else
1088#define propagateFPControl(x) ((void)0)
1089#define updateHWFPControl(x) ((void)0)
1090#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */

1092static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
                                   int realloc); // forward declaration

1095/* Run a parallel region that has been serialized, so runs only in a team of the
 single primary thread. */
1097void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
kmp_info_t *this_thr;
kmp_team_t *serial_team;

KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: called by T#%d\n"
, global_tid); };

/* Skip all this code for autopar serialized loops since it results in
   unacceptable overhead */
if (loc != NULL__null && (loc->flags & KMP_IDENT_AUTOPAR))
  return;

if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
  __kmp_parallel_initialize();
__kmp_resume_if_soft_paused();

this_thr = __kmp_threads[global_tid];
serial_team = this_thr->th.th_serial_team;

/* utilize the serialized team held by this thread */
KMP_DEBUG_ASSERT(serial_team)if (!(serial_team)) { __kmp_debug_assert("serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1116); };
KMP_MB();

if (__kmp_tasking_mode != tskm_immediate_exec) {
  KMP_DEBUG_ASSERT(if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1122); }
      this_thr->th.th_task_team ==if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1122); }
      this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state])if (!(this_thr->th.th_task_team == this_thr->th.th_team
->t.t_task_team[this_thr->th.th_task_state])) { __kmp_debug_assert
("this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1122); };
  KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==if (!(serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null)) { __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 1124); }
                   NULL)if (!(serial_team->t.t_task_team[this_thr->th.th_task_state
] == __null)) { __kmp_debug_assert("serial_team->t.t_task_team[this_thr->th.th_task_state] == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 1124); };
  KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
 "team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
                "team %p, new task_team = NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
 "team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); }
                global_tid, this_thr->th.th_task_team, this_thr->th.th_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
 "team %p, new task_team = NULL\n", global_tid, this_thr->
th.th_task_team, this_thr->th.th_team); };
  this_thr->th.th_task_team = NULL__null;
}

kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
  proc_bind = proc_bind_false;
} else if (proc_bind == proc_bind_default) {
  // No proc_bind clause was specified, so use the current value
  // of proc-bind-var for this parallel region.
  proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
}
// Reset for next parallel region
this_thr->th.th_set_proc_bind = proc_bind_default;

1142#if OMPT_SUPPORT1
ompt_data_t ompt_parallel_data = ompt_data_none{0};
void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid)__ompt_load_return_address(global_tid);
if (ompt_enabled.enabled &&
    this_thr->th.ompt_thread_info.state != ompt_state_overhead) {

  ompt_task_info_t *parent_task_info;
  parent_task_info = OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info));

  parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
  if (ompt_enabled.ompt_callback_parallel_begin) {
    int team_size = 1;

    ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
        &(parent_task_info->task_data), &(parent_task_info->frame),
        &ompt_parallel_data, team_size,
        ompt_parallel_invoker_program | ompt_parallel_team, codeptr);
  }
}
1161#endif // OMPT_SUPPORT

if (this_thr->th.th_team != serial_team) {
  // Nested level will be an index in the nested nthreads array
  int level = this_thr->th.th_team->t.t_level;

  if (serial_team->t.t_serialized) {
    /* this serial team was already used
       TODO increase performance by making this locks more specific */
    kmp_team_t *new_team;

    __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);

    new_team =
        __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1176#if OMPT_SUPPORT1
                            ompt_parallel_data,
1178#endif
                            proc_bind, &this_thr->th.th_current_task->td_icvs,
                            0 USE_NESTED_HOT_ARG(NULL), __null);
    __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
    KMP_ASSERT(new_team)if (!(new_team)) { __kmp_debug_assert("new_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1182); };

    /* setup new serialized team and install it */
    new_team->t.t_threads[0] = this_thr;
    new_team->t.t_parent = this_thr->th.th_team;
    serial_team = new_team;
    this_thr->th.th_serial_team = serial_team;

    KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
        10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
        ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); }
         global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n"
, global_tid, serial_team); };

    /* TODO the above breaks the requirement that if we run out of resources,
       then we can still guarantee that serialized teams are ok, since we may
       need to allocate a new one */
  } else {
    KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
        10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
        ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); }
         global_tid, serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n"
, global_tid, serial_team); };
  }

  /* we have to initialize this serial team */
  KMP_DEBUG_ASSERT(serial_team->t.t_threads)if (!(serial_team->t.t_threads)) { __kmp_debug_assert("serial_team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 1206); };
  KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)if (!(serial_team->t.t_threads[0] == this_thr)) { __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "openmp/runtime/src/kmp_runtime.cpp"
, 1207); };
  KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team)if (!(this_thr->th.th_team != serial_team)) { __kmp_debug_assert
("this_thr->th.th_team != serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1208); };
  serial_team->t.t_ident = loc;
  serial_team->t.t_serialized = 1;
  serial_team->t.t_nproc = 1;
  serial_team->t.t_parent = this_thr->th.th_team;
  serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
  this_thr->th.th_team = serial_team;
  serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;

  KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d curtask=%p\n", global_tid,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); }
                this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d curtask=%p\n"
, global_tid, this_thr->th.th_current_task); };
  KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1)if (!(this_thr->th.th_current_task->td_flags.executing ==
 1)) { __kmp_debug_assert("this_thr->th.th_current_task->td_flags.executing == 1"
, "openmp/runtime/src/kmp_runtime.cpp", 1219); };
  this_thr->th.th_current_task->td_flags.executing = 0;

  __kmp_push_current_task_to_thread(this_thr, serial_team, 0);

  /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
     implicit task for each serialized task represented by
     team->t.t_serialized? */
  copy_icvs(&this_thr->th.th_current_task->td_icvs,
            &this_thr->th.th_current_task->td_parent->td_icvs);

  // Thread value exists in the nested nthreads array for the next nested
  // level
  if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
    this_thr->th.th_current_task->td_icvs.nproc =
        __kmp_nested_nth.nth[level + 1];
  }

  if (__kmp_nested_proc_bind.used &&
      (level + 1 < __kmp_nested_proc_bind.used)) {
    this_thr->th.th_current_task->td_icvs.proc_bind =
        __kmp_nested_proc_bind.bind_types[level + 1];
  }

1243#if USE_DEBUGGER0
  serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
1245#endif
  this_thr->th.th_info.ds.ds_tid = 0;

  /* set thread cache values */
  this_thr->th.th_team_nproc = 1;
  this_thr->th.th_team_master = this_thr;
  this_thr->th.th_team_serialized = 1;

  serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
  serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
  serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save

  propagateFPControl(serial_team);

  /* check if we need to allocate dispatch buffers stack */
  KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)if (!(serial_team->t.t_dispatch)) { __kmp_debug_assert("serial_team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 1260); };
  if (!serial_team->t.t_dispatch->th_disp_buffer) {
    serial_team->t.t_dispatch->th_disp_buffer =
        (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1264)
            sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1264);
  }
  this_thr->th.th_dispatch = serial_team->t.t_dispatch;

  KMP_MB();

} else {
  /* this serialized team is already being used,
   * that's fine, just add another nested level */
  KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team)if (!(this_thr->th.th_team == serial_team)) { __kmp_debug_assert
("this_thr->th.th_team == serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 1273); };
  KMP_DEBUG_ASSERT(serial_team->t.t_threads)if (!(serial_team->t.t_threads)) { __kmp_debug_assert("serial_team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 1274); };
  KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr)if (!(serial_team->t.t_threads[0] == this_thr)) { __kmp_debug_assert
("serial_team->t.t_threads[0] == this_thr", "openmp/runtime/src/kmp_runtime.cpp"
, 1275); };
  ++serial_team->t.t_serialized;
  this_thr->th.th_team_serialized = serial_team->t.t_serialized;

  // Nested level will be an index in the nested nthreads array
  int level = this_thr->th.th_team->t.t_level;
  // Thread value exists in the nested nthreads array for the next nested
  // level
  if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
    this_thr->th.th_current_task->td_icvs.nproc =
        __kmp_nested_nth.nth[level + 1];
  }
  serial_team->t.t_level++;
  KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
 "of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
                "of serial team %p to %d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
 "of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); }
                global_tid, serial_team, serial_team->t.t_level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmpc_serialized_parallel: T#%d increasing nesting level "
 "of serial team %p to %d\n", global_tid, serial_team, serial_team
->t.t_level); };

  /* allocate/push dispatch buffers stack */
  KMP_DEBUG_ASSERT(serial_team->t.t_dispatch)if (!(serial_team->t.t_dispatch)) { __kmp_debug_assert("serial_team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 1293); };
  {
    dispatch_private_info_t *disp_buffer =
        (dispatch_private_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1297)
            sizeof(dispatch_private_info_t))___kmp_allocate((sizeof(dispatch_private_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 1297);
    disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
    serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
  }
  this_thr->th.th_dispatch = serial_team->t.t_dispatch;

  KMP_MB();
}
KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq)if ((serial_team->t.t_cancel_request) != (cancel_noreq)) (
serial_team->t.t_cancel_request) = (cancel_noreq);

// Perform the display affinity functionality for
// serialized parallel regions
if (__kmp_display_affinity) {
  if (this_thr->th.th_prev_level != serial_team->t.t_level ||
      this_thr->th.th_prev_num_threads != 1) {
    // NULL means use the affinity-format-var ICV
    __kmp_aux_display_affinity(global_tid, NULL__null);
    this_thr->th.th_prev_level = serial_team->t.t_level;
    this_thr->th.th_prev_num_threads = 1;
  }
}

if (__kmp_env_consistency_check)
  __kmp_push_parallel(global_tid, NULL__null);
1321#if OMPT_SUPPORT1
serial_team->t.ompt_team_info.master_return_address = codeptr;
if (ompt_enabled.enabled &&
    this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
  OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame.ptr =
      OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);

  ompt_lw_taskteam_t lw_taskteam;
  __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
                          &ompt_parallel_data, codeptr);

  __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
  // don't use lw_taskteam after linking. content was swaped

  /* OMPT implicit task begin */
  if (ompt_enabled.ompt_callback_implicit_task) {
    ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
        ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr)(&(this_thr->th.th_team->t.ompt_team_info.parallel_data
)),
        OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
)), 1, __kmp_tid_from_gtid(global_tid),
        ompt_task_implicit); // TODO: Can this be ompt_task_initial?
    OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->thread_num =
        __kmp_tid_from_gtid(global_tid);
  }

  /* OMPT state */
  this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
  OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->frame.exit_frame.ptr =
      OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
}
1350#endif
1351}

1353// Test if this fork is for a team closely nested in a teams construct
1354static inline bool __kmp_is_fork_in_teams(kmp_info_t *master_th,
                                        microtask_t microtask, int level,
                                        int teams_level, kmp_va_list ap) {
return (master_th->th.th_teams_microtask && ap &&
        microtask != (microtask_t)__kmp_teams_master && level == teams_level);
1359}

1361// Test if this fork is for the teams construct, i.e. to form the outer league
1362// of teams
1363static inline bool __kmp_is_entering_teams(int active_level, int level,
                                         int teams_level, kmp_va_list ap) {
return ((ap == NULL__null && active_level == 0) ||
        (ap && teams_level > 0 && teams_level == level));
1367}

1369// AC: This is start of parallel that is nested inside teams construct.
1370// The team is actual (hot), all workers are ready at the fork barrier.
1371// No lock needed to initialize the team a bit, then free workers.
1372static inline int
1373__kmp_fork_in_teams(ident_t *loc, int gtid, kmp_team_t *parent_team,
                  kmp_int32 argc, kmp_info_t *master_th, kmp_root_t *root,
                  enum fork_context_e call_context, microtask_t microtask,
                  launch_t invoker, int master_set_numthreads, int level,
1377#if OMPT_SUPPORT1
                  ompt_data_t ompt_parallel_data, void *return_address,
1379#endif
                  kmp_va_list ap) {
void **argv;
int i;

parent_team->t.t_ident = loc;
__kmp_alloc_argv_entries(argc, parent_team, TRUE(!0));
parent_team->t.t_argc = argc;
argv = (void **)parent_team->t.t_argv;
for (i = argc - 1; i >= 0; --i) {
  *argv++ = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
}
// Increment our nested depth levels, but not increase the serialization
if (parent_team == master_th->th.th_serial_team) {
  // AC: we are in serialized parallel
  __kmpc_serialized_parallel(loc, gtid);
  KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1)if (!(parent_team->t.t_serialized > 1)) { __kmp_debug_assert
("parent_team->t.t_serialized > 1", "openmp/runtime/src/kmp_runtime.cpp"
, 1395); };

  if (call_context == fork_context_gnu) {
    // AC: need to decrement t_serialized for enquiry functions to work
    // correctly, will restore at join time
    parent_team->t.t_serialized--;
    return TRUE(!0);
  }

1404#if OMPD_SUPPORT1
  parent_team->t.t_pkfn = microtask;
1406#endif

1408#if OMPT_SUPPORT1
  void *dummy;
  void **exit_frame_p;
  ompt_data_t *implicit_task_data;
  ompt_lw_taskteam_t lw_taskteam;

  if (ompt_enabled.enabled) {
    __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
                            &ompt_parallel_data, return_address);
    exit_frame_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);

    __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
    // Don't use lw_taskteam after linking. Content was swapped.

    /* OMPT implicit task begin */
    implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
));
    if (ompt_enabled.ompt_callback_implicit_task) {
      OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num = __kmp_tid_from_gtid(gtid);
      ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
          ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
)), implicit_task_data,
          1, OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
    }

    /* OMPT state */
    master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
  } else {
    exit_frame_p = &dummy;
  }
1436#endif

  // AC: need to decrement t_serialized for enquiry functions to work
  // correctly, will restore at join time
  parent_team->t.t_serialized--;

  {
    KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
    KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
    __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1446#if OMPT_SUPPORT1
                           ,
                           exit_frame_p
1449#endif
                           );
  }

1453#if OMPT_SUPPORT1
  if (ompt_enabled.enabled) {
    *exit_frame_p = NULL__null;
    OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->frame.exit_frame = ompt_data_none{0};
    if (ompt_enabled.ompt_callback_implicit_task) {
      ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
          ompt_scope_end, NULL__null, implicit_task_data, 1,
          OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
    }
    ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
));
    __ompt_lw_taskteam_unlink(master_th);
    if (ompt_enabled.ompt_callback_parallel_end) {
      ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
          &ompt_parallel_data, OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
)),
          OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
 : ompt_parallel_invoker_runtime) | ompt_parallel_team, return_address);
    }
    master_th->th.ompt_thread_info.state = ompt_state_overhead;
  }
1471#endif
  return TRUE(!0);
}

parent_team->t.t_pkfn = microtask;
parent_team->t.t_invoke = invoker;
KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
);
parent_team->t.t_active_level++;
parent_team->t.t_level++;
parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save

// If the threads allocated to the team are less than the thread limit, update
// the thread limit here. th_teams_size.nth is specific to this team nested
// in a teams construct, the team is fully created, and we're about to do
// the actual fork. Best to do this here so that the subsequent uses below
// and in the join have the correct value.
master_th->th.th_teams_size.nth = parent_team->t.t_nproc;

1489#if OMPT_SUPPORT1
if (ompt_enabled.enabled) {
  ompt_lw_taskteam_t lw_taskteam;
  __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, &ompt_parallel_data,
                          return_address);
  __ompt_lw_taskteam_link(&lw_taskteam, master_th, 1, true);
}
1496#endif

/* Change number of threads in the team if requested */
if (master_set_numthreads) { // The parallel has num_threads clause
  if (master_set_numthreads <= master_th->th.th_teams_size.nth) {
    // AC: only can reduce number of threads dynamically, can't increase
    kmp_info_t **other_threads = parent_team->t.t_threads;
    // NOTE: if using distributed barrier, we need to run this code block
    // even when the team size appears not to have changed from the max.
    int old_proc = master_th->th.th_teams_size.nth;
    if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
      __kmp_resize_dist_barrier(parent_team, old_proc, master_set_numthreads);
      __kmp_add_threads_to_team(parent_team, master_set_numthreads);
    }
    parent_team->t.t_nproc = master_set_numthreads;
    for (i = 0; i < master_set_numthreads; ++i) {
      other_threads[i]->th.th_team_nproc = master_set_numthreads;
    }
  }
  // Keep extra threads hot in the team for possible next parallels
  master_th->th.th_set_nproc = 0;
}

1519#if USE_DEBUGGER0
if (__kmp_debugging) { // Let debugger override number of threads.
  int nth = __kmp_omp_num_threads(loc);
  if (nth > 0) { // 0 means debugger doesn't want to change num threads
    master_set_numthreads = nth;
  }
}
1526#endif

// Figure out the proc_bind policy for the nested parallel within teams
kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
// proc_bind_default means don't update
kmp_proc_bind_t proc_bind_icv = proc_bind_default;
if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
  proc_bind = proc_bind_false;
} else {
  // No proc_bind clause specified; use current proc-bind-var
  if (proc_bind == proc_bind_default) {
    proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
  }
  /* else: The proc_bind policy was specified explicitly on parallel clause.
     This overrides proc-bind-var for this parallel region, but does not
     change proc-bind-var. */
  // Figure the value of proc-bind-var for the child threads.
  if ((level + 1 < __kmp_nested_proc_bind.used) &&
      (__kmp_nested_proc_bind.bind_types[level + 1] !=
       master_th->th.th_current_task->td_icvs.proc_bind)) {
    proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
  }
}
KMP_CHECK_UPDATE(parent_team->t.t_proc_bind, proc_bind)if ((parent_team->t.t_proc_bind) != (proc_bind)) (parent_team
->t.t_proc_bind) = (proc_bind);
// Need to change the bind-var ICV to correct value for each implicit task
if (proc_bind_icv != proc_bind_default &&
    master_th->th.th_current_task->td_icvs.proc_bind != proc_bind_icv) {
  kmp_info_t **other_threads = parent_team->t.t_threads;
  for (i = 0; i < master_th->th.th_team_nproc; ++i) {
    other_threads[i]->th.th_current_task->td_icvs.proc_bind = proc_bind_icv;
  }
}
// Reset for next parallel region
master_th->th.th_set_proc_bind = proc_bind_default;

1561#if USE_ITT_BUILD1 && USE_ITT_NOTIFY1
if (((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 && __itt_get_timestamp_ptr__kmp_itt_get_timestamp_ptr__3_0) ||
     KMP_ITT_DEBUG0) &&
    __kmp_forkjoin_frames_mode == 3 &&
    parent_team->t.t_active_level == 1 // only report frames at level 1
    && master_th->th.th_teams_size.nteams == 1) {
  kmp_uint64 tmp_time = __itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0();
  master_th->th.th_frame_time = tmp_time;
  parent_team->t.t_region_time = tmp_time;
}
if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
  KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL)if (!(parent_team->t.t_stack_id == __null)) { __kmp_debug_assert
("parent_team->t.t_stack_id == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 1572); };
  // create new stack stitching id before entering fork barrier
  parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
}
1576#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
1577#if KMP_AFFINITY_SUPPORTED1
__kmp_partition_places(parent_team);
1579#endif

KF_TRACE(10, ("__kmp_fork_in_teams: before internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_in_teams: before internal fork: root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
              "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_in_teams: before internal fork: root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
              root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_in_teams: before internal fork: root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); };
__kmp_internal_fork(loc, gtid, parent_team);
KF_TRACE(10, ("__kmp_fork_in_teams: after internal fork: root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_in_teams: after internal fork: root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
              "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_in_teams: after internal fork: root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); }
              root, parent_team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_in_teams: after internal fork: root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid
); };

if (call_context == fork_context_gnu)
  return TRUE(!0);

/* Invoke microtask for PRIMARY thread */
KA_TRACE(20, ("__kmp_fork_in_teams: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_in_teams: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
              parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_in_teams: T#%d(%d:0) invoke microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); };

if (!parent_team->t.t_invoke(gtid)) {
  KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread")if (!(0)) { __kmp_debug_assert(("cannot invoke microtask for PRIMARY thread"
), "openmp/runtime/src/kmp_runtime.cpp", 1597); };
}
KA_TRACE(20, ("__kmp_fork_in_teams: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_in_teams: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); }
              parent_team->t.t_id, parent_team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_in_teams: T#%d(%d:0) done microtask = %p\n"
, gtid, parent_team->t.t_id, parent_team->t.t_pkfn); };
KMP_MB(); /* Flush all pending memory write invalidates.  */

KA_TRACE(20, ("__kmp_fork_in_teams: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_in_teams: parallel exit T#%d\n"
, gtid); };

return TRUE(!0);
1606}

1608// Create a serialized parallel region
1609static inline int
1610__kmp_serial_fork_call(ident_t *loc, int gtid, enum fork_context_e call_context,
                     kmp_int32 argc, microtask_t microtask, launch_t invoker,
                     kmp_info_t *master_th, kmp_team_t *parent_team,
1613#if OMPT_SUPPORT1
                     ompt_data_t *ompt_parallel_data, void **return_address,
                     ompt_data_t **parent_task_data,
1616#endif
                     kmp_va_list ap) {
kmp_team_t *team;
int i;
void **argv;

1622/* josh todo: hypothetical question: what do we do for OS X*? */
1623#if KMP_OS_LINUX1 &&                                                            \
  (KMP_ARCH_X860 || KMP_ARCH_X86_641 || KMP_ARCH_ARM || KMP_ARCH_AARCH640)
void *args[argc];
1626#else
void **args = (void **)KMP_ALLOCA(argc * sizeof(void *))__builtin_alloca (argc * sizeof(void *));
1628#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \
        KMP_ARCH_AARCH64) */

KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_serial_fork_call: T#%d serializing parallel region\n"
, gtid); }
    20, ("__kmp_serial_fork_call: T#%d serializing parallel region\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_serial_fork_call: T#%d serializing parallel region\n"
, gtid); };

__kmpc_serialized_parallel(loc, gtid);

1636#if OMPD_SUPPORT1
master_th->th.th_serial_team->t.t_pkfn = microtask;
1638#endif

if (call_context == fork_context_intel) {
  /* TODO this sucks, use the compiler itself to pass args! :) */
  master_th->th.th_serial_team->t.t_ident = loc;
  if (!ap) {
    // revert change made in __kmpc_serialized_parallel()
    master_th->th.th_serial_team->t.t_level--;
1646// Get args from parent team for teams construct

1648#if OMPT_SUPPORT1
    void *dummy;
    void **exit_frame_p;
    ompt_task_info_t *task_info;
    ompt_lw_taskteam_t lw_taskteam;

    if (ompt_enabled.enabled) {
      __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
                              ompt_parallel_data, *return_address);

      __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
      // don't use lw_taskteam after linking. content was swaped
      task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
      exit_frame_p = &(task_info->frame.exit_frame.ptr);
      if (ompt_enabled.ompt_callback_implicit_task) {
        OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num = __kmp_tid_from_gtid(gtid);
        ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
            ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
)),
            &(task_info->task_data), 1,
            OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
      }

      /* OMPT state */
      master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
    } else {
      exit_frame_p = &dummy;
    }
1675#endif

    {
      KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
      KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
      __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1681#if OMPT_SUPPORT1
                             ,
                             exit_frame_p
1684#endif
                             );
    }

1688#if OMPT_SUPPORT1
    if (ompt_enabled.enabled) {
      *exit_frame_p = NULL__null;
      if (ompt_enabled.ompt_callback_implicit_task) {
        ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
            ompt_scope_end, NULL__null, &(task_info->task_data), 1,
            OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
      }
      *ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
));
      __ompt_lw_taskteam_unlink(master_th);
      if (ompt_enabled.ompt_callback_parallel_end) {
        ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
            ompt_parallel_data, *parent_task_data,
            OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
 : ompt_parallel_invoker_runtime) | ompt_parallel_team, *return_address);
      }
      master_th->th.ompt_thread_info.state = ompt_state_overhead;
    }
1705#endif
  } else if (microtask == (microtask_t)__kmp_teams_master) {
    KMP_DEBUG_ASSERT(master_th->th.th_team == master_th->th.th_serial_team)if (!(master_th->th.th_team == master_th->th.th_serial_team
)) { __kmp_debug_assert("master_th->th.th_team == master_th->th.th_serial_team"
, "openmp/runtime/src/kmp_runtime.cpp", 1707); };
    team = master_th->th.th_team;
    // team->t.t_pkfn = microtask;
    team->t.t_invoke = invoker;
    __kmp_alloc_argv_entries(argc, team, TRUE(!0));
    team->t.t_argc = argc;
    argv = (void **)team->t.t_argv;
    if (ap) {
      for (i = argc - 1; i >= 0; --i)
        *argv++ = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
    } else {
      for (i = 0; i < argc; ++i)
        // Get args from parent team for teams construct
        argv[i] = parent_team->t.t_argv[i];
    }
    // AC: revert change made in __kmpc_serialized_parallel()
    //     because initial code in teams should have level=0
    team->t.t_level--;
    // AC: call special invoker for outer "parallel" of teams construct
    invoker(gtid);
1727#if OMPT_SUPPORT1
    if (ompt_enabled.enabled) {
      ompt_task_info_t *task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
      if (ompt_enabled.ompt_callback_implicit_task) {
        ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
            ompt_scope_end, NULL__null, &(task_info->task_data), 0,
            OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_initial);
      }
      if (ompt_enabled.ompt_callback_parallel_end) {
        ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
            ompt_parallel_data, *parent_task_data,
            OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
 : ompt_parallel_invoker_runtime) | ompt_parallel_league,
            *return_address);
      }
      master_th->th.ompt_thread_info.state = ompt_state_overhead;
    }
1743#endif
  } else {
    argv = args;
    for (i = argc - 1; i >= 0; --i)
      *argv++ = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
    KMP_MB();

1750#if OMPT_SUPPORT1
    void *dummy;
    void **exit_frame_p;
    ompt_task_info_t *task_info;
    ompt_lw_taskteam_t lw_taskteam;
    ompt_data_t *implicit_task_data;

    if (ompt_enabled.enabled) {
      __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
                              ompt_parallel_data, *return_address);
      __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
      // don't use lw_taskteam after linking. content was swaped
      task_info = OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info));
      exit_frame_p = &(task_info->frame.exit_frame.ptr);

      /* OMPT implicit task begin */
      implicit_task_data = OMPT_CUR_TASK_DATA(master_th)(&(master_th->th.th_current_task->ompt_task_info.task_data
));
      if (ompt_enabled.ompt_callback_implicit_task) {
        ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
            ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
)),
            implicit_task_data, 1, __kmp_tid_from_gtid(gtid),
            ompt_task_implicit);
        OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num = __kmp_tid_from_gtid(gtid);
      }

      /* OMPT state */
      master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
    } else {
      exit_frame_p = &dummy;
    }
1780#endif

    {
      KMP_TIME_PARTITIONED_BLOCK(OMP_parallel)((void)0);
      KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK)((void)0);
      __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1786#if OMPT_SUPPORT1
                             ,
                             exit_frame_p
1789#endif
                             );
    }

1793#if OMPT_SUPPORT1
    if (ompt_enabled.enabled) {
      *exit_frame_p = NULL__null;
      if (ompt_enabled.ompt_callback_implicit_task) {
        ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
            ompt_scope_end, NULL__null, &(task_info->task_data), 1,
            OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
      }

      *ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
));
      __ompt_lw_taskteam_unlink(master_th);
      if (ompt_enabled.ompt_callback_parallel_end) {
        ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
            ompt_parallel_data, *parent_task_data,
            OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
 : ompt_parallel_invoker_runtime) | ompt_parallel_team, *return_address);
      }
      master_th->th.ompt_thread_info.state = ompt_state_overhead;
    }
1811#endif
  }
} else if (call_context == fork_context_gnu) {
1814#if OMPT_SUPPORT1
  if (ompt_enabled.enabled) {
    ompt_lw_taskteam_t lwt;
    __ompt_lw_taskteam_init(&lwt, master_th, gtid, ompt_parallel_data,
                            *return_address);

    lwt.ompt_task_info.frame.exit_frame = ompt_data_none{0};
    __ompt_lw_taskteam_link(&lwt, master_th, 1);
  }
1823// don't use lw_taskteam after linking. content was swaped
1824#endif

  // we were called from GNU native code
  KA_TRACE(20, ("__kmp_serial_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_serial_fork_call: T#%d serial exit\n"
, gtid); };
  return FALSE0;
} else {
  KMP_ASSERT2(call_context < fork_context_last,if (!(call_context < fork_context_last)) { __kmp_debug_assert
(("__kmp_serial_fork_call: unknown fork_context parameter"), "openmp/runtime/src/kmp_runtime.cpp"
, 1831); }
              "__kmp_serial_fork_call: unknown fork_context parameter")if (!(call_context < fork_context_last)) { __kmp_debug_assert
(("__kmp_serial_fork_call: unknown fork_context parameter"), "openmp/runtime/src/kmp_runtime.cpp"
, 1831); };
}

KA_TRACE(20, ("__kmp_serial_fork_call: T#%d serial exit\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_serial_fork_call: T#%d serial exit\n"
, gtid); };
KMP_MB();
return FALSE0;
1837}

1839/* most of the work for a fork */
1840/* return true if we really went parallel, false if serialized */
1841int __kmp_fork_call(ident_t *loc, int gtid,
                  enum fork_context_e call_context, // Intel, GNU, ...
                  kmp_int32 argc, microtask_t microtask, launch_t invoker,
                  kmp_va_list ap) {
void **argv;
int i;
int master_tid;
int master_this_cons;
kmp_team_t *team;
kmp_team_t *parent_team;
kmp_info_t *master_th;
kmp_root_t *root;
int nthreads;
int master_active;
int master_set_numthreads;
int level;
int active_level;
int teams_level;
1859#if KMP_NESTED_HOT_TEAMS1
kmp_hot_team_ptr_t **p_hot_teams;
1861#endif
{ // KMP_TIME_BLOCK
  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call)((void)0);
  KMP_COUNT_VALUE(OMP_PARALLEL_args, argc)((void)0);

  KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: enter T#%d\n"
, gtid); };
  if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL__null) {
    /* Some systems prefer the stack for the root thread(s) to start with */
    /* some gap from the parent stack to prevent false sharing. */
    void *dummy = KMP_ALLOCA(__kmp_stkpadding)__builtin_alloca (__kmp_stkpadding);
    /* These 2 lines below are so this does not get optimized out */
    if (__kmp_stkpadding > KMP_MAX_STKPADDING(2 * 1024 * 1024))
      __kmp_stkpadding += (short)((kmp_int64)dummy);
  }

  /* initialize if needed */
  KMP_DEBUG_ASSERT(if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 1878); }
      __kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 1878); }; // AC: potentially unsafe, not in sync with shutdown
  if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
    __kmp_parallel_initialize();
  __kmp_resume_if_soft_paused();

  /* setup current data */
  // AC: potentially unsafe, not in sync with library shutdown,
  // __kmp_threads can be freed
  master_th = __kmp_threads[gtid];

  parent_team = master_th->th.th_team;
  master_tid = master_th->th.th_info.ds.ds_tid;
  master_this_cons = master_th->th.th_local.this_construct;
  root = master_th->th.th_root;
  master_active = root->r.r_active;
  master_set_numthreads = master_th->th.th_set_nproc;

1895#if OMPT_SUPPORT1
  ompt_data_t ompt_parallel_data = ompt_data_none{0};
  ompt_data_t *parent_task_data;
  ompt_frame_t *ompt_frame;
  void *return_address = NULL__null;

  if (ompt_enabled.enabled) {
    __ompt_get_task_info_internal(0, NULL__null, &parent_task_data, &ompt_frame,
                                  NULL__null, NULL__null);
    return_address = OMPT_LOAD_RETURN_ADDRESS(gtid)__ompt_load_return_address(gtid);
  }
1906#endif

  // Assign affinity to root thread if it hasn't happened yet
  __kmp_assign_root_init_mask();

  // Nested level will be an index in the nested nthreads array
  level = parent_team->t.t_level;
  // used to launch non-serial teams even if nested is not allowed
  active_level = parent_team->t.t_active_level;
  // needed to check nesting inside the teams
  teams_level = master_th->th.th_teams_level;
1917#if KMP_NESTED_HOT_TEAMS1
  p_hot_teams = &master_th->th.th_hot_teams;
  if (*p_hot_teams == NULL__null && __kmp_hot_teams_max_level > 0) {
    *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "openmp/runtime/src/kmp_runtime.cpp", 1921)
        sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level)___kmp_allocate((sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level
), "openmp/runtime/src/kmp_runtime.cpp", 1921);
    (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
    // it is either actual or not needed (when active_level > 0)
    (*p_hot_teams)[0].hot_team_nth = 1;
  }
1926#endif

1928#if OMPT_SUPPORT1
  if (ompt_enabled.enabled) {
    if (ompt_enabled.ompt_callback_parallel_begin) {
      int team_size = master_set_numthreads
                          ? master_set_numthreads
                          : get__nproc_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc);
      int flags = OMPT_INVOKER(call_context)((call_context == fork_context_gnu) ? ompt_parallel_invoker_program
 : ompt_parallel_invoker_runtime) |
                  ((microtask == (microtask_t)__kmp_teams_master)
                       ? ompt_parallel_league
                       : ompt_parallel_team);
      ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)ompt_callback_parallel_begin_callback(
          parent_task_data, ompt_frame, &ompt_parallel_data, team_size, flags,
          return_address);
    }
    master_th->th.ompt_thread_info.state = ompt_state_overhead;
  }
1944#endif

  master_th->th.th_ident = loc;

  // Parallel closely nested in teams construct:
  if (__kmp_is_fork_in_teams(master_th, microtask, level, teams_level, ap)) {
    return __kmp_fork_in_teams(loc, gtid, parent_team, argc, master_th, root,
                               call_context, microtask, invoker,
                               master_set_numthreads, level,
1953#if OMPT_SUPPORT1
                               ompt_parallel_data, return_address,
1955#endif
                               ap);
  } // End parallel closely nested in teams construct

1959#if KMP_DEBUG1
  if (__kmp_tasking_mode != tskm_immediate_exec) {
    KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1962); }
                     parent_team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 1962); };
  }
1964#endif

  // Need this to happen before we determine the number of threads, not while
  // we are allocating the team
  //__kmp_push_current_task_to_thread(master_th, parent_team, 0);

  // Determine the number of threads
  int enter_teams =
      __kmp_is_entering_teams(active_level, level, teams_level, ap);
  if ((!enter_teams &&
       (parent_team->t.t_active_level >=
        master_th->th.th_current_task->td_icvs.max_active_levels)) ||
      (__kmp_library == library_serial)) {
    KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: T#%d serializing team\n"
, gtid); };
    nthreads = 1;
  } else {
    nthreads = master_set_numthreads
                   ? master_set_numthreads
                   // TODO: get nproc directly from current task
                   : get__nproc_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.nproc);
    // Check if we need to take forkjoin lock? (no need for serialized
    // parallel out of teams construct).
    if (nthreads > 1) {
      /* determine how many new threads we can use */
      __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
      /* AC: If we execute teams from parallel region (on host), then teams
         should be created but each can only have 1 thread if nesting is
         disabled. If teams called from serial region, then teams and their
         threads should be created regardless of the nesting setting. */
      nthreads = __kmp_reserve_threads(root, parent_team, master_tid,
                                       nthreads, enter_teams);
      if (nthreads == 1) {
        // Free lock for single thread execution here; for multi-thread
        // execution it will be freed later after team of threads created
        // and initialized
        __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
      }
    }
  }
  KMP_DEBUG_ASSERT(nthreads > 0)if (!(nthreads > 0)) { __kmp_debug_assert("nthreads > 0"
, "openmp/runtime/src/kmp_runtime.cpp", 2003); };

  // If we temporarily changed the set number of threads then restore it now
  master_th->th.th_set_nproc = 0;

  if (nthreads == 1) {
    return __kmp_serial_fork_call(loc, gtid, call_context, argc, microtask,
                                  invoker, master_th, parent_team,
2011#if OMPT_SUPPORT1
                                  &ompt_parallel_data, &return_address,
                                  &parent_task_data,
2014#endif
                                  ap);
  } // if (nthreads == 1)

  // GEH: only modify the executing flag in the case when not serialized
  //      serialized case is handled in kmpc_serialized_parallel
  KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
 "curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
                "curtask=%p, curtask_max_aclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
 "curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
                parent_team->t.t_active_level, master_th,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
 "curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
                master_th->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
 "curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); }
                master_th->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
 "curtask=%p, curtask_max_aclevel=%d\n", parent_team->t.t_active_level
, master_th, master_th->th.th_current_task, master_th->
th.th_current_task->td_icvs.max_active_levels); };
  // TODO: GEH - cannot do this assertion because root thread not set up as
  // executing
  // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
  master_th->th.th_current_task->td_flags.executing = 0;

  if (!master_th->th.th_teams_microtask || level > teams_level) {
    /* Increment our nested depth level */
    KMP_ATOMIC_INC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_add(1, std::memory_order_acq_rel
);
  }

  // See if we need to make a copy of the ICVs.
  int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
  if ((level + 1 < __kmp_nested_nth.used) &&
      (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
    nthreads_icv = __kmp_nested_nth.nth[level + 1];
  } else {
    nthreads_icv = 0; // don't update
  }

  // Figure out the proc_bind_policy for the new team.
  kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
  // proc_bind_default means don't update
  kmp_proc_bind_t proc_bind_icv = proc_bind_default;
  if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
    proc_bind = proc_bind_false;
  } else {
    // No proc_bind clause specified; use current proc-bind-var for this
    // parallel region
    if (proc_bind == proc_bind_default) {
      proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
    }
    // Have teams construct take proc_bind value from KMP_TEAMS_PROC_BIND
    if (master_th->th.th_teams_microtask &&
        microtask == (microtask_t)__kmp_teams_master) {
      proc_bind = __kmp_teams_proc_bind;
    }
    /* else: The proc_bind policy was specified explicitly on parallel clause.
       This overrides proc-bind-var for this parallel region, but does not
       change proc-bind-var. */
    // Figure the value of proc-bind-var for the child threads.
    if ((level + 1 < __kmp_nested_proc_bind.used) &&
        (__kmp_nested_proc_bind.bind_types[level + 1] !=
         master_th->th.th_current_task->td_icvs.proc_bind)) {
      // Do not modify the proc bind icv for the two teams construct forks
      // They just let the proc bind icv pass through
      if (!master_th->th.th_teams_microtask ||
          !(microtask == (microtask_t)__kmp_teams_master || ap == NULL__null))
        proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
    }
  }

  // Reset for next parallel region
  master_th->th.th_set_proc_bind = proc_bind_default;

  if ((nthreads_icv > 0) || (proc_bind_icv != proc_bind_default)) {
    kmp_internal_control_t new_icvs;
    copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
    new_icvs.next = NULL__null;
    if (nthreads_icv > 0) {
      new_icvs.nproc = nthreads_icv;
    }
    if (proc_bind_icv != proc_bind_default) {
      new_icvs.proc_bind = proc_bind_icv;
    }

    /* allocate a new parallel team */
    KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); };
    team = __kmp_allocate_team(root, nthreads, nthreads,
2093#if OMPT_SUPPORT1
                               ompt_parallel_data,
2095#endif
                               proc_bind, &new_icvs,
                               argc USE_NESTED_HOT_ARG(master_th), master_th);
    if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar)
      copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs);
  } else {
    /* allocate a new parallel team */
    KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: before __kmp_allocate_team\n"
); };
    team = __kmp_allocate_team(root, nthreads, nthreads,
2104#if OMPT_SUPPORT1
                               ompt_parallel_data,
2106#endif
                               proc_bind,
                               &master_th->th.th_current_task->td_icvs,
                               argc USE_NESTED_HOT_ARG(master_th), master_th);
    if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar)
      copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs,
                &master_th->th.th_current_task->td_icvs);
  }
  KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); }
      10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n"
, team); };

  /* setup the new team */
  KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid)if ((team->t.t_master_tid) != (master_tid)) (team->t.t_master_tid
) = (master_tid);
  KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons)if ((team->t.t_master_this_cons) != (master_this_cons)) (team
->t.t_master_this_cons) = (master_this_cons);
  KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
);
  KMP_CHECK_UPDATE(team->t.t_parent, parent_team)if ((team->t.t_parent) != (parent_team)) (team->t.t_parent
) = (parent_team);
  KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask)if ((team->t.t_pkfn) != (microtask)) (((team->t.t_pkfn)
)) = (((microtask)));
2123#if OMPT_SUPPORT1
  KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)))
                        return_address)if ((team->t.ompt_team_info.master_return_address) != (return_address
)) (((team->t.ompt_team_info.master_return_address))) = ((
(return_address)));
2126#endif
  KMP_CHECK_UPDATE(team->t.t_invoke, invoker)if ((team->t.t_invoke) != (invoker)) (team->t.t_invoke)
 = (invoker); // TODO move to root, maybe
  // TODO: parent_team->t.t_level == INT_MAX ???
  if (!master_th->th.th_teams_microtask || level > teams_level) {
    int new_level = parent_team->t.t_level + 1;
    KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
 = (new_level);
    new_level = parent_team->t.t_active_level + 1;
    KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level);
  } else {
    // AC: Do not increase parallel level at start of the teams construct
    int new_level = parent_team->t.t_level;
    KMP_CHECK_UPDATE(team->t.t_level, new_level)if ((team->t.t_level) != (new_level)) (team->t.t_level)
 = (new_level);
    new_level = parent_team->t.t_active_level;
    KMP_CHECK_UPDATE(team->t.t_active_level, new_level)if ((team->t.t_active_level) != (new_level)) (team->t.t_active_level
) = (new_level);
  }
  kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid)((parent_team)->t.t_threads[(master_tid)]->th.th_current_task
->td_icvs.sched);
  // set primary thread's schedule as new run-time schedule
  KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched)if ((team->t.t_sched.sched) != (new_sched.sched)) (team->
t.t_sched.sched) = (new_sched.sched);

  KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq)if ((team->t.t_cancel_request) != (cancel_noreq)) (team->
t.t_cancel_request) = (cancel_noreq);
  KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator)if ((team->t.t_def_allocator) != (master_th->th.th_def_allocator
)) (team->t.t_def_allocator) = (master_th->th.th_def_allocator
);

  // Update the floating point rounding in the team if required.
  propagateFPControl(team);
2150#if OMPD_SUPPORT1
  if (ompd_state & OMPD_ENABLE_BP0x1)
    ompd_bp_parallel_begin();
2153#endif

  if (__kmp_tasking_mode != tskm_immediate_exec) {
    // Set primary thread's task team to team's task team. Unless this is hot
    // team, it should be NULL.
    KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2159); }
                     parent_team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == parent_team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2159); };
    KA_TRACE(20, ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
 "%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
                  "%p, new task_team %p / team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
 "%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
                  __kmp_gtid_from_thread(master_th),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
 "%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
                  master_th->th.th_task_team, parent_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
 "%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); }
                  team->t.t_task_team[master_th->th.th_task_state], team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
 "%p, new task_team %p / team %p\n", __kmp_gtid_from_thread(master_th
), master_th->th.th_task_team, parent_team, team->t.t_task_team
[master_th->th.th_task_state], team); };

    if (active_level || master_th->th.th_task_team) {
      // Take a memo of primary thread's task_state
      KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)if (!(master_th->th.th_task_state_memo_stack)) { __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "openmp/runtime/src/kmp_runtime.cpp"
, 2168); };
      if (master_th->th.th_task_state_top >=
          master_th->th.th_task_state_stack_sz) { // increase size
        kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
        kmp_uint8 *old_stack, *new_stack;
        kmp_uint32 i;
        new_stack = (kmp_uint8 *)__kmp_allocate(new_size)___kmp_allocate((new_size), "openmp/runtime/src/kmp_runtime.cpp"
, 2174);
        for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
          new_stack[i] = master_th->th.th_task_state_memo_stack[i];
        }
        for (i = master_th->th.th_task_state_stack_sz; i < new_size;
             ++i) { // zero-init rest of stack
          new_stack[i] = 0;
        }
        old_stack = master_th->th.th_task_state_memo_stack;
        master_th->th.th_task_state_memo_stack = new_stack;
        master_th->th.th_task_state_stack_sz = new_size;
        __kmp_free(old_stack)___kmp_free((old_stack), "openmp/runtime/src/kmp_runtime.cpp"
, 2185);
      }
      // Store primary thread's task_state on stack
      master_th->th
          .th_task_state_memo_stack[master_th->th.th_task_state_top] =
          master_th->th.th_task_state;
      master_th->th.th_task_state_top++;
2192#if KMP_NESTED_HOT_TEAMS1
      if (master_th->th.th_hot_teams &&
          active_level < __kmp_hot_teams_max_level &&
          team == master_th->th.th_hot_teams[active_level].hot_team) {
        // Restore primary thread's nested state if nested hot team
        master_th->th.th_task_state =
            master_th->th
                .th_task_state_memo_stack[master_th->th.th_task_state_top];
      } else {
2201#endif
        master_th->th.th_task_state = 0;
2203#if KMP_NESTED_HOT_TEAMS1
      }
2205#endif
    }
2207#if !KMP_NESTED_HOT_TEAMS1
    KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||if (!((master_th->th.th_task_team == __null) || (team == root
->r.r_hot_team))) { __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "openmp/runtime/src/kmp_runtime.cpp", 2209); }
                     (team == root->r.r_hot_team))if (!((master_th->th.th_task_team == __null) || (team == root
->r.r_hot_team))) { __kmp_debug_assert("(master_th->th.th_task_team == __null) || (team == root->r.r_hot_team)"
, "openmp/runtime/src/kmp_runtime.cpp", 2209); };
2210#endif
  }

  KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
      20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
      ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
       gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); }
       team->t.t_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n"
, gtid, parent_team->t.t_id, team->t.t_master_tid, team
->t.t_id, team->t.t_nproc); };
  KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||if (!(team != root->r.r_hot_team || (team->t.t_master_tid
 == 0 && (team->t.t_parent == root->r.r_root_team
 || team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2221); }
                   (team->t.t_master_tid == 0 &&if (!(team != root->r.r_hot_team || (team->t.t_master_tid
 == 0 && (team->t.t_parent == root->r.r_root_team
 || team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2221); }
                    (team->t.t_parent == root->r.r_root_team ||if (!(team != root->r.r_hot_team || (team->t.t_master_tid
 == 0 && (team->t.t_parent == root->r.r_root_team
 || team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2221); }
                     team->t.t_parent->t.t_serialized)))if (!(team != root->r.r_hot_team || (team->t.t_master_tid
 == 0 && (team->t.t_parent == root->r.r_root_team
 || team->t.t_parent->t.t_serialized)))) { __kmp_debug_assert
("team != root->r.r_hot_team || (team->t.t_master_tid == 0 && (team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized))"
, "openmp/runtime/src/kmp_runtime.cpp", 2221); };
  KMP_MB();

  /* now, setup the arguments */
  argv = (void **)team->t.t_argv;
  if (ap) {
    for (i = argc - 1; i >= 0; --i) {
      void *new_argv = va_arg(kmp_va_deref(ap), void *)__builtin_va_arg((*(ap)), void *);
      KMP_CHECK_UPDATE(*argv, new_argv)if ((*argv) != (new_argv)) (*argv) = (new_argv);
      argv++;
    }
  } else {
    for (i = 0; i < argc; ++i) {
      // Get args from parent team for teams construct
      KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i])if ((argv[i]) != (team->t.t_parent->t.t_argv[i])) (argv
[i]) = (team->t.t_parent->t.t_argv[i]);
    }
  }

  /* now actually fork the threads */
  KMP_CHECK_UPDATE(team->t.t_master_active, master_active)if ((team->t.t_master_active) != (master_active)) (team->
t.t_master_active) = (master_active);
  if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
    root->r.r_active = TRUE(!0);

  __kmp_fork_team_threads(root, team, master_th, gtid, !ap);
  __kmp_setup_icv_copy(team, nthreads,
                       &master_th->th.th_current_task->td_icvs, loc);

2248#if OMPT_SUPPORT1
  master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2250#endif

  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);

2254#if USE_ITT_BUILD1
  if (team->t.t_active_level == 1 // only report frames at level 1
      && !master_th->th.th_teams_microtask) { // not in teams construct
2257#if USE_ITT_NOTIFY1
    if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
        (__kmp_forkjoin_frames_mode == 3 ||
         __kmp_forkjoin_frames_mode == 1)) {
      kmp_uint64 tmp_time = 0;
      if (__itt_get_timestamp_ptr__kmp_itt_get_timestamp_ptr__3_0)
        tmp_time = __itt_get_timestamp(!__kmp_itt_get_timestamp_ptr__3_0) ? 0 : __kmp_itt_get_timestamp_ptr__3_0();
      // Internal fork - report frame begin
      master_th->th.th_frame_time = tmp_time;
      if (__kmp_forkjoin_frames_mode == 3)
        team->t.t_region_time = tmp_time;
    } else
2269// only one notification scheme (either "submit" or "forking/joined", not both)
2270#endif /* USE_ITT_NOTIFY */
      if ((__itt_frame_begin_v3_ptr__kmp_itt_frame_begin_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
          __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
        // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer.
        __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
      }
  }
2277#endif /* USE_ITT_BUILD */

  /* now go on and do the work */
  KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team)if (!(team == __kmp_threads[gtid]->th.th_team)) { __kmp_debug_assert
("team == __kmp_threads[gtid]->th.th_team", "openmp/runtime/src/kmp_runtime.cpp"
, 2280); };
  KMP_MB();
  KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
           ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); }
            root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n"
, root, team, master_th, gtid); };

2286#if USE_ITT_BUILD1
  if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
    // create new stack stitching id before entering fork barrier
    if (!enter_teams) {
      KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL)if (!(team->t.t_stack_id == __null)) { __kmp_debug_assert(
"team->t.t_stack_id == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2290); };
      team->t.t_stack_id = __kmp_itt_stack_caller_create();
    } else if (parent_team->t.t_serialized) {
      // keep stack stitching id in the serialized parent_team;
      // current team will be used for parallel inside the teams;
      // if parent_team is active, then it already keeps stack stitching id
      // for the league of teams
      KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL)if (!(parent_team->t.t_stack_id == __null)) { __kmp_debug_assert
("parent_team->t.t_stack_id == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2297); };
      parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
    }
  }
2301#endif /* USE_ITT_BUILD */

  // AC: skip __kmp_internal_fork at teams construct, let only primary
  // threads execute
  if (ap) {
    __kmp_internal_fork(loc, gtid, team);
    KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
                  "master_th=%p, gtid=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, team, master_th, gtid); }
                  root, team, master_th, gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_internal_fork : after : root=%p, team=%p, "
 "master_th=%p, gtid=%d\n", root, team, master_th, gtid); };
  }

  if (call_context == fork_context_gnu) {
    KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); };
    return TRUE(!0);
  }

  /* Invoke microtask for PRIMARY thread */
  KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
                team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); };
} // END of timer KMP_fork_call block

2322#if KMP_STATS_ENABLED0
// If beginning a teams construct, then change thread state
stats_state_e previous_state = KMP_GET_THREAD_STATE()((void)0);
if (!ap) {
  KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION)((void)0);
}
2328#endif

if (!team->t.t_invoke(gtid)) {
  KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread")if (!(0)) { __kmp_debug_assert(("cannot invoke microtask for PRIMARY thread"
), "openmp/runtime/src/kmp_runtime.cpp", 2331); };
}

2334#if KMP_STATS_ENABLED0
// If was beginning of a teams construct, then reset thread state
if (!ap) {
  KMP_SET_THREAD_STATE(previous_state)((void)0);
}
2339#endif

KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); }
              team->t.t_id, team->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n"
, gtid, team->t.t_id, team->t.t_pkfn); };
KMP_MB(); /* Flush all pending memory write invalidates.  */

KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_fork_call: parallel exit T#%d\n"
, gtid); };
2346#if OMPT_SUPPORT1
if (ompt_enabled.enabled) {
  master_th->th.ompt_thread_info.state = ompt_state_overhead;
}
2350#endif

return TRUE(!0);
2353}

2355#if OMPT_SUPPORT1
2356static inline void __kmp_join_restore_state(kmp_info_t *thread,
                                          kmp_team_t *team) {
// restore state outside the region
thread->th.ompt_thread_info.state =
    ((team->t.t_serialized) ? ompt_state_work_serial
                            : ompt_state_work_parallel);
2362}

2364static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread,
                                 kmp_team_t *team, ompt_data_t *parallel_data,
                                 int flags, void *codeptr) {
ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
if (ompt_enabled.ompt_callback_parallel_end) {
  ompt_callbacks.ompt_callback(ompt_callback_parallel_end)ompt_callback_parallel_end_callback(
      parallel_data, &(task_info->task_data), flags, codeptr);
}

task_info->frame.enter_frame = ompt_data_none{0};
__kmp_join_restore_state(thread, team);
2375}
2376#endif

2378void __kmp_join_call(ident_t *loc, int gtid
2379#if OMPT_SUPPORT1
                   ,
                   enum fork_context_e fork_context
2382#endif
                   ,
                   int exit_teams) {
KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call)((void)0);
kmp_team_t *team;
kmp_team_t *parent_team;
kmp_info_t *master_th;
kmp_root_t *root;
int master_active;

KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: enter T#%d\n"
, gtid); };
1
Assuming 'kmp_a_debug' is < 20→
2
←
Taking false branch→

/* setup current data */
master_th = __kmp_threads[gtid];
root = master_th->th.th_root;
team = master_th->th.th_team;
parent_team = team->t.t_parent;

master_th->th.th_ident = loc;

2402#if OMPT_SUPPORT1
void *team_microtask = (void *)team->t.t_pkfn;
// For GOMP interface with serialized parallel, need the
// __kmpc_end_serialized_parallel to call hooks for OMPT end-implicit-task
// and end-parallel events.
if (ompt_enabled.enabled &&
3
←
Assuming field 'enabled' is 0→
    !(team->t.t_serialized && fork_context == fork_context_gnu)) {
  master_th->th.ompt_thread_info.state = ompt_state_overhead;
}
2411#endif

2413#if KMP_DEBUG1
if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
4
←
Assuming '__kmp_tasking_mode' is equal to tskm_immediate_exec→
  KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
 "th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
                "th_task_team = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
 "th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
                __kmp_gtid_from_thread(master_th), team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
 "th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
                team->t.t_task_team[master_th->th.th_task_state],if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
 "th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); }
                master_th->th.th_task_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
 "th_task_team = %p\n", __kmp_gtid_from_thread(master_th), team
, team->t.t_task_team[master_th->th.th_task_state], master_th
->th.th_task_team); };
  KMP_DEBUG_ASSERT(master_th->th.th_task_team ==if (!(master_th->th.th_task_team == team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2421); }
                   team->t.t_task_team[master_th->th.th_task_state])if (!(master_th->th.th_task_team == team->t.t_task_team
[master_th->th.th_task_state])) { __kmp_debug_assert("master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state]"
, "openmp/runtime/src/kmp_runtime.cpp", 2421); };
}
2423#endif

if (team->t.t_serialized) {
5
←
Assuming field 't_serialized' is 0→
6
←
Taking false branch→
  if (master_th->th.th_teams_microtask) {
    // We are in teams construct
    int level = team->t.t_level;
    int tlevel = master_th->th.th_teams_level;
    if (level == tlevel) {
      // AC: we haven't incremented it earlier at start of teams construct,
      //     so do it here - at the end of teams construct
      team->t.t_level++;
    } else if (level == tlevel + 1) {
      // AC: we are exiting parallel inside teams, need to increment
      // serialization in order to restore it in the next call to
      // __kmpc_end_serialized_parallel
      team->t.t_serialized++;
    }
  }
  __kmpc_end_serialized_parallel(loc, gtid);

2443#if OMPT_SUPPORT1
  if (ompt_enabled.enabled) {
    if (fork_context == fork_context_gnu) {
      __ompt_lw_taskteam_unlink(master_th);
    }
    __kmp_join_restore_state(master_th, parent_team);
  }
2450#endif

  return;
}

master_active = team->t.t_master_active;

if (!exit_teams) {
7
←
Assuming 'exit_teams' is 0→
8
←
Taking true branch→
  // AC: No barrier for internal teams at exit from teams construct.
  //     But there is barrier for external team (league).
  __kmp_internal_join(loc, gtid, team);
2461#if USE_ITT_BUILD1
  if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
9
←
Assuming '__kmp_itt_stack_caller_create_ptr__3_0' is null→
10
←
Taking false branch→
    KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL)if (!(team->t.t_stack_id != __null)) { __kmp_debug_assert(
"team->t.t_stack_id != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2463); };
    // destroy the stack stitching id after join barrier
    __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id);
    team->t.t_stack_id = NULL__null;
  }
2468#endif
} else {
  master_th->th.th_task_state =
      0; // AC: no tasking in teams (out of any parallel)
2472#if USE_ITT_BUILD1
  if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0 && parent_team->t.t_serialized) {
    KMP_DEBUG_ASSERT(parent_team->t.t_stack_id != NULL)if (!(parent_team->t.t_stack_id != __null)) { __kmp_debug_assert
("parent_team->t.t_stack_id != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2474); };
    // destroy the stack stitching id on exit from the teams construct
    // if parent_team is active, then the id will be destroyed later on
    // by master of the league of teams
    __kmp_itt_stack_caller_destroy((__itt_caller)parent_team->t.t_stack_id);
    parent_team->t.t_stack_id = NULL__null;
  }
2481#endif

  if (team->t.t_nproc > 1 &&
      __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
    team->t.b->update_num_threads(team->t.t_nproc);
    __kmp_add_threads_to_team(team, team->t.t_nproc);
  }
}

KMP_MB();

2492#if OMPT_SUPPORT1
ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
void *codeptr = team->t.ompt_team_info.master_return_address;
2495#endif

2497#if USE_ITT_BUILD1
// Mark end of "parallel" region for Intel(R) VTune(TM) analyzer.
if (team->t.t_active_level == 1 &&
11
←
Assuming field 't_active_level' is not equal to 1→
    (!master_th->th.th_teams_microtask || /* not in teams construct */
     master_th->th.th_teams_size.nteams == 1)) {
  master_th->th.th_ident = loc;
  // only one notification scheme (either "submit" or "forking/joined", not
  // both)
  if ((__itt_frame_submit_v3_ptr__kmp_itt_frame_submit_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
      __kmp_forkjoin_frames_mode == 3)
    __kmp_itt_frame_submit(gtid, team->t.t_region_time,
                           master_th->th.th_frame_time, 0, loc,
                           master_th->th.th_team_nproc, 1);
  else if ((__itt_frame_end_v3_ptr__kmp_itt_frame_end_v3_ptr__3_0 || KMP_ITT_DEBUG0) &&
           !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
    __kmp_itt_region_joined(gtid);
} // active_level == 1
2514#endif /* USE_ITT_BUILD */

2516#if KMP_AFFINITY_SUPPORTED1
if (!exit_teams11.1
'exit_teams' is 0
) {
12
←
Taking true branch→
  // Restore master thread's partition.
  master_th->th.th_first_place = team->t.t_first_place;
  master_th->th.th_last_place = team->t.t_last_place;
}
2522#endif // KMP_AFFINITY_SUPPORTED

if (master_th->th.th_teams_microtask && !exit_teams13.1
'exit_teams' is 0
 &&
13
←
Assuming field 'th_teams_microtask' is non-null→
16
←
Taking true branch→
    team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
14
←
Assuming field 't_pkfn' is not equal to __kmp_teams_master→
    team->t.t_level == master_th->th.th_teams_level + 1) {
15
←
Assuming the condition is true→
2527// AC: We need to leave the team structure intact at the end of parallel
2528// inside the teams construct, so that at the next parallel same (hot) team
2529// works, only adjust nesting levels
2530#if OMPT_SUPPORT1
  ompt_data_t ompt_parallel_data = ompt_data_none{0};
  if (ompt_enabled.enabled) {
17
←
Assuming field 'enabled' is 0→
18
←
Taking false branch→
    ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
    if (ompt_enabled.ompt_callback_implicit_task) {
      int ompt_team_size = team->t.t_nproc;
      ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
          ompt_scope_end, NULL__null, &(task_info->task_data), ompt_team_size,
          OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, ompt_task_implicit);
    }
    task_info->frame.exit_frame = ompt_data_none{0};
    task_info->task_data = ompt_data_none{0};
    ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th)(&(master_th->th.th_team->t.ompt_team_info.parallel_data
));
    __ompt_lw_taskteam_unlink(master_th);
  }
2545#endif
  /* Decrement our nested depth level */
  team->t.t_level--;
  team->t.t_active_level--;
  KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
);

  // Restore number of threads in the team if needed. This code relies on
  // the proper adjustment of th_teams_size.nth after the fork in
  // __kmp_teams_master on each teams primary thread in the case that
  // __kmp_reserve_threads reduced it.
  if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
19
←
Assuming field 'th_team_nproc' is < field 'nth'→
20
←
Taking true branch→
    int old_num = master_th->th.th_team_nproc;
    int new_num = master_th->th.th_teams_size.nth;
    kmp_info_t **other_threads = team->t.t_threads;
    team->t.t_nproc = new_num;
    for (int i = 0; i < old_num; ++i) {
21
←
Assuming 'i' is < 'old_num'→
22
←
Loop condition is true.  Entering loop body→
23
←
Assuming 'i' is >= 'old_num'→
24
←
Loop condition is false. Execution continues on line 2564→
      other_threads[i]->th.th_team_nproc = new_num;
    }
    // Adjust states of non-used threads of the team
    for (int i = old_num; i < new_num; ++i) {
25
←
Assuming 'i' is < 'new_num'→
26
←
Loop condition is true.  Entering loop body→
      // Re-initialize thread's barrier data.
      KMP_DEBUG_ASSERT(other_threads[i])if (!(other_threads[i])) { __kmp_debug_assert("other_threads[i]"
, "openmp/runtime/src/kmp_runtime.cpp", 2566); };
27
←
Assuming the condition is true→
28
←
Taking true branch→
      kmp_balign_t *balign = other_threads[i]->th.th_bar;
29
←
'balign' initialized to a null pointer value→
      for (int b = 0; b < bs_last_barrier; ++b) {
30
←
Loop condition is true.  Entering loop body→
        balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
31
←
Dereference of null pointer
        KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 2570); };
2571#if USE_DEBUGGER0
        balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2573#endif
      }
      if (__kmp_tasking_mode != tskm_immediate_exec) {
        // Synchronize thread's task state
        other_threads[i]->th.th_task_state = master_th->th.th_task_state;
      }
    }
  }

2582#if OMPT_SUPPORT1
  if (ompt_enabled.enabled) {
    __kmp_join_ompt(gtid, master_th, parent_team, &ompt_parallel_data,
                    OMPT_INVOKER(fork_context)((fork_context == fork_context_gnu) ? ompt_parallel_invoker_program
 : ompt_parallel_invoker_runtime) | ompt_parallel_team, codeptr);
  }
2587#endif

  return;
}

/* do cleanup and restore the parent team */
master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
master_th->th.th_local.this_construct = team->t.t_master_this_cons;

master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];

/* jc: The following lock has instructions with REL and ACQ semantics,
   separating the parallel user code called in this parallel region
   from the serial user code called after this function returns. */
__kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);

if (!master_th->th.th_teams_microtask ||
    team->t.t_level > master_th->th.th_teams_level) {
  /* Decrement our nested depth level */
  KMP_ATOMIC_DEC(&root->r.r_in_parallel)(&root->r.r_in_parallel)->fetch_sub(1, std::memory_order_acq_rel
);
}
KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0)if (!(root->r.r_in_parallel >= 0)) { __kmp_debug_assert
("root->r.r_in_parallel >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 2608); };

2610#if OMPT_SUPPORT1
if (ompt_enabled.enabled) {
  ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
  if (ompt_enabled.ompt_callback_implicit_task) {
    int flags = (team_microtask == (void *)__kmp_teams_master)
                    ? ompt_task_initial
                    : ompt_task_implicit;
    int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc;
    ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
        ompt_scope_end, NULL__null, &(task_info->task_data), ompt_team_size,
        OMPT_CUR_TASK_INFO(master_th)(&(master_th->th.th_current_task->ompt_task_info))->thread_num, flags);
  }
  task_info->frame.exit_frame = ompt_data_none{0};
  task_info->task_data = ompt_data_none{0};
}
2625#endif

KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); }
              master_th, team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n"
, 0, master_th, team); };
__kmp_pop_current_task_from_thread(master_th);

master_th->th.th_def_allocator = team->t.t_def_allocator;

2633#if OMPD_SUPPORT1
if (ompd_state & OMPD_ENABLE_BP0x1)
  ompd_bp_parallel_end();
2636#endif
updateHWFPControl(team);

if (root->r.r_active != master_active)
  root->r.r_active = master_active;

__kmp_free_team(root, team USE_NESTED_HOT_ARG(, master_th
                          master_th), master_th); // this will free worker threads

/* this race was fun to find. make sure the following is in the critical
   region otherwise assertions may fail occasionally since the old team may be
   reallocated and the hierarchy appears inconsistent. it is actually safe to
   run and won't cause any bugs, but will cause those assertion failures. it's
   only one deref&assign so might as well put this in the critical region */
master_th->th.th_team = parent_team;
master_th->th.th_team_nproc = parent_team->t.t_nproc;
master_th->th.th_team_master = parent_team->t.t_threads[0];
master_th->th.th_team_serialized = parent_team->t.t_serialized;

/* restore serialized team, if need be */
if (parent_team->t.t_serialized &&
    parent_team != master_th->th.th_serial_team &&
    parent_team != root->r.r_root_team) {
  __kmp_free_team(root,
                  master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL), __null);
  master_th->th.th_serial_team = parent_team;
}

if (__kmp_tasking_mode != tskm_immediate_exec) {
  if (master_th->th.th_task_state_top >
      0) { // Restore task state from memo stack
    KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack)if (!(master_th->th.th_task_state_memo_stack)) { __kmp_debug_assert
("master_th->th.th_task_state_memo_stack", "openmp/runtime/src/kmp_runtime.cpp"
, 2667); };
    // Remember primary thread's state if we re-use this nested hot team
    master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
        master_th->th.th_task_state;
    --master_th->th.th_task_state_top; // pop
    // Now restore state at this level
    master_th->th.th_task_state =
        master_th->th
            .th_task_state_memo_stack[master_th->th.th_task_state_top];
  }
  // Copy the task team from the parent team to the primary thread
  master_th->th.th_task_team =
      parent_team->t.t_task_team[master_th->th.th_task_state];
  KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
           ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
            __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); }
            parent_team))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n"
, __kmp_gtid_from_thread(master_th), master_th->th.th_task_team
, parent_team); };
}

// TODO: GEH - cannot do this assertion because root thread not set up as
// executing
// KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
master_th->th.th_current_task->td_flags.executing = 1;

__kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);

2693#if KMP_AFFINITY_SUPPORTED1
if (master_th->th.th_team->t.t_level == 0 && __kmp_affinity.flags.reset) {
  __kmp_reset_root_init_mask(gtid);
}
2697#endif
2698#if OMPT_SUPPORT1
int flags =
    OMPT_INVOKER(fork_context)((fork_context == fork_context_gnu) ? ompt_parallel_invoker_program
 : ompt_parallel_invoker_runtime) |
    ((team_microtask == (void *)__kmp_teams_master) ? ompt_parallel_league
                                                    : ompt_parallel_team);
if (ompt_enabled.enabled) {
  __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags,
                  codeptr);
}
2707#endif

KMP_MB();
KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_join_call: exit T#%d\n"
, gtid); };
2711}

2713/* Check whether we should push an internal control record onto the
 serial team stack.  If so, do it.  */
2715void __kmp_save_internal_controls(kmp_info_t *thread) {

if (thread->th.th_team != thread->th.th_serial_team) {
  return;
}
if (thread->th.th_team->t.t_serialized > 1) {
  int push = 0;

  if (thread->th.th_team->t.t_control_stack_top == NULL__null) {
    push = 1;
  } else {
    if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
        thread->th.th_team->t.t_serialized) {
      push = 1;
    }
  }
  if (push) { /* push a record on the serial team's stack */
    kmp_internal_control_t *control =
        (kmp_internal_control_t *)__kmp_allocate(___kmp_allocate((sizeof(kmp_internal_control_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 2734)
            sizeof(kmp_internal_control_t))___kmp_allocate((sizeof(kmp_internal_control_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 2734);

    copy_icvs(control, &thread->th.th_current_task->td_icvs);

    control->serial_nesting_level = thread->th.th_team->t.t_serialized;

    control->next = thread->th.th_team->t.t_control_stack_top;
    thread->th.th_team->t.t_control_stack_top = control;
  }
}
2744}

2746/* Changes set_nproc */
2747void __kmp_set_num_threads(int new_nth, int gtid) {
kmp_info_t *thread;
kmp_root_t *root;

KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_num_threads: new __kmp_nth = %d\n"
, new_nth); };
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2752); };

if (new_nth < 1)
  new_nth = 1;
else if (new_nth > __kmp_max_nth)
  new_nth = __kmp_max_nth;

KMP_COUNT_VALUE(OMP_set_numthreads, new_nth)((void)0);
thread = __kmp_threads[gtid];
if (thread->th.th_current_task->td_icvs.nproc == new_nth)
  return; // nothing to do

__kmp_save_internal_controls(thread);

set__nproc(thread, new_nth)(((thread)->th.th_current_task->td_icvs.nproc) = (new_nth
));

// If this omp_set_num_threads() call will cause the hot team size to be
// reduced (in the absence of a num_threads clause), then reduce it now,
// rather than waiting for the next parallel region.
root = thread->th.th_root;
if (__kmp_init_parallel && (!root->r.r_active) &&
    (root->r.r_hot_team->t.t_nproc > new_nth)
2774#if KMP_NESTED_HOT_TEAMS1
    && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2776#endif
) {
  kmp_team_t *hot_team = root->r.r_hot_team;
  int f;

  __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);

  if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
    __kmp_resize_dist_barrier(hot_team, hot_team->t.t_nproc, new_nth);
  }
  // Release the extra threads we don't need any more.
  for (f = new_nth; f < hot_team->t.t_nproc; f++) {
    KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)if (!(hot_team->t.t_threads[f] != __null)) { __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2788); };
    if (__kmp_tasking_mode != tskm_immediate_exec) {
      // When decreasing team size, threads no longer in the team should unref
      // task team.
      hot_team->t.t_threads[f]->th.th_task_team = NULL__null;
    }
    __kmp_free_thread(hot_team->t.t_threads[f]);
    hot_team->t.t_threads[f] = NULL__null;
  }
  hot_team->t.t_nproc = new_nth;
2798#if KMP_NESTED_HOT_TEAMS1
  if (thread->th.th_hot_teams) {
    KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team)if (!(hot_team == thread->th.th_hot_teams[0].hot_team)) { __kmp_debug_assert
("hot_team == thread->th.th_hot_teams[0].hot_team", "openmp/runtime/src/kmp_runtime.cpp"
, 2800); };
    thread->th.th_hot_teams[0].hot_team_nth = new_nth;
  }
2803#endif

  if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
    hot_team->t.b->update_num_threads(new_nth);
    __kmp_add_threads_to_team(hot_team, new_nth);
  }

  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);

  // Update the t_nproc field in the threads that are still active.
  for (f = 0; f < new_nth; f++) {
    KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL)if (!(hot_team->t.t_threads[f] != __null)) { __kmp_debug_assert
("hot_team->t.t_threads[f] != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 2814); };
    hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
  }
  // Special flag in case omp_set_num_threads() call
  hot_team->t.t_size_changed = -1;
}
2820}

2822/* Changes max_active_levels */
2823void __kmp_set_max_active_levels(int gtid, int max_active_levels) {
kmp_info_t *thread;

KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
 "%d = (%d)\n", gtid, max_active_levels); }
              "%d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
 "%d = (%d)\n", gtid, max_active_levels); }
              gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: new max_active_levels for thread "
 "%d = (%d)\n", gtid, max_active_levels); };
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2829); };

// validate max_active_levels
if (max_active_levels < 0) {
  KMP_WARNING(ActiveLevelsNegative, max_active_levels)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsNegative
, max_active_levels), __kmp_msg_null);
  // We ignore this call if the user has specified a negative value.
  // The current setting won't be changed. The last valid setting will be
  // used. A warning will be issued (if warnings are allowed as controlled by
  // the KMP_WARNINGS env var).
  KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
 "max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
                "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
 "max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
                gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: the call is ignored: new "
 "max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); };
  return;
}
if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647) {
  // it's OK, the max_active_levels is within the valid range: [ 0;
  // KMP_MAX_ACTIVE_LEVELS_LIMIT ]
  // We allow a zero value. (implementation defined behavior)
} else {
  KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null)
              KMP_MAX_ACTIVE_LEVELS_LIMIT)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_ActiveLevelsExceedLimit
, max_active_levels, 2147483647), __kmp_msg_null);
  max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647;
  // Current upper limit is MAX_INT. (implementation defined behavior)
  // If the input exceeds the upper limit, we correct the input to be the
  // upper limit. (implementation defined behavior)
  // Actually, the flow should never get here until we use MAX_INT limit.
}
KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
 "max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
              "max_active_levels for thread %d = (%d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
 "max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); }
              gtid, max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_max_active_levels: after validation: new "
 "max_active_levels for thread %d = (%d)\n", gtid, max_active_levels
); };

thread = __kmp_threads[gtid];

__kmp_save_internal_controls(thread);

set__max_active_levels(thread, max_active_levels)(((thread)->th.th_current_task->td_icvs.max_active_levels
) = (max_active_levels));
2865}

2867/* Gets max_active_levels */
2868int __kmp_get_max_active_levels(int gtid) {
kmp_info_t *thread;

KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d\n"
, gtid); };
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2872); };

thread = __kmp_threads[gtid];
KMP_DEBUG_ASSERT(thread->th.th_current_task)if (!(thread->th.th_current_task)) { __kmp_debug_assert("thread->th.th_current_task"
, "openmp/runtime/src/kmp_runtime.cpp", 2875); };
KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
 "curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
              "curtask_maxaclevel=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
 "curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
              gtid, thread->th.th_current_task,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
 "curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); }
              thread->th.th_current_task->td_icvs.max_active_levels))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
 "curtask_maxaclevel=%d\n", gtid, thread->th.th_current_task
, thread->th.th_current_task->td_icvs.max_active_levels
); };
return thread->th.th_current_task->td_icvs.max_active_levels;
2881}

2883// nteams-var per-device ICV
2884void __kmp_set_num_teams(int num_teams) {
if (num_teams > 0)
  __kmp_nteams = num_teams;
2887}
2888int __kmp_get_max_teams(void) { return __kmp_nteams; }
2889// teams-thread-limit-var per-device ICV
2890void __kmp_set_teams_thread_limit(int limit) {
if (limit > 0)
  __kmp_teams_thread_limit = limit;
2893}
2894int __kmp_get_teams_thread_limit(void) { return __kmp_teams_thread_limit; }

2896KMP_BUILD_ASSERT(sizeof(kmp_sched_t) == sizeof(int))static_assert(sizeof(kmp_sched_t) == sizeof(int), "Build condition error"
);
2897KMP_BUILD_ASSERT(sizeof(enum sched_type) == sizeof(int))static_assert(sizeof(enum sched_type) == sizeof(int), "Build condition error"
);

2899/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
2900void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
kmp_info_t *thread;
kmp_sched_t orig_kind;
//    kmp_team_t *team;

KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); }
              gtid, (int)kind, chunk))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n"
, gtid, (int)kind, chunk); };
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2907); };

// Check if the kind parameter is valid, correct if needed.
// Valid parameters should fit in one of two intervals - standard or extended:
//       <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
// 2008-01-25: 0,  1 - 4,       5,         100,     101 - 102, 103
orig_kind = kind;
kind = __kmp_sched_without_mods(kind);

if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
    (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
  // TODO: Hint needs attention in case we change the default schedule.
  __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind)__kmp_msg_format(kmp_i18n_msg_ScheduleKindOutOfRange, kind),
            KMP_HNT(DefaultScheduleKindUsed, "static, no chunk")__kmp_msg_format(kmp_i18n_hnt_DefaultScheduleKindUsed, "static, no chunk"
),
            __kmp_msg_null);
  kind = kmp_sched_default;
  chunk = 0; // ignore chunk value in case of bad kind
}

thread = __kmp_threads[gtid];

__kmp_save_internal_controls(thread);

if (kind < kmp_sched_upper_std) {
  if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK1) {
    // differ static chunked vs. unchunked:  chunk should be invalid to
    // indicate unchunked schedule (which is the default)
    thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
  } else {
    thread->th.th_current_task->td_icvs.sched.r_sched_type =
        __kmp_sch_map[kind - kmp_sched_lower - 1];
  }
} else {
  //    __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
  //    kmp_sched_lower - 2 ];
  thread->th.th_current_task->td_icvs.sched.r_sched_type =
      __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
                    kmp_sched_lower - 2];
}
__kmp_sched_apply_mods_intkind(
    orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type));
if (kind == kmp_sched_auto || chunk < 1) {
  // ignore parameter chunk for schedule auto
  thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK1;
} else {
  thread->th.th_current_task->td_icvs.sched.chunk = chunk;
}
2954}

2956/* Gets def_sched_var ICV values */
2957void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
kmp_info_t *thread;
enum sched_type th_type;

KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_schedule: thread %d\n"
, gtid); };
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 2962); };

thread = __kmp_threads[gtid];

th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)(enum sched_type)( (th_type) & ~(kmp_sch_modifier_nonmonotonic
 | kmp_sch_modifier_monotonic))) {
case kmp_sch_static:
case kmp_sch_static_greedy:
case kmp_sch_static_balanced:
  *kind = kmp_sched_static;
  __kmp_sched_apply_mods_stdkind(kind, th_type);
  *chunk = 0; // chunk was not set, try to show this fact via zero value
  return;
case kmp_sch_static_chunked:
  *kind = kmp_sched_static;
  break;
case kmp_sch_dynamic_chunked:
  *kind = kmp_sched_dynamic;
  break;
case kmp_sch_guided_chunked:
case kmp_sch_guided_iterative_chunked:
case kmp_sch_guided_analytical_chunked:
  *kind = kmp_sched_guided;
  break;
case kmp_sch_auto:
  *kind = kmp_sched_auto;
  break;
case kmp_sch_trapezoidal:
  *kind = kmp_sched_trapezoidal;
  break;
2992#if KMP_STATIC_STEAL_ENABLED1
case kmp_sch_static_steal:
  *kind = kmp_sched_static_steal;
  break;
2996#endif
default:
  KMP_FATAL(UnknownSchedulingType, th_type)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownSchedulingType
, th_type), __kmp_msg_null);
}

__kmp_sched_apply_mods_stdkind(kind, th_type);
*chunk = thread->th.th_current_task->td_icvs.sched.chunk;
3003}

3005int __kmp_get_ancestor_thread_num(int gtid, int level) {

int ii, dd;
kmp_team_t *team;
kmp_info_t *thr;

KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_ancestor_thread_num: thread %d %d\n"
, gtid, level); };
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 3012); };

// validate level
if (level == 0)
  return 0;
if (level < 0)
  return -1;
thr = __kmp_threads[gtid];
team = thr->th.th_team;
ii = team->t.t_level;
if (level > ii)
  return -1;

if (thr->th.th_teams_microtask) {
  // AC: we are in teams region where multiple nested teams have same level
  int tlevel = thr->th.th_teams_level; // the level of the teams construct
  if (level <=
      tlevel) { // otherwise usual algorithm works (will not touch the teams)
    KMP_DEBUG_ASSERT(ii >= tlevel)if (!(ii >= tlevel)) { __kmp_debug_assert("ii >= tlevel"
, "openmp/runtime/src/kmp_runtime.cpp", 3030); };
    // AC: As we need to pass by the teams league, we need to artificially
    // increase ii
    if (ii == tlevel) {
      ii += 2; // three teams have same level
    } else {
      ii++; // two teams have same level
    }
  }
}

if (ii == level)
  return __kmp_tid_from_gtid(gtid);

dd = team->t.t_serialized;
level++;
while (ii > level) {
  for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
  }
  if ((team->t.t_serialized) && (!dd)) {
    team = team->t.t_parent;
    continue;
  }
  if (ii > level) {
    team = team->t.t_parent;
    dd = team->t.t_serialized;
    ii--;
  }
}

return (dd > 1) ? (0) : (team->t.t_master_tid);
3061}

3063int __kmp_get_team_size(int gtid, int level) {

int ii, dd;
kmp_team_t *team;
kmp_info_t *thr;

KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_get_team_size: thread %d %d\n"
, gtid, level); };
KMP_DEBUG_ASSERT(__kmp_init_serial)if (!(__kmp_init_serial)) { __kmp_debug_assert("__kmp_init_serial"
, "openmp/runtime/src/kmp_runtime.cpp", 3070); };

// validate level
if (level == 0)
  return 1;
if (level < 0)
  return -1;
thr = __kmp_threads[gtid];
team = thr->th.th_team;
ii = team->t.t_level;
if (level > ii)
  return -1;

if (thr->th.th_teams_microtask) {
  // AC: we are in teams region where multiple nested teams have same level
  int tlevel = thr->th.th_teams_level; // the level of the teams construct
  if (level <=
      tlevel) { // otherwise usual algorithm works (will not touch the teams)
    KMP_DEBUG_ASSERT(ii >= tlevel)if (!(ii >= tlevel)) { __kmp_debug_assert("ii >= tlevel"
, "openmp/runtime/src/kmp_runtime.cpp", 3088); };
    // AC: As we need to pass by the teams league, we need to artificially
    // increase ii
    if (ii == tlevel) {
      ii += 2; // three teams have same level
    } else {
      ii++; // two teams have same level
    }
  }
}

while (ii > level) {
  for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
  }
  if (team->t.t_serialized && (!dd)) {
    team = team->t.t_parent;
    continue;
  }
  if (ii > level) {
    team = team->t.t_parent;
    ii--;
  }
}

return team->t.t_nproc;
3113}

3115kmp_r_sched_t __kmp_get_schedule_global() {
// This routine created because pairs (__kmp_sched, __kmp_chunk) and
// (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults
// independently. So one can get the updated schedule here.

kmp_r_sched_t r_sched;

// create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
// __kmp_guided. __kmp_sched should keep original value, so that user can set
// KMP_SCHEDULE multiple times, and thus have different run-time schedules in
// different roots (even in OMP 2.5)
enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched)(enum sched_type)( (__kmp_sched) & ~(kmp_sch_modifier_nonmonotonic
 | kmp_sch_modifier_monotonic));
enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched)((enum sched_type)( (__kmp_sched) & (kmp_sch_modifier_nonmonotonic
 | kmp_sch_modifier_monotonic)));
if (s == kmp_sch_static) {
  // replace STATIC with more detailed schedule (balanced or greedy)
  r_sched.r_sched_type = __kmp_static;
} else if (s == kmp_sch_guided_chunked) {
  // replace GUIDED with more detailed schedule (iterative or analytical)
  r_sched.r_sched_type = __kmp_guided;
} else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
  r_sched.r_sched_type = __kmp_sched;
}
SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers)(r_sched.r_sched_type = (enum sched_type)((kmp_int32)r_sched.
r_sched_type | (kmp_int32)sched_modifiers));

if (__kmp_chunk < KMP_DEFAULT_CHUNK1) {
  // __kmp_chunk may be wrong here (if it was not ever set)
  r_sched.chunk = KMP_DEFAULT_CHUNK1;
} else {
  r_sched.chunk = __kmp_chunk;
}

return r_sched;
3147}

3149/* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
 at least argc number of *t_argv entries for the requested team. */
3151static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) {

KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 3153); };
if (!realloc || argc > team->t.t_max_argc) {

  KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
 "current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
                 "current entries=%d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
 "current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); }
                 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
 "current entries=%d\n", team->t.t_id, argc, (realloc) ? team
->t.t_max_argc : 0); };
  /* if previously allocated heap space for args, free them */
  if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
    __kmp_free((void *)team->t.t_argv)___kmp_free(((void *)team->t.t_argv), "openmp/runtime/src/kmp_runtime.cpp"
, 3161);

  if (argc <= KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
 sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
 64)) / (sizeof(void *)))) {
    /* use unused space in the cache line for arguments */
    team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
 sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
 64)) / (sizeof(void *)));
    KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
 "argv entries\n", team->t.t_id, team->t.t_max_argc); }
                   "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
 "argv entries\n", team->t.t_id, team->t.t_max_argc); }
                   team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
 "argv entries\n", team->t.t_id, team->t.t_max_argc); };
    team->t.t_argv = &team->t.t_inline_argv[0];
    if (__kmp_storage_map) {
      __kmp_print_storage_map_gtid(
          -1, &team->t.t_inline_argv[0],
          &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
 sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
 64)) / (sizeof(void *)))],
          (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES(int)((4 * 64 - ((3 * (sizeof(void *)) + 2 * sizeof(int) + 2 *
 sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32)) %
 64)) / (sizeof(void *)))), "team_%d.t_inline_argv",
          team->t.t_id);
    }
  } else {
    /* allocate space for arguments in the heap */
    team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES100 >> 1))
                             ? KMP_MIN_MALLOC_ARGV_ENTRIES100
                             : 2 * argc;
    KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
 "argv entries\n", team->t.t_id, team->t.t_max_argc); }
                   "argv entries\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
 "argv entries\n", team->t.t_id, team->t.t_max_argc); }
                   team->t.t_id, team->t.t_max_argc))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
 "argv entries\n", team->t.t_id, team->t.t_max_argc); };
    team->t.t_argv =
        (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc)___kmp_page_allocate((sizeof(void *) * team->t.t_max_argc)
, "openmp/runtime/src/kmp_runtime.cpp", 3186);
    if (__kmp_storage_map) {
      __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
                                   &team->t.t_argv[team->t.t_max_argc],
                                   sizeof(void *) * team->t.t_max_argc,
                                   "team_%d.t_argv", team->t.t_id);
    }
  }
}
3195}

3197static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) {
int i;
int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
team->t.t_threads =
    (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth)___kmp_allocate((sizeof(kmp_info_t *) * max_nth), "openmp/runtime/src/kmp_runtime.cpp"
, 3201);
team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "openmp/runtime/src/kmp_runtime.cpp", 3203)
    sizeof(dispatch_shared_info_t) * num_disp_buff)___kmp_allocate((sizeof(dispatch_shared_info_t) * num_disp_buff
), "openmp/runtime/src/kmp_runtime.cpp", 3203);
team->t.t_dispatch =
    (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth)___kmp_allocate((sizeof(kmp_disp_t) * max_nth), "openmp/runtime/src/kmp_runtime.cpp"
, 3205);
team->t.t_implicit_task_taskdata =
    (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth)___kmp_allocate((sizeof(kmp_taskdata_t) * max_nth), "openmp/runtime/src/kmp_runtime.cpp"
, 3207);
team->t.t_max_nproc = max_nth;

/* setup dispatch buffers */
for (i = 0; i < num_disp_buff; ++i) {
  team->t.t_disp_buffer[i].buffer_index = i;
  team->t.t_disp_buffer[i].doacross_buf_idx = i;
}
3215}

3217static void __kmp_free_team_arrays(kmp_team_t *team) {
/* Note: this does not free the threads in t_threads (__kmp_free_threads) */
int i;
for (i = 0; i < team->t.t_max_nproc; ++i) {
  if (team->t.t_dispatch[i].th_disp_buffer != NULL__null) {
    __kmp_free(team->t.t_dispatch[i].th_disp_buffer)___kmp_free((team->t.t_dispatch[i].th_disp_buffer), "openmp/runtime/src/kmp_runtime.cpp"
, 3222);
    team->t.t_dispatch[i].th_disp_buffer = NULL__null;
  }
}
3226#if KMP_USE_HIER_SCHED0
__kmp_dispatch_free_hierarchies(team);
3228#endif
__kmp_free(team->t.t_threads)___kmp_free((team->t.t_threads), "openmp/runtime/src/kmp_runtime.cpp"
, 3229);
__kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "openmp/runtime/src/kmp_runtime.cpp"
, 3230);
__kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "openmp/runtime/src/kmp_runtime.cpp"
, 3231);
__kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "openmp/runtime/src/kmp_runtime.cpp"
, 3232);
team->t.t_threads = NULL__null;
team->t.t_disp_buffer = NULL__null;
team->t.t_dispatch = NULL__null;
team->t.t_implicit_task_taskdata = 0;
3237}

3239static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
kmp_info_t **oldThreads = team->t.t_threads;

__kmp_free(team->t.t_disp_buffer)___kmp_free((team->t.t_disp_buffer), "openmp/runtime/src/kmp_runtime.cpp"
, 3242);
__kmp_free(team->t.t_dispatch)___kmp_free((team->t.t_dispatch), "openmp/runtime/src/kmp_runtime.cpp"
, 3243);
__kmp_free(team->t.t_implicit_task_taskdata)___kmp_free((team->t.t_implicit_task_taskdata), "openmp/runtime/src/kmp_runtime.cpp"
, 3244);
__kmp_allocate_team_arrays(team, max_nth);

KMP_MEMCPYmemcpy(team->t.t_threads, oldThreads,
           team->t.t_nproc * sizeof(kmp_info_t *));

__kmp_free(oldThreads)___kmp_free((oldThreads), "openmp/runtime/src/kmp_runtime.cpp"
, 3250);
3251}

3253static kmp_internal_control_t __kmp_get_global_icvs(void) {

kmp_r_sched_t r_sched =
    __kmp_get_schedule_global(); // get current state of scheduling globals

KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0)if (!(__kmp_nested_proc_bind.used > 0)) { __kmp_debug_assert
("__kmp_nested_proc_bind.used > 0", "openmp/runtime/src/kmp_runtime.cpp"
, 3258); };

kmp_internal_control_t g_icvs = {
  0, // int serial_nesting_level; //corresponds to value of th_team_serialized
  (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
  // adjustment of threads (per thread)
  (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
  // whether blocktime is explicitly set
  __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
3267#if KMP_USE_MONITOR
  __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
3269// intervals
3270#endif
  __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for
  // next parallel region (per thread)
  // (use a max ub on value if __kmp_parallel_initialize not called yet)
  __kmp_cg_max_nth, // int thread_limit;
  __kmp_dflt_max_active_levels, // int max_active_levels; //internal control
  // for max_active_levels
  r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule
  // {sched,chunk} pair
  __kmp_nested_proc_bind.bind_types[0],
  __kmp_default_device,
  NULL__null // struct kmp_internal_control *next;
};

return g_icvs;
3285}

3287static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) {

kmp_internal_control_t gx_icvs;
gx_icvs.serial_nesting_level =
    0; // probably =team->t.t_serial like in save_inter_controls
copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
gx_icvs.next = NULL__null;

return gx_icvs;
3296}

3298static void __kmp_initialize_root(kmp_root_t *root) {
int f;
kmp_team_t *root_team;
kmp_team_t *hot_team;
int hot_team_max_nth;
kmp_r_sched_t r_sched =
    __kmp_get_schedule_global(); // get current state of scheduling globals
kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
KMP_DEBUG_ASSERT(root)if (!(root)) { __kmp_debug_assert("root", "openmp/runtime/src/kmp_runtime.cpp"
, 3306); };
KMP_ASSERT(!root->r.r_begin)if (!(!root->r.r_begin)) { __kmp_debug_assert("!root->r.r_begin"
, "openmp/runtime/src/kmp_runtime.cpp", 3307); };

/* setup the root state structure */
__kmp_init_lock(&root->r.r_begin_lock);
root->r.r_begin = FALSE0;
root->r.r_active = FALSE0;
root->r.r_in_parallel = 0;
root->r.r_blocktime = __kmp_dflt_blocktime;
3315#if KMP_AFFINITY_SUPPORTED1
root->r.r_affinity_assigned = FALSE0;
3317#endif

/* setup the root team for this task */
/* allocate the root team structure */
KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before root_team\n"
); };

root_team =
    __kmp_allocate_team(root,
                        1, // new_nproc
                        1, // max_nproc
3327#if OMPT_SUPPORT1
                        ompt_data_none{0}, // root parallel id
3329#endif
                        __kmp_nested_proc_bind.bind_types[0], &r_icvs,
                        0 // argc
                        USE_NESTED_HOT_ARG(NULL), __null // primary thread is unknown
    );
3334#if USE_DEBUGGER0
// Non-NULL value should be assigned to make the debugger display the root
// team.
TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0))((root_team->t.t_pkfn)) = (((microtask_t)(~0)));
3338#endif

KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after root_team = %p\n"
, root_team); };

root->r.r_root_team = root_team;
root_team->t.t_control_stack_top = NULL__null;

/* initialize root team */
root_team->t.t_threads[0] = NULL__null;
root_team->t.t_nproc = 1;
root_team->t.t_serialized = 1;
// TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
root_team->t.t_sched.sched = r_sched.sched;
KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
    20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
    ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); }
     root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n"
, root_team->t.t_id, 0, 0); };

/* setup the  hot team for this task */
/* allocate the hot team structure */
KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: before hot_team\n"
); };

hot_team =
    __kmp_allocate_team(root,
                        1, // new_nproc
                        __kmp_dflt_team_nth_ub * 2, // max_nproc
3364#if OMPT_SUPPORT1
                        ompt_data_none{0}, // root parallel id
3366#endif
                        __kmp_nested_proc_bind.bind_types[0], &r_icvs,
                        0 // argc
                        USE_NESTED_HOT_ARG(NULL), __null // primary thread is unknown
    );
KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_root: after hot_team = %p\n"
, hot_team); };

root->r.r_hot_team = hot_team;
root_team->t.t_control_stack_top = NULL__null;

/* first-time initialization */
hot_team->t.t_parent = root_team;

/* initialize hot team */
hot_team_max_nth = hot_team->t.t_max_nproc;
for (f = 0; f < hot_team_max_nth; ++f) {
  hot_team->t.t_threads[f] = NULL__null;
}
hot_team->t.t_nproc = 1;
// TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
hot_team->t.t_sched.sched = r_sched.sched;
hot_team->t.t_size_changed = 0;
3388}

3390#ifdef KMP_DEBUG1

3392typedef struct kmp_team_list_item {
kmp_team_p const *entry;
struct kmp_team_list_item *next;
3395} kmp_team_list_item_t;
3396typedef kmp_team_list_item_t *kmp_team_list_t;

3398static void __kmp_print_structure_team_accum( // Add team to list of teams.
  kmp_team_list_t list, // List of teams.
  kmp_team_p const *team // Team to add.
3401) {

// List must terminate with item where both entry and next are NULL.
// Team is added to the list only once.
// List is sorted in ascending order by team id.
// Team id is *not* a key.

kmp_team_list_t l;

KMP_DEBUG_ASSERT(list != NULL)if (!(list != __null)) { __kmp_debug_assert("list != __null",
 "openmp/runtime/src/kmp_runtime.cpp", 3410); };
if (team == NULL__null) {
  return;
}

__kmp_print_structure_team_accum(list, team->t.t_parent);
__kmp_print_structure_team_accum(list, team->t.t_next_pool);

// Search list for the team.
l = list;
while (l->next != NULL__null && l->entry != team) {
  l = l->next;
}
if (l->next != NULL__null) {
  return; // Team has been added before, exit.
}

// Team is not found. Search list again for insertion point.
l = list;
while (l->next != NULL__null && l->entry->t.t_id <= team->t.t_id) {
  l = l->next;
}

// Insert team.
{
  kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(malloc(sizeof(kmp_team_list_item_t))
      sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
  *item = *l;
  l->entry = team;
  l->next = item;
}
3441}

3443static void __kmp_print_structure_team(char const *title, kmp_team_p const *team

3445) {
__kmp_printf("%s", title);
if (team != NULL__null) {
  __kmp_printf("%2x %p\n", team->t.t_id, team);
} else {
  __kmp_printf(" - (nil)\n");
}
3452}

3454static void __kmp_print_structure_thread(char const *title,
                                       kmp_info_p const *thread) {
__kmp_printf("%s", title);
if (thread != NULL__null) {
  __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
} else {
  __kmp_printf(" - (nil)\n");
}
3462}

3464void __kmp_print_structure(void) {

kmp_team_list_t list;

// Initialize list of teams.
list =
    (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t))malloc(sizeof(kmp_team_list_item_t));
list->entry = NULL__null;
list->next = NULL__null;

__kmp_printf("\n------------------------------\nGlobal Thread "
             "Table\n------------------------------\n");
{
  int gtid;
  for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
    __kmp_printf("%2d", gtid);
    if (__kmp_threads != NULL__null) {
      __kmp_printf(" %p", __kmp_threads[gtid]);
    }
    if (__kmp_root != NULL__null) {
      __kmp_printf(" %p", __kmp_root[gtid]);
    }
    __kmp_printf("\n");
  }
}

// Print out __kmp_threads array.
__kmp_printf("\n------------------------------\nThreads\n--------------------"
             "----------\n");
if (__kmp_threads != NULL__null) {
  int gtid;
  for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
    kmp_info_t const *thread = __kmp_threads[gtid];
    if (thread != NULL__null) {
      __kmp_printf("GTID %2d %p:\n", gtid, thread);
      __kmp_printf("    Our Root:        %p\n", thread->th.th_root);
      __kmp_print_structure_team("    Our Team:     ", thread->th.th_team);
      __kmp_print_structure_team("    Serial Team:  ",
                                 thread->th.th_serial_team);
      __kmp_printf("    Threads:      %2d\n", thread->th.th_team_nproc);
      __kmp_print_structure_thread("    Primary:      ",
                                   thread->th.th_team_master);
      __kmp_printf("    Serialized?:  %2d\n", thread->th.th_team_serialized);
      __kmp_printf("    Set NProc:    %2d\n", thread->th.th_set_nproc);
      __kmp_printf("    Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
      __kmp_print_structure_thread("    Next in pool: ",
                                   thread->th.th_next_pool);
      __kmp_printf("\n");
      __kmp_print_structure_team_accum(list, thread->th.th_team);
      __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
    }
  }
} else {
  __kmp_printf("Threads array is not allocated.\n");
}

// Print out __kmp_root array.
__kmp_printf("\n------------------------------\nUbers\n----------------------"
             "--------\n");
if (__kmp_root != NULL__null) {
  int gtid;
  for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
    kmp_root_t const *root = __kmp_root[gtid];
    if (root != NULL__null) {
      __kmp_printf("GTID %2d %p:\n", gtid, root);
      __kmp_print_structure_team("    Root Team:    ", root->r.r_root_team);
      __kmp_print_structure_team("    Hot Team:     ", root->r.r_hot_team);
      __kmp_print_structure_thread("    Uber Thread:  ",
                                   root->r.r_uber_thread);
      __kmp_printf("    Active?:      %2d\n", root->r.r_active);
      __kmp_printf("    In Parallel:  %2d\n",
                   KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)(&root->r.r_in_parallel)->load(std::memory_order_relaxed
));
      __kmp_printf("\n");
      __kmp_print_structure_team_accum(list, root->r.r_root_team);
      __kmp_print_structure_team_accum(list, root->r.r_hot_team);
    }
  }
} else {
  __kmp_printf("Ubers array is not allocated.\n");
}

__kmp_printf("\n------------------------------\nTeams\n----------------------"
             "--------\n");
while (list->next != NULL__null) {
  kmp_team_p const *team = list->entry;
  int i;
  __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
  __kmp_print_structure_team("    Parent Team:      ", team->t.t_parent);
  __kmp_printf("    Primary TID:      %2d\n", team->t.t_master_tid);
  __kmp_printf("    Max threads:      %2d\n", team->t.t_max_nproc);
  __kmp_printf("    Levels of serial: %2d\n", team->t.t_serialized);
  __kmp_printf("    Number threads:   %2d\n", team->t.t_nproc);
  for (i = 0; i < team->t.t_nproc; ++i) {
    __kmp_printf("    Thread %2d:      ", i);
    __kmp_print_structure_thread("", team->t.t_threads[i]);
  }
  __kmp_print_structure_team("    Next in pool:     ", team->t.t_next_pool);
  __kmp_printf("\n");
  list = list->next;
}

// Print out __kmp_thread_pool and __kmp_team_pool.
__kmp_printf("\n------------------------------\nPools\n----------------------"
             "--------\n");
__kmp_print_structure_thread("Thread pool:          ",
                             CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool));
__kmp_print_structure_team("Team pool:            ",
                           CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool));
__kmp_printf("\n");

// Free team list.
while (list != NULL__null) {
  kmp_team_list_item_t *item = list;
  list = list->next;
  KMP_INTERNAL_FREE(item)free(item);
}
3580}

3582#endif

3584//---------------------------------------------------------------------------
3585//  Stuff for per-thread fast random number generator
3586//  Table of primes
3587static const unsigned __kmp_primes[] = {
  0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
  0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
  0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
  0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
  0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
  0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
  0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
  0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
  0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
  0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
  0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};

3600//---------------------------------------------------------------------------
3601//  __kmp_get_random: Get a random number using a linear congruential method.
3602unsigned short __kmp_get_random(kmp_info_t *thread) {
unsigned x = thread->th.th_x;
unsigned short r = (unsigned short)(x >> 16);

thread->th.th_x = x * thread->th.th_a + 1;

KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); }
              thread->th.th_info.ds.ds_tid, r))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_get_random: THREAD: %d, RETURN: %u\n"
, thread->th.th_info.ds.ds_tid, r); };

return r;
3612}
3613//--------------------------------------------------------
3614// __kmp_init_random: Initialize a random number generator
3615void __kmp_init_random(kmp_info_t *thread) {
unsigned seed = thread->th.th_info.ds.ds_tid;

thread->th.th_a =
    __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))];
thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
KA_TRACE(30,if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); }
         ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_init_random: THREAD: %u; A: %u\n"
, seed, thread->th.th_a); };
3623}

3625#if KMP_OS_WINDOWS0
3626/* reclaim array entries for root threads that are already dead, returns number
* reclaimed */
3628static int __kmp_reclaim_dead_roots(void) {
int i, r = 0;

for (i = 0; i < __kmp_threads_capacity; ++i) {
  if (KMP_UBER_GTID(i) &&
      !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])((void *)(__kmp_threads[i]))) &&
      !__kmp_root[i]
           ->r.r_active) { // AC: reclaim only roots died in non-active state
    r += __kmp_unregister_root_other_thread(i);
  }
}
return r;
3640}
3641#endif

3643/* This function attempts to create free entries in __kmp_threads and
 __kmp_root, and returns the number of free entries generated.

 For Windows* OS static library, the first mechanism used is to reclaim array
 entries for root threads that are already dead.

 On all platforms, expansion is attempted on the arrays __kmp_threads_ and
 __kmp_root, with appropriate update to __kmp_threads_capacity. Array
 capacity is increased by doubling with clipping to __kmp_tp_capacity, if
 threadprivate cache array has been created. Synchronization with
 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.

 After any dead root reclamation, if the clipping value allows array expansion
 to result in the generation of a total of nNeed free slots, the function does
 that expansion. If not, nothing is done beyond the possible initial root
 thread reclamation.

 If any argument is negative, the behavior is undefined. */
3661static int __kmp_expand_threads(int nNeed) {
int added = 0;
int minimumRequiredCapacity;
int newCapacity;
kmp_info_t **newThreads;
kmp_root_t **newRoot;

// All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so
// resizing __kmp_threads does not need additional protection if foreign
// threads are present

3672#if KMP_OS_WINDOWS0 && !KMP_DYNAMIC_LIB1
/* only for Windows static library */
/* reclaim array entries for root threads that are already dead */
added = __kmp_reclaim_dead_roots();

if (nNeed) {
  nNeed -= added;
  if (nNeed < 0)
    nNeed = 0;
}
3682#endif
if (nNeed <= 0)
  return added;

// Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
// __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
// user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
// > __kmp_max_nth in one of two ways:
//
// 1) The initialization thread (gtid = 0) exits.  __kmp_threads[0]
//    may not be reused by another thread, so we may need to increase
//    __kmp_threads_capacity to __kmp_max_nth + 1.
//
// 2) New foreign root(s) are encountered.  We always register new foreign
//    roots. This may cause a smaller # of threads to be allocated at
//    subsequent parallel regions, but the worker threads hang around (and
//    eventually go to sleep) and need slots in the __kmp_threads[] array.
//
// Anyway, that is the reason for moving the check to see if
// __kmp_max_nth was exceeded into __kmp_reserve_threads()
// instead of having it performed here. -BB

KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity)if (!(__kmp_sys_max_nth >= __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_sys_max_nth >= __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 3704); };

/* compute expansion headroom to check if we can expand */
if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
  /* possible expansion too small -- give up */
  return added;
}
minimumRequiredCapacity = __kmp_threads_capacity + nNeed;

newCapacity = __kmp_threads_capacity;
do {
  newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
                                                        : __kmp_sys_max_nth;
} while (newCapacity < minimumRequiredCapacity);
newThreads = (kmp_info_t **)__kmp_allocate(___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "openmp/runtime/src/kmp_runtime.cpp", 3719
)
    (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE)___kmp_allocate(((sizeof(kmp_info_t *) + sizeof(kmp_root_t *)
) * newCapacity + 64), "openmp/runtime/src/kmp_runtime.cpp", 3719
);
newRoot =
    (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity);
KMP_MEMCPYmemcpy(newThreads, __kmp_threads,
           __kmp_threads_capacity * sizeof(kmp_info_t *));
KMP_MEMCPYmemcpy(newRoot, __kmp_root,
           __kmp_threads_capacity * sizeof(kmp_root_t *));
// Put old __kmp_threads array on a list. Any ongoing references to the old
// list will be valid. This list is cleaned up at library shutdown.
kmp_old_threads_list_t *node =
    (kmp_old_threads_list_t *)__kmp_allocate(sizeof(kmp_old_threads_list_t))___kmp_allocate((sizeof(kmp_old_threads_list_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3729);
node->threads = __kmp_threads;
node->next = __kmp_old_threads_list;
__kmp_old_threads_list = node;

*(kmp_info_t * *volatile *)&__kmp_threads = newThreads;
*(kmp_root_t * *volatile *)&__kmp_root = newRoot;
added += newCapacity - __kmp_threads_capacity;
*(volatile int *)&__kmp_threads_capacity = newCapacity;

if (newCapacity > __kmp_tp_capacity) {
  __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
  if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
    __kmp_threadprivate_resize_cache(newCapacity);
  } else { // increase __kmp_tp_capacity to correspond with kmp_threads size
    *(volatile int *)&__kmp_tp_capacity = newCapacity;
  }
  __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
}

return added;
3750}

3752/* Register the current thread as a root thread and obtain our gtid. We must
 have the __kmp_initz_lock held at this point. Argument TRUE only if are the
 thread that calls from __kmp_do_serial_initialize() */
3755int __kmp_register_root(int initial_thread) {
kmp_info_t *root_thread;
kmp_root_t *root;
int gtid;
int capacity;
__kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
KA_TRACE(20, ("__kmp_register_root: entered\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: entered\n"
); };
KMP_MB();

/* 2007-03-02:
   If initial thread did not invoke OpenMP RTL yet, and this thread is not an
   initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not
   work as expected -- it may return false (that means there is at least one
   empty slot in __kmp_threads array), but it is possible the only free slot
   is #0, which is reserved for initial thread and so cannot be used for this
   one. Following code workarounds this bug.

   However, right solution seems to be not reserving slot #0 for initial
   thread because:
   (1) there is no magic in slot #0,
   (2) we cannot detect initial thread reliably (the first thread which does
      serial initialization may be not a real initial thread).
*/
capacity = __kmp_threads_capacity;
if (!initial_thread && TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
  --capacity;
}

// If it is not for initializing the hidden helper team, we need to take
// __kmp_hidden_helper_threads_num out of the capacity because it is included
// in __kmp_threads_capacity.
if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
  capacity -= __kmp_hidden_helper_threads_num;
}

/* see if there are too many threads */
if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
  if (__kmp_tp_cached) {
    __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread),
                KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity)__kmp_msg_format(kmp_i18n_hnt_Set_ALL_THREADPRIVATE, __kmp_tp_capacity
),
                KMP_HNT(PossibleSystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_PossibleSystemLimitOnThreads), __kmp_msg_null);
  } else {
    __kmp_fatal(KMP_MSG(CantRegisterNewThread)__kmp_msg_format(kmp_i18n_msg_CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads)__kmp_msg_format(kmp_i18n_hnt_SystemLimitOnThreads),
                __kmp_msg_null);
  }
}

// When hidden helper task is enabled, __kmp_threads is organized as follows:
// 0: initial thread, also a regular OpenMP thread.
// [1, __kmp_hidden_helper_threads_num]: slots for hidden helper threads.
// [__kmp_hidden_helper_threads_num + 1, __kmp_threads_capacity): slots for
// regular OpenMP threads.
if (TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
  // Find an available thread slot for hidden helper thread. Slots for hidden
  // helper threads start from 1 to __kmp_hidden_helper_threads_num.
  for (gtid = 1; TCR_PTR(__kmp_threads[gtid])((void *)(__kmp_threads[gtid])) != NULL__null &&
                 gtid <= __kmp_hidden_helper_threads_num;
       gtid++)
    ;
  KMP_ASSERT(gtid <= __kmp_hidden_helper_threads_num)if (!(gtid <= __kmp_hidden_helper_threads_num)) { __kmp_debug_assert
("gtid <= __kmp_hidden_helper_threads_num", "openmp/runtime/src/kmp_runtime.cpp"
, 3814); };
  KA_TRACE(1, ("__kmp_register_root: found slot in threads array for "if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array for "
 "hidden helper thread: T#%d\n", gtid); }
               "hidden helper thread: T#%d\n",if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array for "
 "hidden helper thread: T#%d\n", gtid); }
               gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array for "
 "hidden helper thread: T#%d\n", gtid); };
} else {
  /* find an available thread slot */
  // Don't reassign the zero slot since we need that to only be used by
  // initial thread. Slots for hidden helper threads should also be skipped.
  if (initial_thread && TCR_PTR(__kmp_threads[0])((void *)(__kmp_threads[0])) == NULL__null) {
    gtid = 0;
  } else {
    for (gtid = __kmp_hidden_helper_threads_num + 1;
         TCR_PTR(__kmp_threads[gtid])((void *)(__kmp_threads[gtid])) != NULL__null; gtid++)
      ;
  }
  KA_TRACE(if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); }
      1, ("__kmp_register_root: found slot in threads array: T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_register_root: found slot in threads array: T#%d\n"
, gtid); };
  KMP_ASSERT(gtid < __kmp_threads_capacity)if (!(gtid < __kmp_threads_capacity)) { __kmp_debug_assert
("gtid < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 3831); };
}

/* update global accounting */
__kmp_all_nth++;
TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);

// if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
// numbers of procs, and method #2 (keyed API call) for higher numbers.
if (__kmp_adjust_gtid_mode) {
  if (__kmp_all_nth >= __kmp_tls_gtid_min) {
    if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
      TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
    }
  } else {
    if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
      TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
    }
  }
}

3852#ifdef KMP_ADJUST_BLOCKTIME1
/* Adjust blocktime to zero if necessary            */
/* Middle initialization might not have occurred yet */
if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
  if (__kmp_nth > __kmp_avail_proc) {
    __kmp_zero_bt = TRUE(!0);
  }
}
3860#endif /* KMP_ADJUST_BLOCKTIME */

/* setup this new hierarchy */
if (!(root = __kmp_root[gtid])) {
  root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t))___kmp_allocate((sizeof(kmp_root_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3864);
  KMP_DEBUG_ASSERT(!root->r.r_root_team)if (!(!root->r.r_root_team)) { __kmp_debug_assert("!root->r.r_root_team"
, "openmp/runtime/src/kmp_runtime.cpp", 3865); };
}

3868#if KMP_STATS_ENABLED0
// Initialize stats as soon as possible (right after gtid assignment).
__kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
__kmp_stats_thread_ptr->startLife();
KMP_SET_THREAD_STATE(SERIAL_REGION)((void)0);
KMP_INIT_PARTITIONED_TIMERS(OMP_serial)((void)0);
3874#endif
__kmp_initialize_root(root);

/* setup new root thread structure */
if (root->r.r_uber_thread) {
  root_thread = root->r.r_uber_thread;
} else {
  root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3881);
  if (__kmp_storage_map) {
    __kmp_print_thread_storage_map(root_thread, gtid);
  }
  root_thread->th.th_info.ds.ds_gtid = gtid;
3886#if OMPT_SUPPORT1
  root_thread->th.ompt_thread_info.thread_data = ompt_data_none{0};
3888#endif
  root_thread->th.th_root = root;
  if (__kmp_env_consistency_check) {
    root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
  }
3893#if USE_FAST_MEMORY3
  __kmp_initialize_fast_memory(root_thread);
3895#endif /* USE_FAST_MEMORY */

3897#if KMP_USE_BGET1
  KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL)if (!(root_thread->th.th_local.bget_data == __null)) { __kmp_debug_assert
("root_thread->th.th_local.bget_data == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 3898); };
  __kmp_initialize_bget(root_thread);
3900#endif
  __kmp_init_random(root_thread); // Initialize random number generator
}

/* setup the serial team held in reserve by the root thread */
if (!root_thread->th.th_serial_team) {
  kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
  KF_TRACE(10, ("__kmp_register_root: before serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: before serial_team\n"
); };
  root_thread->th.th_serial_team = __kmp_allocate_team(
      root, 1, 1,
3910#if OMPT_SUPPORT1
      ompt_data_none{0}, // root parallel id
3912#endif
      proc_bind_default, &r_icvs, 0 USE_NESTED_HOT_ARG(NULL), __null);
}
KMP_ASSERT(root_thread->th.th_serial_team)if (!(root_thread->th.th_serial_team)) { __kmp_debug_assert
("root_thread->th.th_serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 3915); };
KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); }
              root_thread->th.th_serial_team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_register_root: after serial_team = %p\n"
, root_thread->th.th_serial_team); };

/* drop root_thread into place */
TCW_SYNC_PTR(__kmp_threads[gtid], root_thread)((__kmp_threads[gtid])) = ((root_thread));

root->r.r_root_team->t.t_threads[0] = root_thread;
root->r.r_hot_team->t.t_threads[0] = root_thread;
root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
// AC: the team created in reserve, not for execution (it is unused for now).
root_thread->th.th_serial_team->t.t_serialized = 0;
root->r.r_uber_thread = root_thread;

/* initialize the thread, get it ready to go */
__kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
TCW_4(__kmp_init_gtid, TRUE)(__kmp_init_gtid) = ((!0));

/* prepare the primary thread for get_gtid() */
__kmp_gtid_set_specific(gtid);

3936#if USE_ITT_BUILD1
__kmp_itt_thread_name(gtid);
3938#endif /* USE_ITT_BUILD */

3940#ifdef KMP_TDATA_GTID1
__kmp_gtid = gtid;
3942#endif
__kmp_create_worker(gtid, root_thread, __kmp_stksize);
KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid)if (!(__kmp_gtid_get_specific() == gtid)) { __kmp_debug_assert
("__kmp_gtid_get_specific() == gtid", "openmp/runtime/src/kmp_runtime.cpp"
, 3944); };

KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
 "plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
              "plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
 "plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
              gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
 "plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
              root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
 "plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); }
              KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
 "plain=%u\n", gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team
), root->r.r_hot_team->t.t_id, 0, 0, 0); };
{ // Initialize barrier data.
  int b;
  for (b = 0; b < bs_last_barrier; ++b) {
    root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE0;
3955#if USE_DEBUGGER0
    root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3957#endif
  }
}
KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==if (!(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].
b_arrived == 0)) { __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 3961); }
                 KMP_INIT_BARRIER_STATE)if (!(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].
b_arrived == 0)) { __kmp_debug_assert("root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 3961); };

3963#if KMP_AFFINITY_SUPPORTED1
root_thread->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
root_thread->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
root_thread->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
root_thread->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
3968#endif /* KMP_AFFINITY_SUPPORTED */
root_thread->th.th_def_allocator = __kmp_def_allocator;
root_thread->th.th_prev_level = 0;
root_thread->th.th_prev_num_threads = 1;

kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t))___kmp_allocate((sizeof(kmp_cg_root_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 3973);
tmp->cg_root = root_thread;
tmp->cg_thread_limit = __kmp_cg_max_nth;
tmp->cg_nthreads = 1;
KA_TRACE(100, ("__kmp_register_root: Thread %p created node %p with"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_register_root: Thread %p created node %p with"
 " cg_nthreads init to 1\n", root_thread, tmp); }
               " cg_nthreads init to 1\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_register_root: Thread %p created node %p with"
 " cg_nthreads init to 1\n", root_thread, tmp); }
               root_thread, tmp))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_register_root: Thread %p created node %p with"
 " cg_nthreads init to 1\n", root_thread, tmp); };
tmp->up = NULL__null;
root_thread->th.th_cg_roots = tmp;

__kmp_root_counter++;

3985#if OMPT_SUPPORT1
if (!initial_thread && ompt_enabled.enabled) {

  kmp_info_t *root_thread = ompt_get_thread();

  ompt_set_thread_state(root_thread, ompt_state_overhead);

  if (ompt_enabled.ompt_callback_thread_begin) {
    ompt_callbacks.ompt_callback(ompt_callback_thread_begin)ompt_callback_thread_begin_callback(
        ompt_thread_initial, __ompt_get_thread_data_internal());
  }
  ompt_data_t *task_data;
  ompt_data_t *parallel_data;
  __ompt_get_task_info_internal(0, NULL__null, &task_data, NULL__null, &parallel_data,
                                NULL__null);
  if (ompt_enabled.ompt_callback_implicit_task) {
    ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
        ompt_scope_begin, parallel_data, task_data, 1, 1, ompt_task_initial);
  }

  ompt_set_thread_state(root_thread, ompt_state_work_serial);
}
4007#endif
4008#if OMPD_SUPPORT1
if (ompd_state & OMPD_ENABLE_BP0x1)
  ompd_bp_thread_begin();
4011#endif

KMP_MB();
__kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);

return gtid;
4017}

4019#if KMP_NESTED_HOT_TEAMS1
4020static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
                              const int max_level) {
int i, n, nth;
kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
if (!hot_teams || !hot_teams[level].hot_team) {
  return 0;
}
KMP_DEBUG_ASSERT(level < max_level)if (!(level < max_level)) { __kmp_debug_assert("level < max_level"
, "openmp/runtime/src/kmp_runtime.cpp", 4027); };
kmp_team_t *team = hot_teams[level].hot_team;
nth = hot_teams[level].hot_team_nth;
n = nth - 1; // primary thread is not freed
if (level < max_level - 1) {
  for (i = 0; i < nth; ++i) {
    kmp_info_t *th = team->t.t_threads[i];
    n += __kmp_free_hot_teams(root, th, level + 1, max_level);
    if (i > 0 && th->th.th_hot_teams) {
      __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "openmp/runtime/src/kmp_runtime.cpp"
, 4036);
      th->th.th_hot_teams = NULL__null;
    }
  }
}
__kmp_free_team(root, team, NULL__null);
return n;
4043}
4044#endif

4046// Resets a root thread and clear its root and hot teams.
4047// Returns the number of __kmp_threads entries directly and indirectly freed.
4048static int __kmp_reset_root(int gtid, kmp_root_t *root) {
kmp_team_t *root_team = root->r.r_root_team;
kmp_team_t *hot_team = root->r.r_hot_team;
int n = hot_team->t.t_nproc;
int i;

KMP_DEBUG_ASSERT(!root->r.r_active)if (!(!root->r.r_active)) { __kmp_debug_assert("!root->r.r_active"
, "openmp/runtime/src/kmp_runtime.cpp", 4054); };

root->r.r_root_team = NULL__null;
root->r.r_hot_team = NULL__null;
// __kmp_free_team() does not free hot teams, so we have to clear r_hot_team
// before call to __kmp_free_team().
__kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL), __null);
4061#if KMP_NESTED_HOT_TEAMS1
if (__kmp_hot_teams_max_level >
    0) { // need to free nested hot teams and their threads if any
  for (i = 0; i < hot_team->t.t_nproc; ++i) {
    kmp_info_t *th = hot_team->t.t_threads[i];
    if (__kmp_hot_teams_max_level > 1) {
      n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
    }
    if (th->th.th_hot_teams) {
      __kmp_free(th->th.th_hot_teams)___kmp_free((th->th.th_hot_teams), "openmp/runtime/src/kmp_runtime.cpp"
, 4070);
      th->th.th_hot_teams = NULL__null;
    }
  }
}
4075#endif
__kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL), __null);

// Before we can reap the thread, we need to make certain that all other
// threads in the teams that had this root as ancestor have stopped trying to
// steal tasks.
if (__kmp_tasking_mode != tskm_immediate_exec) {
  __kmp_wait_to_unref_task_teams();
}

4085#if KMP_OS_WINDOWS0
/* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
 "lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
 root->r.r_uber_thread->th.th_info.ds.ds_thread); }
    10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPECif (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
 "lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
 root->r.r_uber_thread->th.th_info.ds.ds_thread); }
         "\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
 "lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
 root->r.r_uber_thread->th.th_info.ds.ds_thread); }
         (LPVOID) & (root->r.r_uber_thread->th),if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
 "lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
 root->r.r_uber_thread->th.th_info.ds.ds_thread); }
         root->r.r_uber_thread->th.th_info.ds.ds_thread))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_reset_root: free handle, th = %p, handle = %"
 "lu" "\n", (LPVOID) & (root->r.r_uber_thread->th),
 root->r.r_uber_thread->th.th_info.ds.ds_thread); };
__kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
4093#endif /* KMP_OS_WINDOWS */

4095#if OMPD_SUPPORT1
if (ompd_state & OMPD_ENABLE_BP0x1)
  ompd_bp_thread_end();
4098#endif

4100#if OMPT_SUPPORT1
ompt_data_t *task_data;
ompt_data_t *parallel_data;
__ompt_get_task_info_internal(0, NULL__null, &task_data, NULL__null, &parallel_data,
                              NULL__null);
if (ompt_enabled.ompt_callback_implicit_task) {
  ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
      ompt_scope_end, parallel_data, task_data, 0, 1, ompt_task_initial);
}
if (ompt_enabled.ompt_callback_thread_end) {
  ompt_callbacks.ompt_callback(ompt_callback_thread_end)ompt_callback_thread_end_callback(
      &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
}
4113#endif

TCW_4(__kmp_nth,(__kmp_nth) = (__kmp_nth - 1)
      __kmp_nth - 1)(__kmp_nth) = (__kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;
KA_TRACE(100, ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
 " to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); }
               " to %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
 " to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); }
               root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
 " to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); }
               root->r.r_uber_thread->th.th_cg_roots->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
 " to %d\n", root->r.r_uber_thread, root->r.r_uber_thread
->th.th_cg_roots, root->r.r_uber_thread->th.th_cg_roots
->cg_nthreads); };
if (i == 1) {
  // need to free contention group structure
  KMP_DEBUG_ASSERT(root->r.r_uber_thread ==if (!(root->r.r_uber_thread == root->r.r_uber_thread->
th.th_cg_roots->cg_root)) { __kmp_debug_assert("root->r.r_uber_thread == root->r.r_uber_thread->th.th_cg_roots->cg_root"
, "openmp/runtime/src/kmp_runtime.cpp", 4125); }
                   root->r.r_uber_thread->th.th_cg_roots->cg_root)if (!(root->r.r_uber_thread == root->r.r_uber_thread->
th.th_cg_roots->cg_root)) { __kmp_debug_assert("root->r.r_uber_thread == root->r.r_uber_thread->th.th_cg_roots->cg_root"
, "openmp/runtime/src/kmp_runtime.cpp", 4125); };
  KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL)if (!(root->r.r_uber_thread->th.th_cg_roots->up == __null
)) { __kmp_debug_assert("root->r.r_uber_thread->th.th_cg_roots->up == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 4126); };
  __kmp_free(root->r.r_uber_thread->th.th_cg_roots)___kmp_free((root->r.r_uber_thread->th.th_cg_roots), "openmp/runtime/src/kmp_runtime.cpp"
, 4127);
  root->r.r_uber_thread->th.th_cg_roots = NULL__null;
}
__kmp_reap_thread(root->r.r_uber_thread, 1);

// We canot put root thread to __kmp_thread_pool, so we have to reap it
// instead of freeing.
root->r.r_uber_thread = NULL__null;
/* mark root as no longer in use */
root->r.r_begin = FALSE0;

return n;
4139}

4141void __kmp_unregister_root_current_thread(int gtid) {
KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: enter T#%d\n"
, gtid); };
/* this lock should be ok, since unregister_root_current_thread is never
   called during an abort, only during a normal close. furthermore, if you
   have the forkjoin lock, you should never try to get the initz lock */
__kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
  KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, "if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
 "exiting T#%d\n", gtid); }
                "exiting T#%d\n",if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
 "exiting T#%d\n", gtid); }
                gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: already finished, "
 "exiting T#%d\n", gtid); };
  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
  return;
}
kmp_root_t *root = __kmp_root[gtid];

KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])if (!(__kmp_threads && __kmp_threads[gtid])) { __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "openmp/runtime/src/kmp_runtime.cpp"
, 4156); };
KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 4157); };
KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)if (!(root == __kmp_threads[gtid]->th.th_root)) { __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "openmp/runtime/src/kmp_runtime.cpp"
, 4158); };
KMP_ASSERT(root->r.r_active == FALSE)if (!(root->r.r_active == 0)) { __kmp_debug_assert("root->r.r_active == FALSE"
, "openmp/runtime/src/kmp_runtime.cpp", 4159); };

KMP_MB();

kmp_info_t *thread = __kmp_threads[gtid];
kmp_team_t *team = thread->th.th_team;
kmp_task_team_t *task_team = thread->th.th_task_team;

// we need to wait for the proxy tasks before finishing the thread
if (task_team != NULL__null && (task_team->tt.tt_found_proxy_tasks ||
                          task_team->tt.tt_hidden_helper_task_encountered)) {
4170#if OMPT_SUPPORT1
  // the runtime is shutting down so we won't report any events
  thread->th.ompt_thread_info.state = ompt_state_undefined;
4173#endif
  __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL), __null);
}

__kmp_reset_root(gtid, root);

KMP_MB();
KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); }
         ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_current_thread: T#%d unregistered\n"
, gtid); };

__kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4184}

4186#if KMP_OS_WINDOWS0
4187/* __kmp_forkjoin_lock must be already held
 Unregisters a root thread that is not the current thread.  Returns the number
 of __kmp_threads entries freed as a result. */
4190static int __kmp_unregister_root_other_thread(int gtid) {
kmp_root_t *root = __kmp_root[gtid];
int r;

KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid))if (kmp_a_debug >= 1) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: enter T#%d\n"
, gtid); };
KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid])if (!(__kmp_threads && __kmp_threads[gtid])) { __kmp_debug_assert
("__kmp_threads && __kmp_threads[gtid]", "openmp/runtime/src/kmp_runtime.cpp"
, 4195); };
KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 4196); };
KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root)if (!(root == __kmp_threads[gtid]->th.th_root)) { __kmp_debug_assert
("root == __kmp_threads[gtid]->th.th_root", "openmp/runtime/src/kmp_runtime.cpp"
, 4197); };
KMP_ASSERT(root->r.r_active == FALSE)if (!(root->r.r_active == 0)) { __kmp_debug_assert("root->r.r_active == FALSE"
, "openmp/runtime/src/kmp_runtime.cpp", 4198); };

r = __kmp_reset_root(gtid, root);
KC_TRACE(10,if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); }
         ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid))if (kmp_c_debug >= 10) { __kmp_debug_printf ("__kmp_unregister_root_other_thread: T#%d unregistered\n"
, gtid); };
return r;
4204}
4205#endif

4207#if KMP_DEBUG1
4208void __kmp_task_info() {

kmp_int32 gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
kmp_int32 tid = __kmp_tid_from_gtid(gtid);
kmp_info_t *this_thr = __kmp_threads[gtid];
kmp_team_t *steam = this_thr->th.th_serial_team;
kmp_team_t *team = this_thr->th.th_team;

__kmp_printf(
    "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p "
    "ptask=%p\n",
    gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
    team->t.t_implicit_task_taskdata[tid].td_parent);
4221}
4222#endif // KMP_DEBUG

4224/* TODO optimize with one big memclr, take out what isn't needed, split
 responsibility to workers as much as possible, and delay initialization of
 features as much as possible  */
4227static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
                                int tid, int gtid) {
/* this_thr->th.th_info.ds.ds_gtid is setup in
   kmp_allocate_thread/create_worker.
   this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
KMP_DEBUG_ASSERT(this_thr != NULL)if (!(this_thr != __null)) { __kmp_debug_assert("this_thr != __null"
, "openmp/runtime/src/kmp_runtime.cpp", 4232); };
KMP_DEBUG_ASSERT(this_thr->th.th_serial_team)if (!(this_thr->th.th_serial_team)) { __kmp_debug_assert("this_thr->th.th_serial_team"
, "openmp/runtime/src/kmp_runtime.cpp", 4233); };
KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 4234); };
KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 4235); };
KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 4236); };
kmp_info_t *master = team->t.t_threads[0];
KMP_DEBUG_ASSERT(master)if (!(master)) { __kmp_debug_assert("master", "openmp/runtime/src/kmp_runtime.cpp"
, 4238); };
KMP_DEBUG_ASSERT(master->th.th_root)if (!(master->th.th_root)) { __kmp_debug_assert("master->th.th_root"
, "openmp/runtime/src/kmp_runtime.cpp", 4239); };

KMP_MB();

TCW_SYNC_PTR(this_thr->th.th_team, team)((this_thr->th.th_team)) = ((team));

this_thr->th.th_info.ds.ds_tid = tid;
this_thr->th.th_set_nproc = 0;
if (__kmp_tasking_mode != tskm_immediate_exec)
  // When tasking is possible, threads are not safe to reap until they are
  // done tasking; this will be set when tasking code is exited in wait
  this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP0;
else // no tasking --> always safe to reap
  this_thr->th.th_reap_state = KMP_SAFE_TO_REAP1;
this_thr->th.th_set_proc_bind = proc_bind_default;
4254#if KMP_AFFINITY_SUPPORTED1
this_thr->th.th_new_place = this_thr->th.th_current_place;
4256#endif
this_thr->th.th_root = master->th.th_root;

/* setup the thread's cache of the team structure */
this_thr->th.th_team_nproc = team->t.t_nproc;
this_thr->th.th_team_master = master;
this_thr->th.th_team_serialized = team->t.t_serialized;

KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata)if (!(team->t.t_implicit_task_taskdata)) { __kmp_debug_assert
("team->t.t_implicit_task_taskdata", "openmp/runtime/src/kmp_runtime.cpp"
, 4264); };

KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
              tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); };

__kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
                         team, tid, TRUE(!0));

KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); }
              tid, gtid, this_thr, this_thr->th.th_current_task))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n"
, tid, gtid, this_thr, this_thr->th.th_current_task); };
// TODO: Initialize ICVs from parent; GEH - isn't that already done in
// __kmp_initialize_team()?

/* TODO no worksharing in speculative threads */
this_thr->th.th_dispatch = &team->t.t_dispatch[tid];

this_thr->th.th_local.this_construct = 0;

if (!this_thr->th.th_pri_common) {
  this_thr->th.th_pri_common =
      (struct common_table *)__kmp_allocate(sizeof(struct common_table))___kmp_allocate((sizeof(struct common_table)), "openmp/runtime/src/kmp_runtime.cpp"
, 4284);
  if (__kmp_storage_map) {
    __kmp_print_storage_map_gtid(
        gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
        sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
  }
  this_thr->th.th_pri_head = NULL__null;
}

if (this_thr != master && // Primary thread's CG root is initialized elsewhere
    this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set
  // Make new thread's CG root same as primary thread's
  KMP_DEBUG_ASSERT(master->th.th_cg_roots)if (!(master->th.th_cg_roots)) { __kmp_debug_assert("master->th.th_cg_roots"
, "openmp/runtime/src/kmp_runtime.cpp", 4296); };
  kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
  if (tmp) {
    // worker changes CG, need to check if old CG should be freed
    int i = tmp->cg_nthreads--;
    KA_TRACE(100, ("__kmp_initialize_info: Thread %p decrement cg_nthreads"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p decrement cg_nthreads"
 " on node %p of thread %p to %d\n", this_thr, tmp, tmp->cg_root
, tmp->cg_nthreads); }
                   " on node %p of thread %p to %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p decrement cg_nthreads"
 " on node %p of thread %p to %d\n", this_thr, tmp, tmp->cg_root
, tmp->cg_nthreads); }
                   this_thr, tmp, tmp->cg_root, tmp->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p decrement cg_nthreads"
 " on node %p of thread %p to %d\n", this_thr, tmp, tmp->cg_root
, tmp->cg_nthreads); };
    if (i == 1) {
      __kmp_free(tmp)___kmp_free((tmp), "openmp/runtime/src/kmp_runtime.cpp", 4305
); // last thread left CG --> free it
    }
  }
  this_thr->th.th_cg_roots = master->th.th_cg_roots;
  // Increment new thread's CG root's counter to add the new thread
  this_thr->th.th_cg_roots->cg_nthreads++;
  KA_TRACE(100, ("__kmp_initialize_info: Thread %p increment cg_nthreads on"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
 " node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
                 " node %p of thread %p to %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
 " node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
                 this_thr, this_thr->th.th_cg_roots,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
 " node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
                 this_thr->th.th_cg_roots->cg_root,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
 " node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); }
                 this_thr->th.th_cg_roots->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
 " node %p of thread %p to %d\n", this_thr, this_thr->th.th_cg_roots
, this_thr->th.th_cg_roots->cg_root, this_thr->th.th_cg_roots
->cg_nthreads); };
  this_thr->th.th_current_task->td_icvs.thread_limit =
      this_thr->th.th_cg_roots->cg_thread_limit;
}

/* Initialize dynamic dispatch */
{
  volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
  // Use team max_nproc since this will never change for the team.
  size_t disp_size =
      sizeof(dispatch_private_info_t) *
      (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
  KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); }
                team->t.t_max_nproc))if (kmp_d_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_info: T#%d max_nproc: %d\n"
, gtid, team->t.t_max_nproc); };
  KMP_ASSERT(dispatch)if (!(dispatch)) { __kmp_debug_assert("dispatch", "openmp/runtime/src/kmp_runtime.cpp"
, 4329); };
  KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 4330); };
  KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid])if (!(dispatch == &team->t.t_dispatch[tid])) { __kmp_debug_assert
("dispatch == &team->t.t_dispatch[tid]", "openmp/runtime/src/kmp_runtime.cpp"
, 4331); };

  dispatch->th_disp_index = 0;
  dispatch->th_doacross_buf_idx = 0;
  if (!dispatch->th_disp_buffer) {
    dispatch->th_disp_buffer =
        (dispatch_private_info_t *)__kmp_allocate(disp_size)___kmp_allocate((disp_size), "openmp/runtime/src/kmp_runtime.cpp"
, 4337);

    if (__kmp_storage_map) {
      __kmp_print_storage_map_gtid(
          gtid, &dispatch->th_disp_buffer[0],
          &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
                                        ? 1
                                        : __kmp_dispatch_num_buffers],
          disp_size,
          "th_%d.th_dispatch.th_disp_buffer "
          "(team_%d.t_dispatch[%d].th_disp_buffer)",
          gtid, team->t.t_id, gtid);
    }
  } else {
    memset(&dispatch->th_disp_buffer[0], '\0', disp_size);
  }

  dispatch->th_dispatch_pr_current = 0;
  dispatch->th_dispatch_sh_current = 0;

  dispatch->th_deo_fcn = 0; /* ORDERED     */
  dispatch->th_dxo_fcn = 0; /* END ORDERED */
}

this_thr->th.th_next_pool = NULL__null;

if (!this_thr->th.th_task_state_memo_stack) {
  size_t i;
  this_thr->th.th_task_state_memo_stack =
      (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8))___kmp_allocate((4 * sizeof(kmp_uint8)), "openmp/runtime/src/kmp_runtime.cpp"
, 4366);
  this_thr->th.th_task_state_top = 0;
  this_thr->th.th_task_state_stack_sz = 4;
  for (i = 0; i < this_thr->th.th_task_state_stack_sz;
       ++i) // zero init the stack
    this_thr->th.th_task_state_memo_stack[i] = 0;
}

KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here)if (!(!this_thr->th.th_spin_here)) { __kmp_debug_assert("!this_thr->th.th_spin_here"
, "openmp/runtime/src/kmp_runtime.cpp", 4374); };
KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0)if (!(this_thr->th.th_next_waiting == 0)) { __kmp_debug_assert
("this_thr->th.th_next_waiting == 0", "openmp/runtime/src/kmp_runtime.cpp"
, 4375); };

KMP_MB();
4378}

4380/* allocate a new thread for the requesting team. this is only called from
 within a forkjoin critical section. we will first try to get an available
 thread from the thread pool. if none is available, we will fork a new one
 assuming we are able to create a new one. this should be assured, as the
 caller should check on this first. */
4385kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
                                int new_tid) {
kmp_team_t *serial_team;
kmp_info_t *new_thr;
int new_gtid;

KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d\n"
, __kmp_get_global_thread_id()); };
KMP_DEBUG_ASSERT(root && team)if (!(root && team)) { __kmp_debug_assert("root && team"
, "openmp/runtime/src/kmp_runtime.cpp", 4392); };
4393#if !KMP_NESTED_HOT_TEAMS1
KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()))if (!((0 == __kmp_tid_from_gtid((__kmp_get_global_thread_id()
))))) { __kmp_debug_assert("(0 == __kmp_tid_from_gtid((__kmp_get_global_thread_id())))"
, "openmp/runtime/src/kmp_runtime.cpp", 4394); };
4395#endif
KMP_MB();

/* first, try to get one from the thread pool */
if (__kmp_thread_pool) {
  new_thr = CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool);
  __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;
  if (new_thr == __kmp_thread_pool_insert_pt) {
    __kmp_thread_pool_insert_pt = NULL__null;
  }
  TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
  __kmp_suspend_initialize_thread(new_thr);
  __kmp_lock_suspend_mx(new_thr);
  if (new_thr->th.th_active_in_pool == TRUE(!0)) {
    KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE)if (!(new_thr->th.th_active == (!0))) { __kmp_debug_assert
("new_thr->th.th_active == (!0)", "openmp/runtime/src/kmp_runtime.cpp"
, 4409); };
    KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth)(&__kmp_thread_pool_active_nth)->fetch_sub(1, std::memory_order_acq_rel
);
    new_thr->th.th_active_in_pool = FALSE0;
  }
  __kmp_unlock_suspend_mx(new_thr);

  KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); }
                __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread T#%d\n"
, __kmp_get_global_thread_id(), new_thr->th.th_info.ds.ds_gtid
); };
  KMP_ASSERT(!new_thr->th.th_team)if (!(!new_thr->th.th_team)) { __kmp_debug_assert("!new_thr->th.th_team"
, "openmp/runtime/src/kmp_runtime.cpp", 4417); };
  KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity)if (!(__kmp_nth < __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_nth < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 4418); };

  /* setup the thread structure */
  __kmp_initialize_info(new_thr, team, new_tid,
                        new_thr->th.th_info.ds.ds_gtid);
  KMP_DEBUG_ASSERT(new_thr->th.th_serial_team)if (!(new_thr->th.th_serial_team)) { __kmp_debug_assert("new_thr->th.th_serial_team"
, "openmp/runtime/src/kmp_runtime.cpp", 4423); };

  TCW_4(__kmp_nth, __kmp_nth + 1)(__kmp_nth) = (__kmp_nth + 1);

  new_thr->th.th_task_state = 0;
  new_thr->th.th_task_state_top = 0;
  new_thr->th.th_task_state_stack_sz = 4;

  if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
    // Make sure pool thread has transitioned to waiting on own thread struct
    KMP_DEBUG_ASSERT(new_thr->th.th_used_in_team.load() == 0)if (!(new_thr->th.th_used_in_team.load() == 0)) { __kmp_debug_assert
("new_thr->th.th_used_in_team.load() == 0", "openmp/runtime/src/kmp_runtime.cpp"
, 4433); };
    // Thread activated in __kmp_allocate_team when increasing team size
  }

4437#ifdef KMP_ADJUST_BLOCKTIME1
  /* Adjust blocktime back to zero if necessary */
  /* Middle initialization might not have occurred yet */
  if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
    if (__kmp_nth > __kmp_avail_proc) {
      __kmp_zero_bt = TRUE(!0);
    }
  }
4445#endif /* KMP_ADJUST_BLOCKTIME */

4447#if KMP_DEBUG1
  // If thread entered pool via __kmp_free_thread, wait_flag should !=
  // KMP_BARRIER_PARENT_FLAG.
  int b;
  kmp_balign_t *balign = new_thr->th.th_bar;
  for (b = 0; b < bs_last_barrier; ++b)
    KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 4453); };
4454#endif

  KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); }
                __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d using thread %p T#%d\n"
, __kmp_get_global_thread_id(), new_thr, new_thr->th.th_info
.ds.ds_gtid); };

  KMP_MB();
  return new_thr;
}

/* no, well fork a new one */
KMP_ASSERT(__kmp_nth == __kmp_all_nth)if (!(__kmp_nth == __kmp_all_nth)) { __kmp_debug_assert("__kmp_nth == __kmp_all_nth"
, "openmp/runtime/src/kmp_runtime.cpp", 4464); };
KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity)if (!(__kmp_all_nth < __kmp_threads_capacity)) { __kmp_debug_assert
("__kmp_all_nth < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 4465); };

4467#if KMP_USE_MONITOR
// If this is the first worker thread the RTL is creating, then also
// launch the monitor thread.  We try to do this as early as possible.
if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
  __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
  if (!TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
    KF_TRACE(10, ("before __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("before __kmp_create_monitor\n"
); };
    TCW_4(__kmp_init_monitor, 1)(__kmp_init_monitor) = (1);
    __kmp_create_monitor(&__kmp_monitor);
    KF_TRACE(10, ("after __kmp_create_monitor\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after __kmp_create_monitor\n"
); };
4477#if KMP_OS_WINDOWS0
    // AC: wait until monitor has started. This is a fix for CQ232808.
    // The reason is that if the library is loaded/unloaded in a loop with
    // small (parallel) work in between, then there is high probability that
    // monitor thread started after the library shutdown. At shutdown it is
    // too late to cope with the problem, because when the primary thread is
    // in DllMain (process detach) the monitor has no chances to start (it is
    // blocked), and primary thread has no means to inform the monitor that
    // the library has gone, because all the memory which the monitor can
    // access is going to be released/reset.
    while (TCR_4(__kmp_init_monitor)(__kmp_init_monitor) < 2) {
      KMP_YIELD(TRUE){ __kmp_x86_pause(); if (((!0)) && (((__kmp_use_yield
 == 1) || (__kmp_use_yield == 2 && (((__kmp_nth) >
 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc))))))) __kmp_yield
(); };
    }
    KF_TRACE(10, ("after monitor thread has started\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("after monitor thread has started\n"
); };
4491#endif
  }
  __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
}
4495#endif

KMP_MB();

{
  int new_start_gtid = TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)
                           ? 1
                           : __kmp_hidden_helper_threads_num + 1;

  for (new_gtid = new_start_gtid; TCR_PTR(__kmp_threads[new_gtid])((void *)(__kmp_threads[new_gtid])) != NULL__null;
       ++new_gtid) {
    KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity)if (!(new_gtid < __kmp_threads_capacity)) { __kmp_debug_assert
("new_gtid < __kmp_threads_capacity", "openmp/runtime/src/kmp_runtime.cpp"
, 4506); };
  }

  if (TCR_4(__kmp_init_hidden_helper_threads)(__kmp_init_hidden_helper_threads)) {
    KMP_DEBUG_ASSERT(new_gtid <= __kmp_hidden_helper_threads_num)if (!(new_gtid <= __kmp_hidden_helper_threads_num)) { __kmp_debug_assert
("new_gtid <= __kmp_hidden_helper_threads_num", "openmp/runtime/src/kmp_runtime.cpp"
, 4510); };
  }
}

/* allocate space for it. */
new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t))___kmp_allocate((sizeof(kmp_info_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 4515);

TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr)((__kmp_threads[new_gtid])) = ((new_thr));

4519#if USE_ITT_BUILD1 && USE_ITT_NOTIFY1 && KMP_DEBUG1
// suppress race conditions detection on synchronization flags in debug mode
// this helps to analyze library internals eliminating false positives
__itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(
    __itt_suppress_range, __itt_suppress_threading_errors0x000000ff,
    &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc));
__itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(
    __itt_suppress_range, __itt_suppress_threading_errors0x000000ff,
    &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state));
4528#if KMP_OS_WINDOWS0
__itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(
    __itt_suppress_range, __itt_suppress_threading_errors0x000000ff,
    &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init));
4532#else
__itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
                          __itt_suppress_threading_errors0x000000ff,
                          &new_thr->th.th_suspend_init_count,
                          sizeof(new_thr->th.th_suspend_init_count));
4537#endif
// TODO: check if we need to also suppress b_arrived flags
__itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
                          __itt_suppress_threading_errors0x000000ff,
                          CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go)const_cast<kmp_uint64 *>(&new_thr->th.th_bar[0].
bb.b_go),
                          sizeof(new_thr->th.th_bar[0].bb.b_go));
__itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
                          __itt_suppress_threading_errors0x000000ff,
                          CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go)const_cast<kmp_uint64 *>(&new_thr->th.th_bar[1].
bb.b_go),
                          sizeof(new_thr->th.th_bar[1].bb.b_go));
__itt_suppress_mark_range(!__kmp_itt_suppress_mark_range_ptr__3_0) ? (void)0 : __kmp_itt_suppress_mark_range_ptr__3_0(__itt_suppress_range,
                          __itt_suppress_threading_errors0x000000ff,
                          CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go)const_cast<kmp_uint64 *>(&new_thr->th.th_bar[2].
bb.b_go),
                          sizeof(new_thr->th.th_bar[2].bb.b_go));
4551#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */
if (__kmp_storage_map) {
  __kmp_print_thread_storage_map(new_thr, new_gtid);
}

// add the reserve serialized team, initialized from the team's primary thread
{
  kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
  KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before th_serial/serial_team\n"
); };
  new_thr->th.th_serial_team = serial_team =
      (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4562#if OMPT_SUPPORT1
                                        ompt_data_none{0}, // root parallel id
4564#endif
                                        proc_bind_default, &r_icvs,
                                        0 USE_NESTED_HOT_ARG(NULL), __null);
}
KMP_ASSERT(serial_team)if (!(serial_team)) { __kmp_debug_assert("serial_team", "openmp/runtime/src/kmp_runtime.cpp"
, 4568); };
serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for
// execution (it is unused for now).
serial_team->t.t_threads[0] = new_thr;
KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
         ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); }
          new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n"
, new_thr); };

/* setup the thread structures */
__kmp_initialize_info(new_thr, team, new_tid, new_gtid);

4579#if USE_FAST_MEMORY3
__kmp_initialize_fast_memory(new_thr);
4581#endif /* USE_FAST_MEMORY */

4583#if KMP_USE_BGET1
KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL)if (!(new_thr->th.th_local.bget_data == __null)) { __kmp_debug_assert
("new_thr->th.th_local.bget_data == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4584); };
__kmp_initialize_bget(new_thr);
4586#endif

__kmp_init_random(new_thr); // Initialize random number generator

/* Initialize these only once when thread is grabbed for a team allocation */
KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
         ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); }
          __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n"
, __kmp_get_global_thread_id(), 0, 0); };

int b;
kmp_balign_t *balign = new_thr->th.th_bar;
for (b = 0; b < bs_last_barrier; ++b) {
  balign[b].bb.b_go = KMP_INIT_BARRIER_STATE0;
  balign[b].bb.team = NULL__null;
  balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING0;
  balign[b].bb.use_oncore_barrier = 0;
}

TCW_PTR(new_thr->th.th_sleep_loc, NULL)((new_thr->th.th_sleep_loc)) = ((__null));
new_thr->th.th_sleep_loc_type = flag_unset;

new_thr->th.th_spin_here = FALSE0;
new_thr->th.th_next_waiting = 0;
4609#if KMP_OS_UNIX1
new_thr->th.th_blocking = false;
4611#endif

4613#if KMP_AFFINITY_SUPPORTED1
new_thr->th.th_current_place = KMP_PLACE_UNDEFINED(-2);
new_thr->th.th_new_place = KMP_PLACE_UNDEFINED(-2);
new_thr->th.th_first_place = KMP_PLACE_UNDEFINED(-2);
new_thr->th.th_last_place = KMP_PLACE_UNDEFINED(-2);
4618#endif
new_thr->th.th_def_allocator = __kmp_def_allocator;
new_thr->th.th_prev_level = 0;
new_thr->th.th_prev_num_threads = 1;

TCW_4(new_thr->th.th_in_pool, FALSE)(new_thr->th.th_in_pool) = (0);
new_thr->th.th_active_in_pool = FALSE0;
TCW_4(new_thr->th.th_active, TRUE)(new_thr->th.th_active) = ((!0));

/* adjust the global counters */
__kmp_all_nth++;
__kmp_nth++;

// if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
// numbers of procs, and method #2 (keyed API call) for higher numbers.
if (__kmp_adjust_gtid_mode) {
  if (__kmp_all_nth >= __kmp_tls_gtid_min) {
    if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 2) {
      TCW_4(__kmp_gtid_mode, 2)(__kmp_gtid_mode) = (2);
    }
  } else {
    if (TCR_4(__kmp_gtid_mode)(__kmp_gtid_mode) != 1) {
      TCW_4(__kmp_gtid_mode, 1)(__kmp_gtid_mode) = (1);
    }
  }
}

4645#ifdef KMP_ADJUST_BLOCKTIME1
/* Adjust blocktime back to zero if necessary       */
/* Middle initialization might not have occurred yet */
if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
  if (__kmp_nth > __kmp_avail_proc) {
    __kmp_zero_bt = TRUE(!0);
  }
}
4653#endif /* KMP_ADJUST_BLOCKTIME */

/* actually fork it and create the new worker thread */
KF_TRACE(if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); }
    10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: before __kmp_create_worker: %p\n"
, new_thr); };
__kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
KF_TRACE(10,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); }
         ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_thread: after __kmp_create_worker: %p\n"
, new_thr); };

KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); }
              new_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_thread: T#%d forked T#%d\n"
, __kmp_get_global_thread_id(), new_gtid); };
KMP_MB();
return new_thr;
4666}

4668/* Reinitialize team for reuse.
 The hot team code calls this case at every fork barrier, so EPCC barrier
 test are extremely sensitive to changes in it, esp. writes to the team
 struct, which cause a cache invalidation in all threads.
 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4673static void __kmp_reinitialize_team(kmp_team_t *team,
                                  kmp_internal_control_t *new_icvs,
                                  ident_t *loc) {
KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
              team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n"
, team->t.t_threads[0], team); };
KMP_DEBUG_ASSERT(team && new_icvs)if (!(team && new_icvs)) { __kmp_debug_assert("team && new_icvs"
, "openmp/runtime/src/kmp_runtime.cpp", 4678); };
KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc)if (!((!(__kmp_init_parallel)) || new_icvs->nproc)) { __kmp_debug_assert
("(!(__kmp_init_parallel)) || new_icvs->nproc", "openmp/runtime/src/kmp_runtime.cpp"
, 4679); };
KMP_CHECK_UPDATE(team->t.t_ident, loc)if ((team->t.t_ident) != (loc)) (team->t.t_ident) = (loc
);

KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID())if ((team->t.t_id) != ((~0))) (team->t.t_id) = ((~0));
// Copy ICVs to the primary thread's implicit taskdata
__kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE0);
copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);

KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); }
              team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n"
, team->t.t_threads[0], team); };
4689}

4691/* Initialize the team data structure.
 This assumes the t_threads and t_max_nproc are already set.
 Also, we don't touch the arguments */
4694static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
                                kmp_internal_control_t *new_icvs,
                                ident_t *loc) {
KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: enter: team=%p\n"
, team); };

/* verify */
KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 4700); };
KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc)if (!(new_nproc <= team->t.t_max_nproc)) { __kmp_debug_assert
("new_nproc <= team->t.t_max_nproc", "openmp/runtime/src/kmp_runtime.cpp"
, 4701); };
KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 4702); };
KMP_MB();

team->t.t_master_tid = 0; /* not needed */
/* team->t.t_master_bar;        not needed */
team->t.t_serialized = new_nproc > 1 ? 0 : 1;
team->t.t_nproc = new_nproc;

/* team->t.t_parent     = NULL; TODO not needed & would mess up hot team */
team->t.t_next_pool = NULL__null;
/* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess
 * up hot team */

TCW_SYNC_PTR(team->t.t_pkfn, NULL)((team->t.t_pkfn)) = ((__null)); /* not needed */
team->t.t_invoke = NULL__null; /* not needed */

// TODO???: team->t.t_max_active_levels       = new_max_active_levels;
team->t.t_sched.sched = new_icvs->sched.sched;

4721#if KMP_ARCH_X860 || KMP_ARCH_X86_641
team->t.t_fp_control_saved = FALSE0; /* not needed */
team->t.t_x87_fpu_control_word = 0; /* not needed */
team->t.t_mxcsr = 0; /* not needed */
4725#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */

team->t.t_construct = 0;

team->t.t_ordered.dt.t_value = 0;
team->t.t_master_active = FALSE0;

4732#ifdef KMP_DEBUG1
team->t.t_copypriv_data = NULL__null; /* not necessary, but nice for debugging */
4734#endif
4735#if KMP_OS_WINDOWS0
team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
4737#endif

team->t.t_control_stack_top = NULL__null;

__kmp_reinitialize_team(team, new_icvs, loc);

KMP_MB();
KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_initialize_team: exit: team=%p\n"
, team); };
4745}

4747#if (KMP_OS_LINUX1 || KMP_OS_FREEBSD0) && KMP_AFFINITY_SUPPORTED1
4748/* Sets full mask for thread and returns old mask, no changes to structures. */
4749static void
4750__kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
  int status;
  if (old_mask != NULL__null) {
    status = __kmp_get_system_affinity(old_mask, TRUE)(old_mask)->get_system_affinity((!0));
    int error = errno(*__errno_location ());
    if (status != 0) {
      __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError)__kmp_msg_format(kmp_i18n_msg_ChangeThreadAffMaskError), KMP_ERR(error)__kmp_msg_error_code(error),
                  __kmp_msg_null);
    }
  }
  __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE)(__kmp_affin_fullMask)->set_system_affinity((!0));
}
4763}
4764#endif

4766#if KMP_AFFINITY_SUPPORTED1

4768// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
4769// It calculates the worker + primary thread's partition based upon the parent
4770// thread's partition, and binds each worker to a thread in their partition.
4771// The primary thread's partition should already include its current binding.
4772static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
// Do not partition places for the hidden helper team
if (KMP_HIDDEN_HELPER_TEAM(team)(team->t.t_threads[0] == __kmp_hidden_helper_main_thread))
  return;
// Copy the primary thread's place partition to the team struct
kmp_info_t *master_th = team->t.t_threads[0];
KMP_DEBUG_ASSERT(master_th != NULL)if (!(master_th != __null)) { __kmp_debug_assert("master_th != __null"
, "openmp/runtime/src/kmp_runtime.cpp", 4778); };
kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
int first_place = master_th->th.th_first_place;
int last_place = master_th->th.th_last_place;
int masters_place = master_th->th.th_current_place;
int num_masks = __kmp_affinity.num_masks;
team->t.t_first_place = first_place;
team->t.t_last_place = last_place;

KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
 "bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
              "bound to place %d partition = [%d,%d]\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
 "bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
              proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
 "bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); }
              team->t.t_id, masters_place, first_place, last_place))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
 "bound to place %d partition = [%d,%d]\n", proc_bind, __kmp_gtid_from_thread
(team->t.t_threads[0]), team->t.t_id, masters_place, first_place
, last_place); };

switch (proc_bind) {

case proc_bind_default:
  // Serial teams might have the proc_bind policy set to proc_bind_default.
  // Not an issue -- we don't rebind primary thread for any proc_bind policy.
  KMP_DEBUG_ASSERT(team->t.t_nproc == 1)if (!(team->t.t_nproc == 1)) { __kmp_debug_assert("team->t.t_nproc == 1"
, "openmp/runtime/src/kmp_runtime.cpp", 4797); };
  break;

case proc_bind_primary: {
  int f;
  int n_th = team->t.t_nproc;
  for (f = 1; f < n_th; f++) {
    kmp_info_t *th = team->t.t_threads[f];
    KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4805); };
    th->th.th_first_place = first_place;
    th->th.th_last_place = last_place;
    th->th.th_new_place = masters_place;
    if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
        team->t.t_display_affinity != 1) {
      team->t.t_display_affinity = 1;
    }

    KA_TRACE(100, ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
                   "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
                   __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); }
                   f, masters_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: primary: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, masters_place, first_place, last_place
); };
  }
} break;

case proc_bind_close: {
  int f;
  int n_th = team->t.t_nproc;
  int n_places;
  if (first_place <= last_place) {
    n_places = last_place - first_place + 1;
  } else {
    n_places = num_masks - first_place + last_place + 1;
  }
  if (n_th <= n_places) {
    int place = masters_place;
    for (f = 1; f < n_th; f++) {
      kmp_info_t *th = team->t.t_threads[f];
      KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4834); };

      if (place == last_place) {
        place = first_place;
      } else if (place == (num_masks - 1)) {
        place = 0;
      } else {
        place++;
      }
      th->th.th_first_place = first_place;
      th->th.th_last_place = last_place;
      th->th.th_new_place = place;
      if (__kmp_display_affinity && place != th->th.th_current_place &&
          team->t.t_display_affinity != 1) {
        team->t.t_display_affinity = 1;
      }

      KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
                     "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
                     __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); }
                     team->t.t_id, f, place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, place, first_place, last_place); };
    }
  } else {
    int S, rem, gap, s_count;
    S = n_th / n_places;
    s_count = 0;
    rem = n_th - (S * n_places);
    gap = rem > 0 ? n_places / rem : n_places;
    int place = masters_place;
    int gap_ct = gap;
    for (f = 0; f < n_th; f++) {
      kmp_info_t *th = team->t.t_threads[f];
      KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4866); };

      th->th.th_first_place = first_place;
      th->th.th_last_place = last_place;
      th->th.th_new_place = place;
      if (__kmp_display_affinity && place != th->th.th_current_place &&
          team->t.t_display_affinity != 1) {
        team->t.t_display_affinity = 1;
      }
      s_count++;

      if ((s_count == S) && rem && (gap_ct == gap)) {
        // do nothing, add an extra thread to place on next iteration
      } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
        // we added an extra thread to this place; move to next place
        if (place == last_place) {
          place = first_place;
        } else if (place == (num_masks - 1)) {
          place = 0;
        } else {
          place++;
        }
        s_count = 0;
        gap_ct = 1;
        rem--;
      } else if (s_count == S) { // place full; don't add extra
        if (place == last_place) {
          place = first_place;
        } else if (place == (num_masks - 1)) {
          place = 0;
        } else {
          place++;
        }
        gap_ct++;
        s_count = 0;
      }

      KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
               ("__kmp_partition_places: close: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
                "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
                __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); }
                th->th.th_new_place, first_place, last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, first_place
, last_place); };
    }
    KMP_DEBUG_ASSERT(place == masters_place)if (!(place == masters_place)) { __kmp_debug_assert("place == masters_place"
, "openmp/runtime/src/kmp_runtime.cpp", 4909); };
  }
} break;

case proc_bind_spread: {
  int f;
  int n_th = team->t.t_nproc;
  int n_places;
  int thidx;
  if (first_place <= last_place) {
    n_places = last_place - first_place + 1;
  } else {
    n_places = num_masks - first_place + last_place + 1;
  }
  if (n_th <= n_places) {
    int place = -1;

    if (n_places != num_masks) {
      int S = n_places / n_th;
      int s_count, rem, gap, gap_ct;

      place = masters_place;
      rem = n_places - n_th * S;
      gap = rem ? n_th / rem : 1;
      gap_ct = gap;
      thidx = n_th;
      if (update_master_only == 1)
        thidx = 1;
      for (f = 0; f < thidx; f++) {
        kmp_info_t *th = team->t.t_threads[f];
        KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 4939); };

        th->th.th_first_place = place;
        th->th.th_new_place = place;
        if (__kmp_display_affinity && place != th->th.th_current_place &&
            team->t.t_display_affinity != 1) {
          team->t.t_display_affinity = 1;
        }
        s_count = 1;
        while (s_count < S) {
          if (place == last_place) {
            place = first_place;
          } else if (place == (num_masks - 1)) {
            place = 0;
          } else {
            place++;
          }
          s_count++;
        }
        if (rem && (gap_ct == gap)) {
          if (place == last_place) {
            place = first_place;
          } else if (place == (num_masks - 1)) {
            place = 0;
          } else {
            place++;
          }
          rem--;
          gap_ct = 0;
        }
        th->th.th_last_place = place;
        gap_ct++;

        if (place == last_place) {
          place = first_place;
        } else if (place == (num_masks - 1)) {
          place = 0;
        } else {
          place++;
        }

        KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
                 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
                  "partition = [%d,%d], num_masks: %u\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
                  __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
                  f, th->th.th_new_place, th->th.th_first_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); }
                  th->th.th_last_place, num_masks))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], num_masks: %u\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, num_masks
); };
      }
    } else {
      /* Having uniform space of available computation places I can create
         T partitions of round(P/T) size and put threads into the first
         place of each partition. */
      double current = static_cast<double>(masters_place);
      double spacing =
          (static_cast<double>(n_places + 1) / static_cast<double>(n_th));
      int first, last;
      kmp_info_t *th;

      thidx = n_th + 1;
      if (update_master_only == 1)
        thidx = 1;
      for (f = 0; f < thidx; f++) {
        first = static_cast<int>(current);
        last = static_cast<int>(current + spacing) - 1;
        KMP_DEBUG_ASSERT(last >= first)if (!(last >= first)) { __kmp_debug_assert("last >= first"
, "openmp/runtime/src/kmp_runtime.cpp", 5003); };
        if (first >= n_places) {
          if (masters_place) {
            first -= n_places;
            last -= n_places;
            if (first == (masters_place + 1)) {
              KMP_DEBUG_ASSERT(f == n_th)if (!(f == n_th)) { __kmp_debug_assert("f == n_th", "openmp/runtime/src/kmp_runtime.cpp"
, 5009); };
              first--;
            }
            if (last == masters_place) {
              KMP_DEBUG_ASSERT(f == (n_th - 1))if (!(f == (n_th - 1))) { __kmp_debug_assert("f == (n_th - 1)"
, "openmp/runtime/src/kmp_runtime.cpp", 5013); };
              last--;
            }
          } else {
            KMP_DEBUG_ASSERT(f == n_th)if (!(f == n_th)) { __kmp_debug_assert("f == n_th", "openmp/runtime/src/kmp_runtime.cpp"
, 5017); };
            first = 0;
            last = 0;
          }
        }
        if (last >= n_places) {
          last = (n_places - 1);
        }
        place = first;
        current += spacing;
        if (f < n_th) {
          KMP_DEBUG_ASSERT(0 <= first)if (!(0 <= first)) { __kmp_debug_assert("0 <= first", "openmp/runtime/src/kmp_runtime.cpp"
, 5028); };
          KMP_DEBUG_ASSERT(n_places > first)if (!(n_places > first)) { __kmp_debug_assert("n_places > first"
, "openmp/runtime/src/kmp_runtime.cpp", 5029); };
          KMP_DEBUG_ASSERT(0 <= last)if (!(0 <= last)) { __kmp_debug_assert("0 <= last", "openmp/runtime/src/kmp_runtime.cpp"
, 5030); };
          KMP_DEBUG_ASSERT(n_places > last)if (!(n_places > last)) { __kmp_debug_assert("n_places > last"
, "openmp/runtime/src/kmp_runtime.cpp", 5031); };
          KMP_DEBUG_ASSERT(last_place >= first_place)if (!(last_place >= first_place)) { __kmp_debug_assert("last_place >= first_place"
, "openmp/runtime/src/kmp_runtime.cpp", 5032); };
          th = team->t.t_threads[f];
          KMP_DEBUG_ASSERT(th)if (!(th)) { __kmp_debug_assert("th", "openmp/runtime/src/kmp_runtime.cpp"
, 5034); };
          th->th.th_first_place = first;
          th->th.th_new_place = place;
          th->th.th_last_place = last;
          if (__kmp_display_affinity && place != th->th.th_current_place &&
              team->t.t_display_affinity != 1) {
            team->t.t_display_affinity = 1;
          }
          KA_TRACE(100,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
                   ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
                    "partition = [%d,%d], spacing = %.4f\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
                    __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
                    team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; }
                    th->th.th_first_place, th->th.th_last_place, spacing))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d], spacing = %.4f\n", __kmp_gtid_from_thread
(team->t.t_threads[f]), team->t.t_id, f, th->th.th_new_place
, th->th.th_first_place, th->th.th_last_place, spacing)
; };
        }
      }
    }
    KMP_DEBUG_ASSERT(update_master_only || place == masters_place)if (!(update_master_only || place == masters_place)) { __kmp_debug_assert
("update_master_only || place == masters_place", "openmp/runtime/src/kmp_runtime.cpp"
, 5051); };
  } else {
    int S, rem, gap, s_count;
    S = n_th / n_places;
    s_count = 0;
    rem = n_th - (S * n_places);
    gap = rem > 0 ? n_places / rem : n_places;
    int place = masters_place;
    int gap_ct = gap;
    thidx = n_th;
    if (update_master_only == 1)
      thidx = 1;
    for (f = 0; f < thidx; f++) {
      kmp_info_t *th = team->t.t_threads[f];
      KMP_DEBUG_ASSERT(th != NULL)if (!(th != __null)) { __kmp_debug_assert("th != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 5065); };

      th->th.th_first_place = place;
      th->th.th_last_place = place;
      th->th.th_new_place = place;
      if (__kmp_display_affinity && place != th->th.th_current_place &&
          team->t.t_display_affinity != 1) {
        team->t.t_display_affinity = 1;
      }
      s_count++;

      if ((s_count == S) && rem && (gap_ct == gap)) {
        // do nothing, add an extra thread to place on next iteration
      } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
        // we added an extra thread to this place; move on to next place
        if (place == last_place) {
          place = first_place;
        } else if (place == (num_masks - 1)) {
          place = 0;
        } else {
          place++;
        }
        s_count = 0;
        gap_ct = 1;
        rem--;
      } else if (s_count == S) { // place is full; don't add extra thread
        if (place == last_place) {
          place = first_place;
        } else if (place == (num_masks - 1)) {
          place = 0;
        } else {
          place++;
        }
        gap_ct++;
        s_count = 0;
      }

      KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
                     "partition = [%d,%d]\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
                     __kmp_gtid_from_thread(team->t.t_threads[f]),if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
                     team->t.t_id, f, th->th.th_new_place,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); }
                     th->th.th_first_place, th->th.th_last_place))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
 "partition = [%d,%d]\n", __kmp_gtid_from_thread(team->t.t_threads
[f]), team->t.t_id, f, th->th.th_new_place, th->th.th_first_place
, th->th.th_last_place); };
    }
    KMP_DEBUG_ASSERT(update_master_only || place == masters_place)if (!(update_master_only || place == masters_place)) { __kmp_debug_assert
("update_master_only || place == masters_place", "openmp/runtime/src/kmp_runtime.cpp"
, 5108); };
  }
} break;

default:
  break;
}

KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_partition_places: exit T#%d\n"
, team->t.t_id); };
5117}

5119#endif // KMP_AFFINITY_SUPPORTED

5121/* allocate a new team data structure to use.  take one off of the free pool if
 available */
5123kmp_team_t *
5124__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
5125#if OMPT_SUPPORT1
                  ompt_data_t ompt_parallel_data,
5127#endif
                  kmp_proc_bind_t new_proc_bind,
                  kmp_internal_control_t *new_icvs,
                  int argc USE_NESTED_HOT_ARG(kmp_info_t *master), kmp_info_t *master) {
KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team)((void)0);
int f;
kmp_team_t *team;
int use_hot_team = !root->r.r_active;
int level = 0;
int do_place_partition = 1;

KA_TRACE(20, ("__kmp_allocate_team: called\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: called\n"
); };
KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0)if (!(new_nproc >= 1 && argc >= 0)) { __kmp_debug_assert
("new_nproc >= 1 && argc >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 5139); };
KMP_DEBUG_ASSERT(max_nproc >= new_nproc)if (!(max_nproc >= new_nproc)) { __kmp_debug_assert("max_nproc >= new_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5140); };
KMP_MB();

5143#if KMP_NESTED_HOT_TEAMS1
kmp_hot_team_ptr_t *hot_teams;
if (master) {
  team = master->th.th_team;
  level = team->t.t_active_level;
  if (master->th.th_teams_microtask) { // in teams construct?
    if (master->th.th_teams_size.nteams > 1 &&
        ( // #teams > 1
            team->t.t_pkfn ==
                (microtask_t)__kmp_teams_master || // inner fork of the teams
            master->th.th_teams_level <
                team->t.t_level)) { // or nested parallel inside the teams
      ++level; // not increment if #teams==1, or for outer fork of the teams;
      // increment otherwise
    }
    // Do not perform the place partition if inner fork of the teams
    // Wait until nested parallel region encountered inside teams construct
    if ((master->th.th_teams_size.nteams == 1 &&
         master->th.th_teams_level >= team->t.t_level) ||
        (team->t.t_pkfn == (microtask_t)__kmp_teams_master))
      do_place_partition = 0;
  }
  hot_teams = master->th.th_hot_teams;
  if (level < __kmp_hot_teams_max_level && hot_teams &&
      hot_teams[level].hot_team) {
    // hot team has already been allocated for given level
    use_hot_team = 1;
  } else {
    use_hot_team = 0;
  }
} else {
  // check we won't access uninitialized hot_teams, just in case
  KMP_DEBUG_ASSERT(new_nproc == 1)if (!(new_nproc == 1)) { __kmp_debug_assert("new_nproc == 1",
 "openmp/runtime/src/kmp_runtime.cpp", 5175); };
}
5177#endif
// Optimization to use a "hot" team
if (use_hot_team && new_nproc > 1) {
  KMP_DEBUG_ASSERT(new_nproc <= max_nproc)if (!(new_nproc <= max_nproc)) { __kmp_debug_assert("new_nproc <= max_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5180); };
5181#if KMP_NESTED_HOT_TEAMS1
  team = hot_teams[level].hot_team;
5183#else
  team = root->r.r_hot_team;
5185#endif
5186#if KMP_DEBUG1
  if (__kmp_tasking_mode != tskm_immediate_exec) {
    KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
 "task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
                  "task_team[1] = %p before reinit\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
 "task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); }
                  team->t.t_task_team[0], team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
 "task_team[1] = %p before reinit\n", team->t.t_task_team[
0], team->t.t_task_team[1]); };
  }
5192#endif

  if (team->t.t_nproc != new_nproc &&
      __kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
    // Distributed barrier may need a resize
    int old_nthr = team->t.t_nproc;
    __kmp_resize_dist_barrier(team, old_nthr, new_nproc);
  }

  // If not doing the place partition, then reset the team's proc bind
  // to indicate that partitioning of all threads still needs to take place
  if (do_place_partition == 0)
    team->t.t_proc_bind = proc_bind_default;
  // Has the number of threads changed?
  /* Let's assume the most common case is that the number of threads is
     unchanged, and put that case first. */
  if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
    KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team\n"
); };
    // This case can mean that omp_set_num_threads() was called and the hot
    // team size was already reduced, so we check the special flag
    if (team->t.t_size_changed == -1) {
      team->t.t_size_changed = 1;
    } else {
      KMP_CHECK_UPDATE(team->t.t_size_changed, 0)if ((team->t.t_size_changed) != (0)) (team->t.t_size_changed
) = (0);
    }

    // TODO???: team->t.t_max_active_levels = new_max_active_levels;
    kmp_r_sched_t new_sched = new_icvs->sched;
    // set primary thread's schedule as new run-time schedule
    KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched)if ((team->t.t_sched.sched) != (new_sched.sched)) (team->
t.t_sched.sched) = (new_sched.sched);

    __kmp_reinitialize_team(team, new_icvs,
                            root->r.r_uber_thread->th.th_ident);

    KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
                  team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); };
    __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);

5230#if KMP_AFFINITY_SUPPORTED1
    if ((team->t.t_size_changed == 0) &&
        (team->t.t_proc_bind == new_proc_bind)) {
      if (new_proc_bind == proc_bind_spread) {
        if (do_place_partition) {
          // add flag to update only master for spread
          __kmp_partition_places(team, 1);
        }
      }
      KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: "if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
 "proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
                     "proc_bind = %d, partition = [%d,%d]\n",if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
 "proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
                     team->t.t_id, new_proc_bind, team->t.t_first_place,if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
 "proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); }
                     team->t.t_last_place))if (kmp_a_debug >= 200) { __kmp_debug_printf ("__kmp_allocate_team: reusing hot team #%d bindings: "
 "proc_bind = %d, partition = [%d,%d]\n", team->t.t_id, new_proc_bind
, team->t.t_first_place, team->t.t_last_place); };
    } else {
      if (do_place_partition) {
        KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind);
        __kmp_partition_places(team);
      }
    }
5249#else
    KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind);
5251#endif /* KMP_AFFINITY_SUPPORTED */
  } else if (team->t.t_nproc > new_nproc) {
    KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
             ("__kmp_allocate_team: decreasing hot team thread count to %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); }
              new_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: decreasing hot team thread count to %d\n"
, new_nproc); };

    team->t.t_size_changed = 1;
    if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
      // Barrier size already reduced earlier in this function
      // Activate team threads via th_used_in_team
      __kmp_add_threads_to_team(team, new_nproc);
    }
5263#if KMP_NESTED_HOT_TEAMS1
    if (__kmp_hot_teams_mode == 0) {
      // AC: saved number of threads should correspond to team's value in this
      // mode, can be bigger in mode 1, when hot team has threads in reserve
      KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc)if (!(hot_teams[level].hot_team_nth == team->t.t_nproc)) {
 __kmp_debug_assert("hot_teams[level].hot_team_nth == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5267); };
      hot_teams[level].hot_team_nth = new_nproc;
5269#endif // KMP_NESTED_HOT_TEAMS
      /* release the extra threads we don't need any more */
      for (f = new_nproc; f < team->t.t_nproc; f++) {
        KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "openmp/runtime/src/kmp_runtime.cpp", 5272); };
        if (__kmp_tasking_mode != tskm_immediate_exec) {
          // When decreasing team size, threads no longer in the team should
          // unref task team.
          team->t.t_threads[f]->th.th_task_team = NULL__null;
        }
        __kmp_free_thread(team->t.t_threads[f]);
        team->t.t_threads[f] = NULL__null;
      }
5281#if KMP_NESTED_HOT_TEAMS1
    } // (__kmp_hot_teams_mode == 0)
    else {
      // When keeping extra threads in team, switch threads to wait on own
      // b_go flag
      for (f = new_nproc; f < team->t.t_nproc; ++f) {
        KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "openmp/runtime/src/kmp_runtime.cpp", 5287); };
        kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
        for (int b = 0; b < bs_last_barrier; ++b) {
          if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG2) {
            balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG3;
          }
          KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0)if ((balign[b].bb.leaf_kids) != (0)) (balign[b].bb.leaf_kids)
 = (0);
        }
      }
    }
5297#endif // KMP_NESTED_HOT_TEAMS
    team->t.t_nproc = new_nproc;
    // TODO???: team->t.t_max_active_levels = new_max_active_levels;
    KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched)if ((team->t.t_sched.sched) != (new_icvs->sched.sched))
 (team->t.t_sched.sched) = (new_icvs->sched.sched);
    __kmp_reinitialize_team(team, new_icvs,
                            root->r.r_uber_thread->th.th_ident);

    // Update remaining threads
    for (f = 0; f < new_nproc; ++f) {
      team->t.t_threads[f]->th.th_team_nproc = new_nproc;
    }

    // restore the current task state of the primary thread: should be the
    // implicit task
    KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); }
                  team->t.t_threads[0], team))if (kmp_f_debug >= 10) { __kmp_debug_printf ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n"
, 0, team->t.t_threads[0], team); };

    __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);

5316#ifdef KMP_DEBUG1
    for (f = 0; f < team->t.t_nproc; f++) {
      KMP_DEBUG_ASSERT(team->t.t_threads[f] &&if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5320); }
                       team->t.t_threads[f]->th.th_team_nproc ==if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5320); }
                           team->t.t_nproc)if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5320); };
    }
5322#endif

    if (do_place_partition) {
      KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind);
5326#if KMP_AFFINITY_SUPPORTED1
      __kmp_partition_places(team);
5328#endif
    }
  } else { // team->t.t_nproc < new_nproc
5331#if (KMP_OS_LINUX1 || KMP_OS_FREEBSD0) && KMP_AFFINITY_SUPPORTED1
    kmp_affin_mask_t *old_mask;
    if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
      KMP_CPU_ALLOC(old_mask)(old_mask = __kmp_affinity_dispatch->allocate_mask());
    }
5336#endif

    KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
             ("__kmp_allocate_team: increasing hot team thread count to %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); }
              new_nproc))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: increasing hot team thread count to %d\n"
, new_nproc); };
    int old_nproc = team->t.t_nproc; // save old value and use to update only
    team->t.t_size_changed = 1;

5344#if KMP_NESTED_HOT_TEAMS1
    int avail_threads = hot_teams[level].hot_team_nth;
    if (new_nproc < avail_threads)
      avail_threads = new_nproc;
    kmp_info_t **other_threads = team->t.t_threads;
    for (f = team->t.t_nproc; f < avail_threads; ++f) {
      // Adjust barrier data of reserved threads (if any) of the team
      // Other data will be set in __kmp_initialize_info() below.
      int b;
      kmp_balign_t *balign = other_threads[f]->th.th_bar;
      for (b = 0; b < bs_last_barrier; ++b) {
        balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
        KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 5356); };
5357#if USE_DEBUGGER0
        balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5359#endif
      }
    }
    if (hot_teams[level].hot_team_nth >= new_nproc) {
      // we have all needed threads in reserve, no need to allocate any
      // this only possible in mode 1, cannot have reserved threads in mode 0
      KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1)if (!(__kmp_hot_teams_mode == 1)) { __kmp_debug_assert("__kmp_hot_teams_mode == 1"
, "openmp/runtime/src/kmp_runtime.cpp", 5365); };
      team->t.t_nproc = new_nproc; // just get reserved threads involved
    } else {
      // We may have some threads in reserve, but not enough;
      // get reserved threads involved if any.
      team->t.t_nproc = hot_teams[level].hot_team_nth;
      hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size
5372#endif // KMP_NESTED_HOT_TEAMS
      if (team->t.t_max_nproc < new_nproc) {
        /* reallocate larger arrays */
        __kmp_reallocate_team_arrays(team, new_nproc);
        __kmp_reinitialize_team(team, new_icvs, NULL__null);
      }

5379#if (KMP_OS_LINUX1 || KMP_OS_FREEBSD0) && KMP_AFFINITY_SUPPORTED1
      /* Temporarily set full mask for primary thread before creation of
         workers. The reason is that workers inherit the affinity from the
         primary thread, so if a lot of workers are created on the single
         core quickly, they don't get a chance to set their own affinity for
         a long time. */
      __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5386#endif

      /* allocate new threads for the hot team */
      for (f = team->t.t_nproc; f < new_nproc; f++) {
        kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
        KMP_DEBUG_ASSERT(new_worker)if (!(new_worker)) { __kmp_debug_assert("new_worker", "openmp/runtime/src/kmp_runtime.cpp"
, 5391); };
        team->t.t_threads[f] = new_worker;

        KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
 "join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
                 ("__kmp_allocate_team: team %d init T#%d arrived: "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
 "join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  "join=%llu, plain=%llu\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
 "join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
 "join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  team->t.t_bar[bs_forkjoin_barrier].b_arrived,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
 "join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); }
                  team->t.t_bar[bs_plain_barrier].b_arrived))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init T#%d arrived: "
 "join=%llu, plain=%llu\n", team->t.t_id, __kmp_gtid_from_tid
(f, team), team->t.t_id, f, team->t.t_bar[bs_forkjoin_barrier
].b_arrived, team->t.t_bar[bs_plain_barrier].b_arrived); };

        { // Initialize barrier data for new threads.
          int b;
          kmp_balign_t *balign = new_worker->th.th_bar;
          for (b = 0; b < bs_last_barrier; ++b) {
            balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
            KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 5407); }
                             KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 5407); };
5408#if USE_DEBUGGER0
            balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5410#endif
          }
        }
      }

5415#if (KMP_OS_LINUX1 || KMP_OS_FREEBSD0) && KMP_AFFINITY_SUPPORTED1
      if (KMP_AFFINITY_CAPABLE()(__kmp_affin_mask_size > 0)) {
        /* Restore initial primary thread's affinity mask */
        __kmp_set_system_affinity(old_mask, TRUE)(old_mask)->set_system_affinity((!0));
        KMP_CPU_FREE(old_mask)__kmp_affinity_dispatch->deallocate_mask(old_mask);
      }
5421#endif
5422#if KMP_NESTED_HOT_TEAMS1
    } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
5424#endif // KMP_NESTED_HOT_TEAMS
    if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
      // Barrier size already increased earlier in this function
      // Activate team threads via th_used_in_team
      __kmp_add_threads_to_team(team, new_nproc);
    }
    /* make sure everyone is syncronized */
    // new threads below
    __kmp_initialize_team(team, new_nproc, new_icvs,
                          root->r.r_uber_thread->th.th_ident);

    /* reinitialize the threads */
    KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc)if (!(team->t.t_nproc == new_nproc)) { __kmp_debug_assert(
"team->t.t_nproc == new_nproc", "openmp/runtime/src/kmp_runtime.cpp"
, 5436); };
    for (f = 0; f < team->t.t_nproc; ++f)
      __kmp_initialize_info(team->t.t_threads[f], team, f,
                            __kmp_gtid_from_tid(f, team));

    if (level) { // set th_task_state for new threads in nested hot team
      // __kmp_initialize_info() no longer zeroes th_task_state, so we should
      // only need to set the th_task_state for the new threads. th_task_state
      // for primary thread will not be accurate until after this in
      // __kmp_fork_call(), so we look to the primary thread's memo_stack to
      // get the correct value.
      for (f = old_nproc; f < team->t.t_nproc; ++f)
        team->t.t_threads[f]->th.th_task_state =
            team->t.t_threads[0]->th.th_task_state_memo_stack[level];
    } else { // set th_task_state for new threads in non-nested hot team
      // copy primary thread's state
      kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state;
      for (f = old_nproc; f < team->t.t_nproc; ++f)
        team->t.t_threads[f]->th.th_task_state = old_state;
    }

5457#ifdef KMP_DEBUG1
    for (f = 0; f < team->t.t_nproc; ++f) {
      KMP_DEBUG_ASSERT(team->t.t_threads[f] &&if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5461); }
                       team->t.t_threads[f]->th.th_team_nproc ==if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5461); }
                           team->t.t_nproc)if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 5461); };
    }
5463#endif

    if (do_place_partition) {
      KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind)if ((team->t.t_proc_bind) != (new_proc_bind)) (team->t.
t_proc_bind) = (new_proc_bind);
5467#if KMP_AFFINITY_SUPPORTED1
      __kmp_partition_places(team);
5469#endif
    }
  } // Check changes in number of threads

  kmp_info_t *master = team->t.t_threads[0];
  if (master->th.th_teams_microtask) {
    for (f = 1; f < new_nproc; ++f) {
      // propagate teams construct specific info to workers
      kmp_info_t *thr = team->t.t_threads[f];
      thr->th.th_teams_microtask = master->th.th_teams_microtask;
      thr->th.th_teams_level = master->th.th_teams_level;
      thr->th.th_teams_size = master->th.th_teams_size;
    }
  }
5483#if KMP_NESTED_HOT_TEAMS1
  if (level) {
    // Sync barrier state for nested hot teams, not needed for outermost hot
    // team.
    for (f = 1; f < new_nproc; ++f) {
      kmp_info_t *thr = team->t.t_threads[f];
      int b;
      kmp_balign_t *balign = thr->th.th_bar;
      for (b = 0; b < bs_last_barrier; ++b) {
        balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
        KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG)if (!(balign[b].bb.wait_flag != 2)) { __kmp_debug_assert("balign[b].bb.wait_flag != 2"
, "openmp/runtime/src/kmp_runtime.cpp", 5493); };
5494#if USE_DEBUGGER0
        balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5496#endif
      }
    }
  }
5500#endif // KMP_NESTED_HOT_TEAMS

  /* reallocate space for arguments if necessary */
  __kmp_alloc_argv_entries(argc, team, TRUE(!0));
  KMP_CHECK_UPDATE(team->t.t_argc, argc)if ((team->t.t_argc) != (argc)) (team->t.t_argc) = (argc
);
  // The hot team re-uses the previous task team,
  // if untouched during the previous release->gather phase.

  KF_TRACE(10, (" hot_team = %p\n", team))if (kmp_f_debug >= 10) { __kmp_debug_printf (" hot_team = %p\n"
, team); };

5510#if KMP_DEBUG1
  if (__kmp_tasking_mode != tskm_immediate_exec) {
    KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
 "task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
                  "task_team[1] = %p after reinit\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
 "task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); }
                  team->t.t_task_team[0], team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: hot team task_team[0] = %p "
 "task_team[1] = %p after reinit\n", team->t.t_task_team[0
], team->t.t_task_team[1]); };
  }
5516#endif

5518#if OMPT_SUPPORT1
  __ompt_team_assign_id(team, ompt_parallel_data);
5520#endif

  KMP_MB();

  return team;
}

/* next, let's try to take one from the team pool */
KMP_MB();
for (team = CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool); (team);) {
  /* TODO: consider resizing undersized teams instead of reaping them, now
     that we have a resizing mechanism */
  if (team->t.t_max_nproc >= max_nproc) {
    /* take this team from the team pool */
    __kmp_team_pool = team->t.t_next_pool;

    if (max_nproc > 1 &&
        __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
      if (!team->t.b) { // Allocate barrier structure
        team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub);
      }
    }

    /* setup the team for fresh use */
    __kmp_initialize_team(team, new_nproc, new_icvs, NULL__null);

    KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
 "task_team[1] %p to NULL\n", &team->t.t_task_team[0],
 &team->t.t_task_team[1]); }
                  "task_team[1] %p to NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
 "task_team[1] %p to NULL\n", &team->t.t_task_team[0],
 &team->t.t_task_team[1]); }
                  &team->t.t_task_team[0], &team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and "
 "task_team[1] %p to NULL\n", &team->t.t_task_team[0],
 &team->t.t_task_team[1]); };
    team->t.t_task_team[0] = NULL__null;
    team->t.t_task_team[1] = NULL__null;

    /* reallocate space for arguments if necessary */
    __kmp_alloc_argv_entries(argc, team, TRUE(!0));
    KMP_CHECK_UPDATE(team->t.t_argc, argc)if ((team->t.t_argc) != (argc)) (team->t.t_argc) = (argc
);

    KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
        20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
             team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); };
    { // Initialize barrier data.
      int b;
      for (b = 0; b < bs_last_barrier; ++b) {
        team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE0;
5563#if USE_DEBUGGER0
        team->t.t_bar[b].b_master_arrived = 0;
        team->t.t_bar[b].b_team_arrived = 0;
5566#endif
      }
    }

    team->t.t_proc_bind = new_proc_bind;

    KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: using team from pool %d.\n"
, team->t.t_id); }
                  team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: using team from pool %d.\n"
, team->t.t_id); };

5575#if OMPT_SUPPORT1
    __ompt_team_assign_id(team, ompt_parallel_data);
5577#endif

    KMP_MB();

    return team;
  }

  /* reap team if it is too small, then loop back and check the next one */
  // not sure if this is wise, but, will be redone during the hot-teams
  // rewrite.
  /* TODO: Use technique to find the right size hot-team, don't reap them */
  team = __kmp_reap_team(team);
  __kmp_team_pool = team;
}

/* nothing available in the pool, no matter, make a new team! */
KMP_MB();
team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t))___kmp_allocate((sizeof(kmp_team_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 5594);

/* and set it up */
team->t.t_max_nproc = max_nproc;
if (max_nproc > 1 &&
    __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
  // Allocate barrier structure
  team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub);
}

/* NOTE well, for some reason allocating one big buffer and dividing it up
   seems to really hurt performance a lot on the P4, so, let's not use this */
__kmp_allocate_team_arrays(team, max_nproc);

KA_TRACE(20, ("__kmp_allocate_team: making a new team\n"))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: making a new team\n"
); };
__kmp_initialize_team(team, new_nproc, new_icvs, NULL__null);

KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
 "%p to NULL\n", &team->t.t_task_team[0], &team->
t.t_task_team[1]); }
              "%p to NULL\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
 "%p to NULL\n", &team->t.t_task_team[0], &team->
t.t_task_team[1]); }
              &team->t.t_task_team[0], &team->t.t_task_team[1]))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
 "%p to NULL\n", &team->t.t_task_team[0], &team->
t.t_task_team[1]); };
team->t.t_task_team[0] = NULL__null; // to be removed, as __kmp_allocate zeroes
// memory, no need to duplicate
team->t.t_task_team[1] = NULL__null; // to be removed, as __kmp_allocate zeroes
// memory, no need to duplicate

if (__kmp_storage_map) {
  __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc);
}

/* allocate space for arguments */
__kmp_alloc_argv_entries(argc, team, FALSE0);
team->t.t_argc = argc;

KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
         ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); }
          team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n"
, team->t.t_id, 0, 0); };
{ // Initialize barrier data.
  int b;
  for (b = 0; b < bs_last_barrier; ++b) {
    team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE0;
5634#if USE_DEBUGGER0
    team->t.t_bar[b].b_master_arrived = 0;
    team->t.t_bar[b].b_team_arrived = 0;
5637#endif
  }
}

team->t.t_proc_bind = new_proc_bind;

5643#if OMPT_SUPPORT1
__ompt_team_assign_id(team, ompt_parallel_data);
team->t.ompt_serialized_team_info = NULL__null;
5646#endif

KMP_MB();

KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: done creating a new team %d.\n"
, team->t.t_id); }
              team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_allocate_team: done creating a new team %d.\n"
, team->t.t_id); };

return team;
5654}

5656/* TODO implement hot-teams at all levels */
5657/* TODO implement lazy thread release on demand (disband request) */

5659/* free the team.  return it to the team pool.  release all the threads
* associated with it */
5661void __kmp_free_team(kmp_root_t *root,
                   kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master), kmp_info_t *master) {
int f;
KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d freeing team %d\n"
, __kmp_get_global_thread_id(), team->t.t_id); }
              team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d freeing team %d\n"
, __kmp_get_global_thread_id(), team->t.t_id); };

/* verify state */
KMP_DEBUG_ASSERT(root)if (!(root)) { __kmp_debug_assert("root", "openmp/runtime/src/kmp_runtime.cpp"
, 5668); };
KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 5669); };
KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc)if (!(team->t.t_nproc <= team->t.t_max_nproc)) { __kmp_debug_assert
("team->t.t_nproc <= team->t.t_max_nproc", "openmp/runtime/src/kmp_runtime.cpp"
, 5670); };
KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 5671); };

int use_hot_team = team == root->r.r_hot_team;
5674#if KMP_NESTED_HOT_TEAMS1
int level;
if (master) {
  level = team->t.t_active_level - 1;
  if (master->th.th_teams_microtask) { // in teams construct?
    if (master->th.th_teams_size.nteams > 1) {
      ++level; // level was not increased in teams construct for
      // team_of_masters
    }
    if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
        master->th.th_teams_level == team->t.t_level) {
      ++level; // level was not increased in teams construct for
      // team_of_workers before the parallel
    } // team->t.t_level will be increased inside parallel
  }
5689#if KMP_DEBUG1
  kmp_hot_team_ptr_t *hot_teams = master->th.th_hot_teams;
5691#endif
  if (level < __kmp_hot_teams_max_level) {
    KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team)if (!(team == hot_teams[level].hot_team)) { __kmp_debug_assert
("team == hot_teams[level].hot_team", "openmp/runtime/src/kmp_runtime.cpp"
, 5693); };
    use_hot_team = 1;
  }
}
5697#endif // KMP_NESTED_HOT_TEAMS

/* team is done working */
TCW_SYNC_PTR(team->t.t_pkfn,((team->t.t_pkfn)) = ((__null))
             NULL)((team->t.t_pkfn)) = ((__null)); // Important for Debugging Support Library.
5702#if KMP_OS_WINDOWS0
team->t.t_copyin_counter = 0; // init counter for possible reuse
5704#endif
// Do not reset pointer to parent team to NULL for hot teams.

/* if we are non-hot team, release our threads */
if (!use_hot_team) {
  if (__kmp_tasking_mode != tskm_immediate_exec) {
    // Wait for threads to reach reapable state
    for (f = 1; f < team->t.t_nproc; ++f) {
      KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "openmp/runtime/src/kmp_runtime.cpp", 5712); };
      kmp_info_t *th = team->t.t_threads[f];
      volatile kmp_uint32 *state = &th->th.th_reap_state;
      while (*state != KMP_SAFE_TO_REAP1) {
5716#if KMP_OS_WINDOWS0
        // On Windows a thread can be killed at any time, check this
        DWORD ecode;
        if (!__kmp_is_thread_alive(th, &ecode)) {
          *state = KMP_SAFE_TO_REAP1; // reset the flag for dead thread
          break;
        }
5723#endif
        // first check if thread is sleeping
        kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
        if (fl.is_sleeping())
          fl.resume(__kmp_gtid_from_thread(th));
        KMP_CPU_PAUSE()__kmp_x86_pause();
      }
    }

    // Delete task teams
    int tt_idx;
    for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
      kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
      if (task_team != NULL__null) {
        for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams
          KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "openmp/runtime/src/kmp_runtime.cpp", 5738); };
          team->t.t_threads[f]->th.th_task_team = NULL__null;
        }
        KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); }
            20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); }
            ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); }
             __kmp_get_gtid(), task_team, team->t.t_id))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n"
, __kmp_get_global_thread_id(), task_team, team->t.t_id); };
5745#if KMP_NESTED_HOT_TEAMS1
        __kmp_free_task_team(master, task_team);
5747#endif
        team->t.t_task_team[tt_idx] = NULL__null;
      }
    }
  }

  // Reset pointer to parent team only for non-hot teams.
  team->t.t_parent = NULL__null;
  team->t.t_level = 0;
  team->t.t_active_level = 0;

  /* free the worker threads */
  for (f = 1; f < team->t.t_nproc; ++f) {
    KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "openmp/runtime/src/kmp_runtime.cpp", 5760); };
    if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
      KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team),__sync_bool_compare_and_swap((volatile kmp_uint32 *)(&(team
->t.t_threads[f]->th.th_used_in_team)), (kmp_uint32)(1)
, (kmp_uint32)(2))
                                  1, 2)__sync_bool_compare_and_swap((volatile kmp_uint32 *)(&(team
->t.t_threads[f]->th.th_used_in_team)), (kmp_uint32)(1)
, (kmp_uint32)(2));
    }
    __kmp_free_thread(team->t.t_threads[f]);
  }

  if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
    if (team->t.b) {
      // wake up thread at old location
      team->t.b->go_release();
      if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647)) {
        for (f = 1; f < team->t.t_nproc; ++f) {
          if (team->t.b->sleep[f].sleep) {
            __kmp_atomic_resume_64(
                team->t.t_threads[f]->th.th_info.ds.ds_gtid,
                (kmp_atomic_flag_64<> *)NULL__null);
          }
        }
      }
      // Wait for threads to be removed from team
      for (int f = 1; f < team->t.t_nproc; ++f) {
        while (team->t.t_threads[f]->th.th_used_in_team.load() != 0)
          KMP_CPU_PAUSE()__kmp_x86_pause();
      }
    }
  }

  for (f = 1; f < team->t.t_nproc; ++f) {
    team->t.t_threads[f] = NULL__null;
  }

  if (team->t.t_max_nproc > 1 &&
      __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
    distributedBarrier::deallocate(team->t.b);
    team->t.b = NULL__null;
  }
  /* put the team back in the team pool */
  /* TODO limit size of team pool, call reap_team if pool too large */
  team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool);
  __kmp_team_pool = (volatile kmp_team_t *)team;
} else { // Check if team was created for primary threads in teams construct
  // See if first worker is a CG root
  KMP_DEBUG_ASSERT(team->t.t_threads[1] &&if (!(team->t.t_threads[1] && team->t.t_threads
[1]->th.th_cg_roots)) { __kmp_debug_assert("team->t.t_threads[1] && team->t.t_threads[1]->th.th_cg_roots"
, "openmp/runtime/src/kmp_runtime.cpp", 5805); }
                   team->t.t_threads[1]->th.th_cg_roots)if (!(team->t.t_threads[1] && team->t.t_threads
[1]->th.th_cg_roots)) { __kmp_debug_assert("team->t.t_threads[1] && team->t.t_threads[1]->th.th_cg_roots"
, "openmp/runtime/src/kmp_runtime.cpp", 5805); };
  if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {
    // Clean up the CG root nodes on workers so that this team can be re-used
    for (f = 1; f < team->t.t_nproc; ++f) {
      kmp_info_t *thr = team->t.t_threads[f];
      KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&if (!(thr && thr->th.th_cg_roots && thr->
th.th_cg_roots->cg_root == thr)) { __kmp_debug_assert("thr && thr->th.th_cg_roots && thr->th.th_cg_roots->cg_root == thr"
, "openmp/runtime/src/kmp_runtime.cpp", 5811); }
                       thr->th.th_cg_roots->cg_root == thr)if (!(thr && thr->th.th_cg_roots && thr->
th.th_cg_roots->cg_root == thr)) { __kmp_debug_assert("thr && thr->th.th_cg_roots && thr->th.th_cg_roots->cg_root == thr"
, "openmp/runtime/src/kmp_runtime.cpp", 5811); };
      // Pop current CG root off list
      kmp_cg_root_t *tmp = thr->th.th_cg_roots;
      thr->th.th_cg_roots = tmp->up;
      KA_TRACE(100, ("__kmp_free_team: Thread %p popping node %p and moving"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_team: Thread %p popping node %p and moving"
 " up to node %p. cg_nthreads was %d\n", thr, tmp, thr->th
.th_cg_roots, tmp->cg_nthreads); }
                     " up to node %p. cg_nthreads was %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_team: Thread %p popping node %p and moving"
 " up to node %p. cg_nthreads was %d\n", thr, tmp, thr->th
.th_cg_roots, tmp->cg_nthreads); }
                     thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_team: Thread %p popping node %p and moving"
 " up to node %p. cg_nthreads was %d\n", thr, tmp, thr->th
.th_cg_roots, tmp->cg_nthreads); };
      int i = tmp->cg_nthreads--;
      if (i == 1) {
        __kmp_free(tmp)___kmp_free((tmp), "openmp/runtime/src/kmp_runtime.cpp", 5820
); // free CG if we are the last thread in it
      }
      // Restore current task's thread_limit from CG root
      if (thr->th.th_cg_roots)
        thr->th.th_current_task->td_icvs.thread_limit =
            thr->th.th_cg_roots->cg_thread_limit;
    }
  }
}

KMP_MB();
5831}

5833/* reap the team.  destroy it, reclaim all its resources and free its memory */
5834kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
kmp_team_t *next_pool = team->t.t_next_pool;

KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 5837); };
KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 5838); };
KMP_DEBUG_ASSERT(team->t.t_disp_buffer)if (!(team->t.t_disp_buffer)) { __kmp_debug_assert("team->t.t_disp_buffer"
, "openmp/runtime/src/kmp_runtime.cpp", 5839); };
KMP_DEBUG_ASSERT(team->t.t_threads)if (!(team->t.t_threads)) { __kmp_debug_assert("team->t.t_threads"
, "openmp/runtime/src/kmp_runtime.cpp", 5840); };
KMP_DEBUG_ASSERT(team->t.t_argv)if (!(team->t.t_argv)) { __kmp_debug_assert("team->t.t_argv"
, "openmp/runtime/src/kmp_runtime.cpp", 5841); };

/* TODO clean the threads that are a part of this? */

/* free stuff */
__kmp_free_team_arrays(team);
if (team->t.t_argv != &team->t.t_inline_argv[0])
  __kmp_free((void *)team->t.t_argv)___kmp_free(((void *)team->t.t_argv), "openmp/runtime/src/kmp_runtime.cpp"
, 5848);
__kmp_free(team)___kmp_free((team), "openmp/runtime/src/kmp_runtime.cpp", 5849
);

KMP_MB();
return next_pool;
5853}

5855// Free the thread.  Don't reap it, just place it on the pool of available
5856// threads.
5857//
5858// Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid
5859// binding for the affinity mechanism to be useful.
5860//
5861// Now, we always keep the free list (__kmp_thread_pool) sorted by gtid.
5862// However, we want to avoid a potential performance problem by always
5863// scanning through the list to find the correct point at which to insert
5864// the thread (potential N**2 behavior).  To do this we keep track of the
5865// last place a thread struct was inserted (__kmp_thread_pool_insert_pt).
5866// With single-level parallelism, threads will always be added to the tail
5867// of the list, kept track of by __kmp_thread_pool_insert_pt.  With nested
5868// parallelism, all bets are off and we may need to scan through the entire
5869// free list.
5870//
5871// This change also has a potentially large performance benefit, for some
5872// applications.  Previously, as threads were freed from the hot team, they
5873// would be placed back on the free list in inverse order.  If the hot team
5874// grew back to it's original size, then the freed thread would be placed
5875// back on the hot team in reverse order.  This could cause bad cache
5876// locality problems on programs where the size of the hot team regularly
5877// grew and shrunk.
5878//
5879// Now, for single-level parallelism, the OMP tid is always == gtid.
5880void __kmp_free_thread(kmp_info_t *this_th) {
int gtid;
kmp_info_t **scan;

KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n"
, __kmp_get_global_thread_id(), this_th->th.th_info.ds.ds_gtid
); }
              __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n"
, __kmp_get_global_thread_id(), this_th->th.th_info.ds.ds_gtid
); };

KMP_DEBUG_ASSERT(this_th)if (!(this_th)) { __kmp_debug_assert("this_th", "openmp/runtime/src/kmp_runtime.cpp"
, 5887); };

// When moving thread to pool, switch thread to wait on own b_go flag, and
// uninitialized (NULL team).
int b;
kmp_balign_t *balign = this_th->th.th_bar;
for (b = 0; b < bs_last_barrier; ++b) {
  if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG2)
    balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG3;
  balign[b].bb.team = NULL__null;
  balign[b].bb.leaf_kids = 0;
}
this_th->th.th_task_state = 0;
this_th->th.th_reap_state = KMP_SAFE_TO_REAP1;

/* put thread back on the free pool */
TCW_PTR(this_th->th.th_team, NULL)((this_th->th.th_team)) = ((__null));
TCW_PTR(this_th->th.th_root, NULL)((this_th->th.th_root)) = ((__null));
TCW_PTR(this_th->th.th_dispatch, NULL)((this_th->th.th_dispatch)) = ((__null)); /* NOT NEEDED */

while (this_th->th.th_cg_roots) {
  this_th->th.th_cg_roots->cg_nthreads--;
  KA_TRACE(100, ("__kmp_free_thread: Thread %p decrement cg_nthreads on node"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_thread: Thread %p decrement cg_nthreads on node"
 " %p of thread  %p to %d\n", this_th, this_th->th.th_cg_roots
, this_th->th.th_cg_roots->cg_root, this_th->th.th_cg_roots
->cg_nthreads); }
                 " %p of thread  %p to %d\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_thread: Thread %p decrement cg_nthreads on node"
 " %p of thread  %p to %d\n", this_th, this_th->th.th_cg_roots
, this_th->th.th_cg_roots->cg_root, this_th->th.th_cg_roots
->cg_nthreads); }
                 this_th, this_th->th.th_cg_roots,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_thread: Thread %p decrement cg_nthreads on node"
 " %p of thread  %p to %d\n", this_th, this_th->th.th_cg_roots
, this_th->th.th_cg_roots->cg_root, this_th->th.th_cg_roots
->cg_nthreads); }
                 this_th->th.th_cg_roots->cg_root,if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_thread: Thread %p decrement cg_nthreads on node"
 " %p of thread  %p to %d\n", this_th, this_th->th.th_cg_roots
, this_th->th.th_cg_roots->cg_root, this_th->th.th_cg_roots
->cg_nthreads); }
                 this_th->th.th_cg_roots->cg_nthreads))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_free_thread: Thread %p decrement cg_nthreads on node"
 " %p of thread  %p to %d\n", this_th, this_th->th.th_cg_roots
, this_th->th.th_cg_roots->cg_root, this_th->th.th_cg_roots
->cg_nthreads); };
  kmp_cg_root_t *tmp = this_th->th.th_cg_roots;
  if (tmp->cg_root == this_th) { // Thread is a cg_root
    KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0)if (!(tmp->cg_nthreads == 0)) { __kmp_debug_assert("tmp->cg_nthreads == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 5916); };
    KA_TRACE(if (kmp_a_debug >= 5) { __kmp_debug_printf ("__kmp_free_thread: Thread %p freeing node %p\n"
, this_th, tmp); }
        5, ("__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp))if (kmp_a_debug >= 5) { __kmp_debug_printf ("__kmp_free_thread: Thread %p freeing node %p\n"
, this_th, tmp); };
    this_th->th.th_cg_roots = tmp->up;
    __kmp_free(tmp)___kmp_free((tmp), "openmp/runtime/src/kmp_runtime.cpp", 5920
);
  } else { // Worker thread
    if (tmp->cg_nthreads == 0) { // last thread leaves contention group
      __kmp_free(tmp)___kmp_free((tmp), "openmp/runtime/src/kmp_runtime.cpp", 5923
);
    }
    this_th->th.th_cg_roots = NULL__null;
    break;
  }
}

/* If the implicit task assigned to this thread can be used by other threads
 * -> multiple threads can share the data and try to free the task at
 * __kmp_reap_thread at exit. This duplicate use of the task data can happen
 * with higher probability when hot team is disabled but can occurs even when
 * the hot team is enabled */
__kmp_free_implicit_task(this_th);
this_th->th.th_current_task = NULL__null;

// If the __kmp_thread_pool_insert_pt is already past the new insert
// point, then we need to re-scan the entire list.
gtid = this_th->th.th_info.ds.ds_gtid;
if (__kmp_thread_pool_insert_pt != NULL__null) {
  KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL)if (!(__kmp_thread_pool != __null)) { __kmp_debug_assert("__kmp_thread_pool != __null"
, "openmp/runtime/src/kmp_runtime.cpp", 5942); };
  if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
    __kmp_thread_pool_insert_pt = NULL__null;
  }
}

// Scan down the list to find the place to insert the thread.
// scan is the address of a link in the list, possibly the address of
// __kmp_thread_pool itself.
//
// In the absence of nested parallelism, the for loop will have 0 iterations.
if (__kmp_thread_pool_insert_pt != NULL__null) {
  scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
} else {
  scan = CCAST(kmp_info_t **, &__kmp_thread_pool)const_cast<kmp_info_t **>(&__kmp_thread_pool);
}
for (; (*scan != NULL__null) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
     scan = &((*scan)->th.th_next_pool))
  ;

// Insert the new element on the list, and set __kmp_thread_pool_insert_pt
// to its address.
TCW_PTR(this_th->th.th_next_pool, *scan)((this_th->th.th_next_pool)) = ((*scan));
__kmp_thread_pool_insert_pt = *scan = this_th;
KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||if (!((this_th->th.th_next_pool == __null) || (this_th->
th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th
.th_info.ds.ds_gtid))) { __kmp_debug_assert("(this_th->th.th_next_pool == __null) || (this_th->th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info.ds.ds_gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 5968); }
                 (this_th->th.th_info.ds.ds_gtid <if (!((this_th->th.th_next_pool == __null) || (this_th->
th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th
.th_info.ds.ds_gtid))) { __kmp_debug_assert("(this_th->th.th_next_pool == __null) || (this_th->th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info.ds.ds_gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 5968); }
                  this_th->th.th_next_pool->th.th_info.ds.ds_gtid))if (!((this_th->th.th_next_pool == __null) || (this_th->
th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th
.th_info.ds.ds_gtid))) { __kmp_debug_assert("(this_th->th.th_next_pool == __null) || (this_th->th.th_info.ds.ds_gtid < this_th->th.th_next_pool->th.th_info.ds.ds_gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 5968); };
TCW_4(this_th->th.th_in_pool, TRUE)(this_th->th.th_in_pool) = ((!0));
__kmp_suspend_initialize_thread(this_th);
__kmp_lock_suspend_mx(this_th);
if (this_th->th.th_active == TRUE(!0)) {
  KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth)(&__kmp_thread_pool_active_nth)->fetch_add(1, std::memory_order_acq_rel
);
  this_th->th.th_active_in_pool = TRUE(!0);
}
5976#if KMP_DEBUG1
else {
  KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE)if (!(this_th->th.th_active_in_pool == 0)) { __kmp_debug_assert
("this_th->th.th_active_in_pool == 0", "openmp/runtime/src/kmp_runtime.cpp"
, 5978); };
}
5980#endif
__kmp_unlock_suspend_mx(this_th);

TCW_4(__kmp_nth, __kmp_nth - 1)(__kmp_nth) = (__kmp_nth - 1);

5985#ifdef KMP_ADJUST_BLOCKTIME1
/* Adjust blocktime back to user setting or default if necessary */
/* Middle initialization might never have occurred                */
if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
  KMP_DEBUG_ASSERT(__kmp_avail_proc > 0)if (!(__kmp_avail_proc > 0)) { __kmp_debug_assert("__kmp_avail_proc > 0"
, "openmp/runtime/src/kmp_runtime.cpp", 5989); };
  if (__kmp_nth <= __kmp_avail_proc) {
    __kmp_zero_bt = FALSE0;
  }
}
5994#endif /* KMP_ADJUST_BLOCKTIME */

KMP_MB();
5997}

5999/* ------------------------------------------------------------------------ */

6001void *__kmp_launch_thread(kmp_info_t *this_thr) {
6002#if OMP_PROFILING_SUPPORT0
ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE");
// TODO: add a configuration option for time granularity
if (ProfileTraceFile)
  llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget");
6007#endif

int gtid = this_thr->th.th_info.ds.ds_gtid;
/*    void                 *stack_data;*/
kmp_team_t **volatile pteam;

KMP_MB();
KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_launch_thread: T#%d start\n"
, gtid); };

if (__kmp_env_consistency_check) {
  this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak?
}

6020#if OMPD_SUPPORT1
if (ompd_state & OMPD_ENABLE_BP0x1)
  ompd_bp_thread_begin();
6023#endif

6025#if OMPT_SUPPORT1
ompt_data_t *thread_data = nullptr;
if (ompt_enabled.enabled) {
  thread_data = &(this_thr->th.ompt_thread_info.thread_data);
  *thread_data = ompt_data_none{0};

  this_thr->th.ompt_thread_info.state = ompt_state_overhead;
  this_thr->th.ompt_thread_info.wait_id = 0;
  this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0)__builtin_frame_address(0);
  this_thr->th.ompt_thread_info.parallel_flags = 0;
  if (ompt_enabled.ompt_callback_thread_begin) {
    ompt_callbacks.ompt_callback(ompt_callback_thread_begin)ompt_callback_thread_begin_callback(
        ompt_thread_worker, thread_data);
  }
  this_thr->th.ompt_thread_info.state = ompt_state_idle;
}
6041#endif

/* This is the place where threads wait for work */
while (!TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
  KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid])if (!(this_thr == __kmp_threads[gtid])) { __kmp_debug_assert(
"this_thr == __kmp_threads[gtid]", "openmp/runtime/src/kmp_runtime.cpp"
, 6045); };
  KMP_MB();

  /* wait for work to do */
  KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d waiting for work\n"
, gtid); };

  /* No tid yet since not part of a team */
  __kmp_fork_barrier(gtid, KMP_GTID_DNE(-2));

6054#if OMPT_SUPPORT1
  if (ompt_enabled.enabled) {
    this_thr->th.ompt_thread_info.state = ompt_state_overhead;
  }
6058#endif

  pteam = &this_thr->th.th_team;

  /* have we been allocated? */
  if (TCR_SYNC_PTR(*pteam)((void *)(*pteam)) && !TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
    /* we were just woken up, so run our new task */
    if (TCR_SYNC_PTR((*pteam)->t.t_pkfn)((void *)((*pteam)->t.t_pkfn)) != NULL__null) {
      int rc;
      KA_TRACE(20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam
)->t.t_pkfn); }
               ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam
)->t.t_pkfn); }
                gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam
)->t.t_pkfn); }
                (*pteam)->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n"
, gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam
)->t.t_pkfn); };

      updateHWFPControl(*pteam);

6074#if OMPT_SUPPORT1
      if (ompt_enabled.enabled) {
        this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
      }
6078#endif

      rc = (*pteam)->t.t_invoke(gtid);
      KMP_ASSERT(rc)if (!(rc)) { __kmp_debug_assert("rc", "openmp/runtime/src/kmp_runtime.cpp"
, 6081); };

      KMP_MB();
      KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n"
, gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam
)->t.t_pkfn); }
                    gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n"
, gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam
)->t.t_pkfn); }
                    (*pteam)->t.t_pkfn))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n"
, gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam
)->t.t_pkfn); };
    }
6088#if OMPT_SUPPORT1
    if (ompt_enabled.enabled) {
      /* no frame set while outside task */
      __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none{0};

      this_thr->th.ompt_thread_info.state = ompt_state_overhead;
    }
6095#endif
    /* join barrier after parallel region */
    __kmp_join_barrier(gtid);
  }
}
TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done)((void *)((intptr_t)__kmp_global.g.g_done));

6102#if OMPD_SUPPORT1
if (ompd_state & OMPD_ENABLE_BP0x1)
  ompd_bp_thread_end();
6105#endif

6107#if OMPT_SUPPORT1
if (ompt_enabled.ompt_callback_thread_end) {
  ompt_callbacks.ompt_callback(ompt_callback_thread_end)ompt_callback_thread_end_callback(thread_data);
}
6111#endif

this_thr->th.th_task_team = NULL__null;
/* run the destructors for the threadprivate data for this thread */
__kmp_common_destroy_gtid(gtid);

KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_launch_thread: T#%d done\n"
, gtid); };
KMP_MB();

6120#if OMP_PROFILING_SUPPORT0
llvm::timeTraceProfilerFinishThread();
6122#endif
return this_thr;
6124}

6126/* ------------------------------------------------------------------------ */

6128void __kmp_internal_end_dest(void *specific_gtid) {
// Make sure no significant bits are lost
int gtid;
__kmp_type_convert((kmp_intptr_t)specific_gtid - 1, &gtid);

KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_internal_end_dest: T#%d\n"
, gtid); };
/* NOTE: the gtid is stored as gitd+1 in the thread-local-storage
 * this is because 0 is reserved for the nothing-stored case */

__kmp_internal_end_thread(gtid);
6138}

6140#if KMP_OS_UNIX1 && KMP_DYNAMIC_LIB1

6142__attribute__((destructor)) void __kmp_internal_end_dtor(void) {
__kmp_internal_end_atexit();
6144}

6146#endif

6148/* [Windows] josh: when the atexit handler is called, there may still be more
 than one thread alive */
6150void __kmp_internal_end_atexit(void) {
KA_TRACE(30, ("__kmp_internal_end_atexit\n"))if (kmp_a_debug >= 30) { __kmp_debug_printf ("__kmp_internal_end_atexit\n"
); };
/* [Windows]
   josh: ideally, we want to completely shutdown the library in this atexit
   handler, but stat code that depends on thread specific data for gtid fails
   because that data becomes unavailable at some point during the shutdown, so
   we call __kmp_internal_end_thread instead. We should eventually remove the
   dependency on __kmp_get_specific_gtid in the stat code and use
   __kmp_internal_end_library to cleanly shutdown the library.

   // TODO: Can some of this comment about GVS be removed?
   I suspect that the offending stat code is executed when the calling thread
   tries to clean up a dead root thread's data structures, resulting in GVS
   code trying to close the GVS structures for that thread, but since the stat
   code uses __kmp_get_specific_gtid to get the gtid with the assumption that
   the calling thread is cleaning up itself instead of another thread, it get
   confused. This happens because allowing a thread to unregister and cleanup
   another thread is a recent modification for addressing an issue.
   Based on the current design (20050722), a thread may end up
   trying to unregister another thread only if thread death does not trigger
   the calling of __kmp_internal_end_thread.  For Linux* OS, there is the
   thread specific data destructor function to detect thread death. For
   Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there
   is nothing.  Thus, the workaround is applicable only for Windows static
   stat library. */
__kmp_internal_end_library(-1);
6176#if KMP_OS_WINDOWS0
__kmp_close_console();
6178#endif
6179}

6181static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
// It is assumed __kmp_forkjoin_lock is acquired.

int gtid;

KMP_DEBUG_ASSERT(thread != NULL)if (!(thread != __null)) { __kmp_debug_assert("thread != __null"
, "openmp/runtime/src/kmp_runtime.cpp", 6186); };

gtid = thread->th.th_info.ds.ds_gtid;

if (!is_root) {
  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647)) {
    /* Assume the threads are at the fork barrier here */
    KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n"
, gtid); }
        20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n"
, gtid); }
             gtid))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n"
, gtid); };
    if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
      while (
          !KMP_COMPARE_AND_STORE_ACQ32(&(thread->th.th_used_in_team), 0, 3)__sync_bool_compare_and_swap((volatile kmp_uint32 *)(&(thread
->th.th_used_in_team)), (kmp_uint32)(0), (kmp_uint32)(3)))
        KMP_CPU_PAUSE()__kmp_x86_pause();
      __kmp_resume_32(gtid, (kmp_flag_32<false, false> *)NULL__null);
    } else {
      /* Need release fence here to prevent seg faults for tree forkjoin
         barrier (GEH) */
      kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
                         thread);
      __kmp_release_64(&flag);
    }
  }

  // Terminate OS thread.
  __kmp_reap_worker(thread);

  // The thread was killed asynchronously.  If it was actively
  // spinning in the thread pool, decrement the global count.
  //
  // There is a small timing hole here - if the worker thread was just waking
  // up after sleeping in the pool, had reset it's th_active_in_pool flag but
  // not decremented the global counter __kmp_thread_pool_active_nth yet, then
  // the global counter might not get updated.
  //
  // Currently, this can only happen as the library is unloaded,
  // so there are no harmful side effects.
  if (thread->th.th_active_in_pool) {
    thread->th.th_active_in_pool = FALSE0;
    KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth)(&__kmp_thread_pool_active_nth)->fetch_sub(1, std::memory_order_acq_rel
);
    KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0)if (!(__kmp_thread_pool_active_nth >= 0)) { __kmp_debug_assert
("__kmp_thread_pool_active_nth >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 6226); };
  }
}

__kmp_free_implicit_task(thread);

6232// Free the fast memory for tasking
6233#if USE_FAST_MEMORY3
__kmp_free_fast_memory(thread);
6235#endif /* USE_FAST_MEMORY */

__kmp_suspend_uninitialize_thread(thread);

KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread)if (!(__kmp_threads[gtid] == thread)) { __kmp_debug_assert("__kmp_threads[gtid] == thread"
, "openmp/runtime/src/kmp_runtime.cpp", 6239); };
TCW_SYNC_PTR(__kmp_threads[gtid], NULL)((__kmp_threads[gtid])) = ((__null));

--__kmp_all_nth;
// __kmp_nth was decremented when thread is added to the pool.

6245#ifdef KMP_ADJUST_BLOCKTIME1
/* Adjust blocktime back to user setting or default if necessary */
/* Middle initialization might never have occurred                */
if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
  KMP_DEBUG_ASSERT(__kmp_avail_proc > 0)if (!(__kmp_avail_proc > 0)) { __kmp_debug_assert("__kmp_avail_proc > 0"
, "openmp/runtime/src/kmp_runtime.cpp", 6249); };
  if (__kmp_nth <= __kmp_avail_proc) {
    __kmp_zero_bt = FALSE0;
  }
}
6254#endif /* KMP_ADJUST_BLOCKTIME */

/* free the memory being used */
if (__kmp_env_consistency_check) {
  if (thread->th.th_cons) {
    __kmp_free_cons_stack(thread->th.th_cons);
    thread->th.th_cons = NULL__null;
  }
}

if (thread->th.th_pri_common != NULL__null) {
  __kmp_free(thread->th.th_pri_common)___kmp_free((thread->th.th_pri_common), "openmp/runtime/src/kmp_runtime.cpp"
, 6265);
  thread->th.th_pri_common = NULL__null;
}

if (thread->th.th_task_state_memo_stack != NULL__null) {
  __kmp_free(thread->th.th_task_state_memo_stack)___kmp_free((thread->th.th_task_state_memo_stack), "openmp/runtime/src/kmp_runtime.cpp"
, 6270);
  thread->th.th_task_state_memo_stack = NULL__null;
}

6274#if KMP_USE_BGET1
if (thread->th.th_local.bget_data != NULL__null) {
  __kmp_finalize_bget(thread);
}
6278#endif

6280#if KMP_AFFINITY_SUPPORTED1
if (thread->th.th_affin_mask != NULL__null) {
  KMP_CPU_FREE(thread->th.th_affin_mask)__kmp_affinity_dispatch->deallocate_mask(thread->th.th_affin_mask
);
  thread->th.th_affin_mask = NULL__null;
}
6285#endif /* KMP_AFFINITY_SUPPORTED */

6287#if KMP_USE_HIER_SCHED0
if (thread->th.th_hier_bar_data != NULL__null) {
  __kmp_free(thread->th.th_hier_bar_data)___kmp_free((thread->th.th_hier_bar_data), "openmp/runtime/src/kmp_runtime.cpp"
, 6289);
  thread->th.th_hier_bar_data = NULL__null;
}
6292#endif

__kmp_reap_team(thread->th.th_serial_team);
thread->th.th_serial_team = NULL__null;
__kmp_free(thread)___kmp_free((thread), "openmp/runtime/src/kmp_runtime.cpp", 6296
);

KMP_MB();

6300} // __kmp_reap_thread

6302static void __kmp_itthash_clean(kmp_info_t *th) {
6303#if USE_ITT_NOTIFY1
if (__kmp_itt_region_domains.count > 0) {
  for (int i = 0; i < KMP_MAX_FRAME_DOMAINS; ++i) {
    kmp_itthash_entry_t *bucket = __kmp_itt_region_domains.buckets[i];
    while (bucket) {
      kmp_itthash_entry_t *next = bucket->next_in_bucket;
      __kmp_thread_free(th, bucket)___kmp_thread_free((th), (bucket), "openmp/runtime/src/kmp_runtime.cpp"
, 6309);
      bucket = next;
    }
  }
}
if (__kmp_itt_barrier_domains.count > 0) {
  for (int i = 0; i < KMP_MAX_FRAME_DOMAINS; ++i) {
    kmp_itthash_entry_t *bucket = __kmp_itt_barrier_domains.buckets[i];
    while (bucket) {
      kmp_itthash_entry_t *next = bucket->next_in_bucket;
      __kmp_thread_free(th, bucket)___kmp_thread_free((th), (bucket), "openmp/runtime/src/kmp_runtime.cpp"
, 6319);
      bucket = next;
    }
  }
}
6324#endif
6325}

6327static void __kmp_internal_end(void) {
int i;

/* First, unregister the library */
__kmp_unregister_library();

6333#if KMP_OS_WINDOWS0
/* In Win static library, we can't tell when a root actually dies, so we
   reclaim the data structures for any root threads that have died but not
   unregistered themselves, in order to shut down cleanly.
   In Win dynamic library we also can't tell when a thread dies.  */
__kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of
6339// dead roots
6340#endif

for (i = 0; i < __kmp_threads_capacity; i++)
  if (__kmp_root[i])
    if (__kmp_root[i]->r.r_active)
      break;
KMP_MB(); /* Flush all pending memory write invalidates.  */
TCW_SYNC_4(__kmp_global.g.g_done, TRUE)(__kmp_global.g.g_done) = ((!0));

if (i < __kmp_threads_capacity) {
6350#if KMP_USE_MONITOR
  // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor??
  KMP_MB(); /* Flush all pending memory write invalidates.  */

  // Need to check that monitor was initialized before reaping it. If we are
  // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then
  // __kmp_monitor will appear to contain valid data, but it is only valid in
  // the parent process, not the child.
  // New behavior (201008): instead of keying off of the flag
  // __kmp_init_parallel, the monitor thread creation is keyed off
  // of the new flag __kmp_init_monitor.
  __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
  if (TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
    __kmp_reap_monitor(&__kmp_monitor);
    TCW_4(__kmp_init_monitor, 0)(__kmp_init_monitor) = (0);
  }
  __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
  KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end: monitor reaped\n"
); };
6368#endif // KMP_USE_MONITOR
} else {
6370/* TODO move this to cleanup code */
6371#ifdef KMP_DEBUG1
  /* make sure that everything has properly ended */
  for (i = 0; i < __kmp_threads_capacity; i++) {
    if (__kmp_root[i]) {
      //                    KMP_ASSERT( ! KMP_UBER_GTID( i ) );         // AC:
      //                    there can be uber threads alive here
      KMP_ASSERT(!__kmp_root[i]->r.r_active)if (!(!__kmp_root[i]->r.r_active)) { __kmp_debug_assert("!__kmp_root[i]->r.r_active"
, "openmp/runtime/src/kmp_runtime.cpp", 6377); }; // TODO: can they be active?
    }
  }
6380#endif

  KMP_MB();

  // Reap the worker threads.
  // This is valid for now, but be careful if threads are reaped sooner.
  while (__kmp_thread_pool != NULL__null) { // Loop thru all the thread in the pool.
    // Get the next thread from the pool.
    kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool)const_cast<kmp_info_t *>(__kmp_thread_pool);
    __kmp_thread_pool = thread->th.th_next_pool;
    // Reap it.
    KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP)if (!(thread->th.th_reap_state == 1)) { __kmp_debug_assert
("thread->th.th_reap_state == 1", "openmp/runtime/src/kmp_runtime.cpp"
, 6391); };
    thread->th.th_next_pool = NULL__null;
    thread->th.th_in_pool = FALSE0;
    __kmp_reap_thread(thread, 0);
  }
  __kmp_thread_pool_insert_pt = NULL__null;

  // Reap teams.
  while (__kmp_team_pool != NULL__null) { // Loop thru all the teams in the pool.
    // Get the next team from the pool.
    kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool)const_cast<kmp_team_t *>(__kmp_team_pool);
    __kmp_team_pool = team->t.t_next_pool;
    // Reap it.
    team->t.t_next_pool = NULL__null;
    __kmp_reap_team(team);
  }

  __kmp_reap_task_teams();

6410#if KMP_OS_UNIX1
  // Threads that are not reaped should not access any resources since they
  // are going to be deallocated soon, so the shutdown sequence should wait
  // until all threads either exit the final spin-waiting loop or begin
  // sleeping after the given blocktime.
  for (i = 0; i < __kmp_threads_capacity; i++) {
    kmp_info_t *thr = __kmp_threads[i];
    while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)(&thr->th.th_blocking)->load(std::memory_order_acquire
))
      KMP_CPU_PAUSE()__kmp_x86_pause();
  }
6420#endif

  for (i = 0; i < __kmp_threads_capacity; ++i) {
    // TBD: Add some checking...
    // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL );
  }

  /* Make sure all threadprivate destructors get run by joining with all
     worker threads before resetting this flag */
  TCW_SYNC_4(__kmp_init_common, FALSE)(__kmp_init_common) = (0);

  KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end: all workers reaped\n"
); };
  KMP_MB();

6434#if KMP_USE_MONITOR
  // See note above: One of the possible fixes for CQ138434 / CQ140126
  //
  // FIXME: push both code fragments down and CSE them?
  // push them into __kmp_cleanup() ?
  __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
  if (TCR_4(__kmp_init_monitor)(__kmp_init_monitor)) {
    __kmp_reap_monitor(&__kmp_monitor);
    TCW_4(__kmp_init_monitor, 0)(__kmp_init_monitor) = (0);
  }
  __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
  KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end: monitor reaped\n"
); };
6446#endif
} /* else !__kmp_global.t_active */
TCW_4(__kmp_init_gtid, FALSE)(__kmp_init_gtid) = (0);
KMP_MB(); /* Flush all pending memory write invalidates.  */

__kmp_cleanup();
6452#if OMPT_SUPPORT1
ompt_fini();
6454#endif
6455}

6457void __kmp_internal_end_library(int gtid_req) {
/* if we have already cleaned up, don't try again, it wouldn't be pretty */
/* this shouldn't be a race condition because __kmp_internal_end() is the
   only place to clear __kmp_serial_init */
/* we'll check this later too, after we get the lock */
// 2009-09-06: We do not set g_abort without setting g_done. This check looks
// redundant, because the next check will work in any case.
if (__kmp_global.g.g_abort) {
  KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n"))if (kmp_a_debug >= 11) { __kmp_debug_printf ("__kmp_internal_end_library: abort, exiting\n"
); };
  /* TODO abort? */
  return;
}
if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
  KA_TRACE(10, ("__kmp_internal_end_library: already finished\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: already finished\n"
); };
  return;
}

// If hidden helper team has been initialized, we need to deinit it
if (TCR_4(__kmp_init_hidden_helper)(__kmp_init_hidden_helper) &&
    !TCR_4(__kmp_hidden_helper_team_done)(__kmp_hidden_helper_team_done)) {
  TCW_SYNC_4(__kmp_hidden_helper_team_done, TRUE)(__kmp_hidden_helper_team_done) = ((!0));
  // First release the main thread to let it continue its work
  __kmp_hidden_helper_main_thread_release();
  // Wait until the hidden helper team has been destroyed
  __kmp_hidden_helper_threads_deinitz_wait();
}

KMP_MB(); /* Flush all pending memory write invalidates.  */
/* find out who we are and what we should do */
{
  int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
  KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: enter T#%d  (%d)\n"
, gtid, gtid_req); }
      10, ("__kmp_internal_end_library: enter T#%d  (%d)\n", gtid, gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: enter T#%d  (%d)\n"
, gtid, gtid_req); };
  if (gtid == KMP_GTID_SHUTDOWN(-3)) {
    KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: !__kmp_init_runtime, system "
 "already shutdown\n"); }
                  "already shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: !__kmp_init_runtime, system "
 "already shutdown\n"); };
    return;
  } else if (gtid == KMP_GTID_MONITOR(-4)) {
    KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: monitor thread, gtid not "
 "registered, or system shutdown\n"); }
                  "registered, or system shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: monitor thread, gtid not "
 "registered, or system shutdown\n"); };
    return;
  } else if (gtid == KMP_GTID_DNE(-2)) {
    KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: gtid not registered or system "
 "shutdown\n"); }
                  "shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: gtid not registered or system "
 "shutdown\n"); };
    /* we don't know who we are, but we may still shutdown the library */
  } else if (KMP_UBER_GTID(gtid)) {
    /* unregister ourselves as an uber thread.  gtid is no longer valid */
    if (__kmp_root[gtid]->r.r_active) {
      __kmp_global.g.g_abort = -1;
      TCW_SYNC_4(__kmp_global.g.g_done, TRUE)(__kmp_global.g.g_done) = ((!0));
      __kmp_unregister_library();
      KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: root still active, abort T#%d\n"
, gtid); }
               ("__kmp_internal_end_library: root still active, abort T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: root still active, abort T#%d\n"
, gtid); }
                gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: root still active, abort T#%d\n"
, gtid); };
      return;
    } else {
      __kmp_itthash_clean(__kmp_threads[gtid]);
      KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: unregistering sibling T#%d\n"
, gtid); }
          10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: unregistering sibling T#%d\n"
, gtid); }
          ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: unregistering sibling T#%d\n"
, gtid); };
      __kmp_unregister_root_current_thread(gtid);
    }
  } else {
6520/* worker threads may call this function through the atexit handler, if they
* call exit() */
6522/* For now, skip the usual subsequent processing and just dump the debug buffer.
 TODO: do a thorough shutdown instead */
6524#ifdef DUMP_DEBUG_ON_EXIT
    if (__kmp_debug_buf)
      __kmp_dump_debug_buffer();
6527#endif
    // added unregister library call here when we switch to shm linux
    // if we don't, it will leave lots of files in /dev/shm
    // cleanup shared memory file before exiting.
    __kmp_unregister_library();
    return;
  }
}
/* synchronize the termination process */
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);

/* have we already finished */
if (__kmp_global.g.g_abort) {
  KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: abort, exiting\n"
); };
  /* TODO abort? */
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}
if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}

/* We need this lock to enforce mutex between this reading of
   __kmp_threads_capacity and the writing by __kmp_register_root.
   Alternatively, we can use a counter of roots that is atomically updated by
   __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
   __kmp_internal_end_*.  */
__kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);

/* now we can safely conduct the actual termination */
__kmp_internal_end();

__kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
__kmp_release_bootstrap_lock(&__kmp_initz_lock);

KA_TRACE(10, ("__kmp_internal_end_library: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_library: exit\n"
); };

6565#ifdef DUMP_DEBUG_ON_EXIT
if (__kmp_debug_buf)
  __kmp_dump_debug_buffer();
6568#endif

6570#if KMP_OS_WINDOWS0
__kmp_close_console();
6572#endif

__kmp_fini_allocator();

6576} // __kmp_internal_end_library

6578void __kmp_internal_end_thread(int gtid_req) {
int i;

/* if we have already cleaned up, don't try again, it wouldn't be pretty */
/* this shouldn't be a race condition because __kmp_internal_end() is the
 * only place to clear __kmp_serial_init */
/* we'll check this later too, after we get the lock */
// 2009-09-06: We do not set g_abort without setting g_done. This check looks
// redundant, because the next check will work in any case.
if (__kmp_global.g.g_abort) {
  KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n"))if (kmp_a_debug >= 11) { __kmp_debug_printf ("__kmp_internal_end_thread: abort, exiting\n"
); };
  /* TODO abort? */
  return;
}
if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
  KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: already finished\n"
); };
  return;
}

// If hidden helper team has been initialized, we need to deinit it
if (TCR_4(__kmp_init_hidden_helper)(__kmp_init_hidden_helper) &&
    !TCR_4(__kmp_hidden_helper_team_done)(__kmp_hidden_helper_team_done)) {
  TCW_SYNC_4(__kmp_hidden_helper_team_done, TRUE)(__kmp_hidden_helper_team_done) = ((!0));
  // First release the main thread to let it continue its work
  __kmp_hidden_helper_main_thread_release();
  // Wait until the hidden helper team has been destroyed
  __kmp_hidden_helper_threads_deinitz_wait();
}

KMP_MB(); /* Flush all pending memory write invalidates.  */

/* find out who we are and what we should do */
{
  int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
  KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: enter T#%d  (%d)\n"
, gtid, gtid_req); }
           ("__kmp_internal_end_thread: enter T#%d  (%d)\n", gtid, gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: enter T#%d  (%d)\n"
, gtid, gtid_req); };
  if (gtid == KMP_GTID_SHUTDOWN(-3)) {
    KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: !__kmp_init_runtime, system "
 "already shutdown\n"); }
                  "already shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: !__kmp_init_runtime, system "
 "already shutdown\n"); };
    return;
  } else if (gtid == KMP_GTID_MONITOR(-4)) {
    KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: monitor thread, gtid not "
 "registered, or system shutdown\n"); }
                  "registered, or system shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: monitor thread, gtid not "
 "registered, or system shutdown\n"); };
    return;
  } else if (gtid == KMP_GTID_DNE(-2)) {
    KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system "if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: gtid not registered or system "
 "shutdown\n"); }
                  "shutdown\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: gtid not registered or system "
 "shutdown\n"); };
    return;
    /* we don't know who we are */
  } else if (KMP_UBER_GTID(gtid)) {
    /* unregister ourselves as an uber thread.  gtid is no longer valid */
    if (__kmp_root[gtid]->r.r_active) {
      __kmp_global.g.g_abort = -1;
      TCW_SYNC_4(__kmp_global.g.g_done, TRUE)(__kmp_global.g.g_done) = ((!0));
      KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: root still active, abort T#%d\n"
, gtid); }
               ("__kmp_internal_end_thread: root still active, abort T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: root still active, abort T#%d\n"
, gtid); }
                gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: root still active, abort T#%d\n"
, gtid); };
      return;
    } else {
      KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: unregistering sibling T#%d\n"
, gtid); }
                    gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: unregistering sibling T#%d\n"
, gtid); };
      __kmp_unregister_root_current_thread(gtid);
    }
  } else {
    /* just a worker thread, let's leave */
    KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread T#%d\n"
, gtid); };

    if (gtid >= 0) {
      __kmp_threads[gtid]->th.th_task_team = NULL__null;
    }

    KA_TRACE(10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n"
, gtid); }
             ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n",if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n"
, gtid); }
              gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n"
, gtid); };
    return;
  }
}
6655#if KMP_DYNAMIC_LIB1
if (__kmp_pause_status != kmp_hard_paused)
// AC: lets not shutdown the dynamic library at the exit of uber thread,
// because we will better shutdown later in the library destructor.
{
  KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: exiting T#%d\n"
, gtid_req); };
  return;
}
6663#endif
/* synchronize the termination process */
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);

/* have we already finished */
if (__kmp_global.g.g_abort) {
  KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: abort, exiting\n"
); };
  /* TODO abort? */
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}
if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done) || !__kmp_init_serial) {
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}

/* We need this lock to enforce mutex between this reading of
   __kmp_threads_capacity and the writing by __kmp_register_root.
   Alternatively, we can use a counter of roots that is atomically updated by
   __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
   __kmp_internal_end_*.  */

/* should we finish the run-time?  are all siblings done? */
__kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);

for (i = 0; i < __kmp_threads_capacity; ++i) {
  if (KMP_UBER_GTID(i)) {
    KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n"
, i); }
        10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n"
, i); }
        ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n"
, i); };
    __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
    __kmp_release_bootstrap_lock(&__kmp_initz_lock);
    return;
  }
}

/* now we can safely conduct the actual termination */

__kmp_internal_end();

__kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
__kmp_release_bootstrap_lock(&__kmp_initz_lock);

KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_internal_end_thread: exit T#%d\n"
, gtid_req); };

6708#ifdef DUMP_DEBUG_ON_EXIT
if (__kmp_debug_buf)
  __kmp_dump_debug_buffer();
6711#endif
6712} // __kmp_internal_end_thread

6714// -----------------------------------------------------------------------------
6715// Library registration stuff.

6717static long __kmp_registration_flag = 0;
6718// Random value used to indicate library initialization.
6719static char *__kmp_registration_str = NULL__null;
6720// Value to be saved in env var __KMP_REGISTERED_LIB_<pid>.

6722static inline char *__kmp_reg_status_name() {
6723/* On RHEL 3u5 if linked statically, getpid() returns different values in
 each thread. If registration and unregistration go in different threads
 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env
 env var can not be found, because the name will contain different pid. */
6727// macOS* complains about name being too long with additional getuid()
6728#if KMP_OS_UNIX1 && !KMP_OS_DARWIN0 && KMP_DYNAMIC_LIB1
return __kmp_str_format("__KMP_REGISTERED_LIB_%d_%d", (int)getpid(),
                        (int)getuid());
6731#else
return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid());
6733#endif
6734} // __kmp_reg_status_get

6736void __kmp_register_library_startup(void) {

char *name = __kmp_reg_status_name(); // Name of the environment variable.
int done = 0;
union {
  double dtime;
  long ltime;
} time;
6744#if KMP_ARCH_X860 || KMP_ARCH_X86_641
__kmp_initialize_system_tick();
6746#endif
__kmp_read_system_time(&time.dtime);
__kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
__kmp_registration_str =
    __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag,
                     __kmp_registration_flag, KMP_LIBRARY_FILE"libomp.so.5");

KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name,if (kmp_a_debug >= 50) { __kmp_debug_printf ("__kmp_register_library_startup: %s=\"%s\"\n"
, name, __kmp_registration_str); }
              __kmp_registration_str))if (kmp_a_debug >= 50) { __kmp_debug_printf ("__kmp_register_library_startup: %s=\"%s\"\n"
, name, __kmp_registration_str); };

while (!done) {

  char *value = NULL__null; // Actual value of the environment variable.

6760#if defined(KMP_USE_SHM)
  char *shm_name = __kmp_str_format("/%s", name);
  int shm_preexist = 0;
  char *data1;
  int fd1 = shm_open(shm_name, O_CREAT0100 | O_EXCL0200 | O_RDWR02, 0666);
  if ((fd1 == -1) && (errno(*__errno_location ()) == EEXIST17)) {
    // file didn't open because it already exists.
    // try opening existing file
    fd1 = shm_open(shm_name, O_RDWR02, 0666);
    if (fd1 == -1) { // file didn't open
      // error out here
      __kmp_fatal(KMP_MSG(FunctionError, "Can't open SHM")__kmp_msg_format(kmp_i18n_msg_FunctionError, "Can't open SHM"
), KMP_ERR(0)__kmp_msg_error_code(0),
                  __kmp_msg_null);
    } else {
      // able to open existing file
      shm_preexist = 1;
    }
  } else if (fd1 == -1) { // SHM didn't open; it was due to error other than
    // already exists.
    // error out here.
    __kmp_fatal(KMP_MSG(FunctionError, "Can't open SHM2")__kmp_msg_format(kmp_i18n_msg_FunctionError, "Can't open SHM2"
), KMP_ERR(errno)__kmp_msg_error_code((*__errno_location ())),
                __kmp_msg_null);
  }
  if (shm_preexist == 0) {
    // we created SHM now set size
    if (ftruncate(fd1, SHM_SIZE1024) == -1) {
      // error occured setting size;
      __kmp_fatal(KMP_MSG(FunctionError, "Can't set size of SHM")__kmp_msg_format(kmp_i18n_msg_FunctionError, "Can't set size of SHM"
),
                  KMP_ERR(errno)__kmp_msg_error_code((*__errno_location ())), __kmp_msg_null);
    }
  }
  data1 =
      (char *)mmap(0, SHM_SIZE1024, PROT_READ0x1 | PROT_WRITE0x2, MAP_SHARED0x01, fd1, 0);
  if (data1 == MAP_FAILED((void *) -1)) {
    // failed to map shared memory
    __kmp_fatal(KMP_MSG(FunctionError, "Can't map SHM")__kmp_msg_format(kmp_i18n_msg_FunctionError, "Can't map SHM"), KMP_ERR(errno)__kmp_msg_error_code((*__errno_location ())),
                __kmp_msg_null);
  }
  if (shm_preexist == 0) { // set data to SHM, set value
    KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str)strcpy(data1, __kmp_registration_str);
  }
  // Read value from either what we just wrote or existing file.
  value = __kmp_str_format("%s", data1); // read value from SHM
  munmap(data1, SHM_SIZE1024);
  close(fd1);
6805#else // Windows and unix with static library
  // Set environment variable, but do not overwrite if it is exist.
  __kmp_env_set(name, __kmp_registration_str, 0);
  // read value to see if it got set
  value = __kmp_env_get(name);
6810#endif

  if (value != NULL__null && strcmp(value, __kmp_registration_str) == 0) {
    done = 1; // Ok, environment variable set successfully, exit the loop.
  } else {
    // Oops. Write failed. Another copy of OpenMP RTL is in memory.
    // Check whether it alive or dead.
    int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead.
    char *tail = value;
    char *flag_addr_str = NULL__null;
    char *flag_val_str = NULL__null;
    char const *file_name = NULL__null;
    __kmp_str_split(tail, '-', &flag_addr_str, &tail);
    __kmp_str_split(tail, '-', &flag_val_str, &tail);
    file_name = tail;
    if (tail != NULL__null) {
      unsigned long *flag_addr = 0;
      unsigned long flag_val = 0;
      KMP_SSCANFsscanf(flag_addr_str, "%p", RCAST(void **, &flag_addr)reinterpret_cast<void **>(&flag_addr));
      KMP_SSCANFsscanf(flag_val_str, "%lx", &flag_val);
      if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) {
        // First, check whether environment-encoded address is mapped into
        // addr space.
        // If so, dereference it to see if it still has the right value.
        if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
          neighbor = 1;
        } else {
          // If not, then we know the other copy of the library is no longer
          // running.
          neighbor = 2;
        }
      }
    }
    switch (neighbor) {
    case 0: // Cannot parse environment variable -- neighbor status unknown.
      // Assume it is the incompatible format of future version of the
      // library. Assume the other library is alive.
      // WARN( ... ); // TODO: Issue a warning.
      file_name = "unknown library";
      KMP_FALLTHROUGH()[[fallthrough]];
    // Attention! Falling to the next case. That's intentional.
    case 1: { // Neighbor is alive.
      // Check it is allowed.
      char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK");
      if (!__kmp_str_match_true(duplicate_ok)) {
        // That's not allowed. Issue fatal error.
        __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name)__kmp_msg_format(kmp_i18n_msg_DuplicateLibrary, "libomp.so.5"
, file_name),
                    KMP_HNT(DuplicateLibrary)__kmp_msg_format(kmp_i18n_hnt_DuplicateLibrary), __kmp_msg_null);
      }
      KMP_INTERNAL_FREE(duplicate_ok)free(duplicate_ok);
      __kmp_duplicate_library_ok = 1;
      done = 1; // Exit the loop.
    } break;
    case 2: { // Neighbor is dead.

6865#if defined(KMP_USE_SHM)
      // close shared memory.
      shm_unlink(shm_name); // this removes file in /dev/shm
6868#else
      // Clear the variable and try to register library again.
      __kmp_env_unset(name);
6871#endif
    } break;
    default: {
      KMP_DEBUG_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_runtime.cpp"
, 6874); };
    } break;
    }
  }
  KMP_INTERNAL_FREE((void *)value)free((void *)value);
6879#if defined(KMP_USE_SHM)
  KMP_INTERNAL_FREE((void *)shm_name)free((void *)shm_name);
6881#endif
} // while
KMP_INTERNAL_FREE((void *)name)free((void *)name);

6885} // func __kmp_register_library_startup

6887void __kmp_unregister_library(void) {

char *name = __kmp_reg_status_name();
char *value = NULL__null;

6892#if defined(KMP_USE_SHM)
char *shm_name = __kmp_str_format("/%s", name);
int fd1 = shm_open(shm_name, O_RDONLY00, 0666);
if (fd1 == -1) {
  // file did not open. return.
  return;
}
char *data1 = (char *)mmap(0, SHM_SIZE1024, PROT_READ0x1, MAP_SHARED0x01, fd1, 0);
if (data1 != MAP_FAILED((void *) -1)) {
  value = __kmp_str_format("%s", data1); // read value from SHM
  munmap(data1, SHM_SIZE1024);
}
close(fd1);
6905#else
value = __kmp_env_get(name);
6907#endif

KMP_DEBUG_ASSERT(__kmp_registration_flag != 0)if (!(__kmp_registration_flag != 0)) { __kmp_debug_assert("__kmp_registration_flag != 0"
, "openmp/runtime/src/kmp_runtime.cpp", 6909); };
KMP_DEBUG_ASSERT(__kmp_registration_str != NULL)if (!(__kmp_registration_str != __null)) { __kmp_debug_assert
("__kmp_registration_str != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 6910); };
if (value != NULL__null && strcmp(value, __kmp_registration_str) == 0) {
6912//  Ok, this is our variable. Delete it.
6913#if defined(KMP_USE_SHM)
  shm_unlink(shm_name); // this removes file in /dev/shm
6915#else
  __kmp_env_unset(name);
6917#endif
}

6920#if defined(KMP_USE_SHM)
KMP_INTERNAL_FREE(shm_name)free(shm_name);
6922#endif

KMP_INTERNAL_FREE(__kmp_registration_str)free(__kmp_registration_str);
KMP_INTERNAL_FREE(value)free(value);
KMP_INTERNAL_FREE(name)free(name);

__kmp_registration_flag = 0;
__kmp_registration_str = NULL__null;

6931} // __kmp_unregister_library

6933// End of Library registration stuff.
6934// -----------------------------------------------------------------------------

6936#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))

6938static void __kmp_check_mic_type() {
kmp_cpuid_t cpuid_state = {0};
kmp_cpuid_t *cs_p = &cpuid_state;
__kmp_x86_cpuid(1, 0, cs_p);
// We don't support mic1 at the moment
if ((cs_p->eax & 0xff0) == 0xB10) {
  __kmp_mic_type = mic2;
} else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
  __kmp_mic_type = mic3;
} else {
  __kmp_mic_type = non_mic;
}
6950}

6952#endif /* KMP_MIC_SUPPORTED */

6954#if KMP_HAVE_UMWAIT((0 || 1) && (1 || 0) && !0)
6955static void __kmp_user_level_mwait_init() {
struct kmp_cpuid buf;
__kmp_x86_cpuid(7, 0, &buf);
__kmp_waitpkg_enabled = ((buf.ecx >> 5) & 1);
__kmp_umwait_enabled = __kmp_waitpkg_enabled && __kmp_user_level_mwait;
__kmp_tpause_enabled = __kmp_waitpkg_enabled && (__kmp_tpause_state > 0);
KF_TRACE(30, ("__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n",if (kmp_f_debug >= 30) { __kmp_debug_printf ("__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n"
, __kmp_umwait_enabled); }
              __kmp_umwait_enabled))if (kmp_f_debug >= 30) { __kmp_debug_printf ("__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n"
, __kmp_umwait_enabled); };
6963}
6964#elif KMP_HAVE_MWAIT((0 || 1) && (1 || 0) && !0)
6965#ifndef AT_INTELPHIUSERMWAIT
6966// Spurious, non-existent value that should always fail to return anything.
6967// Will be replaced with the correct value when we know that.
6968#define AT_INTELPHIUSERMWAIT 10000
6969#endif
6970// getauxval() function is available in RHEL7 and SLES12. If a system with an
6971// earlier OS is used to build the RTL, we'll use the following internal
6972// function when the entry is not found.
6973unsigned long getauxval(unsigned long) KMP_WEAK_ATTRIBUTE_EXTERNAL;
6974unsigned long getauxval(unsigned long) { return 0; }

6976static void __kmp_user_level_mwait_init() {
// When getauxval() and correct value of AT_INTELPHIUSERMWAIT are available
// use them to find if the user-level mwait is enabled. Otherwise, forcibly
// set __kmp_mwait_enabled=TRUE on Intel MIC if the environment variable
// KMP_USER_LEVEL_MWAIT was set to TRUE.
if (__kmp_mic_type == mic3) {
  unsigned long res = getauxval(AT_INTELPHIUSERMWAIT);
  if ((res & 0x1) || __kmp_user_level_mwait) {
    __kmp_mwait_enabled = TRUE(!0);
    if (__kmp_user_level_mwait) {
      KMP_INFORM(EnvMwaitWarn)__kmp_msg(kmp_ms_inform, __kmp_msg_format(kmp_i18n_msg_EnvMwaitWarn
), __kmp_msg_null);
    }
  } else {
    __kmp_mwait_enabled = FALSE0;
  }
}
KF_TRACE(30, ("__kmp_user_level_mwait_init: __kmp_mic_type = %d, "if (kmp_f_debug >= 30) { __kmp_debug_printf ("__kmp_user_level_mwait_init: __kmp_mic_type = %d, "
 "__kmp_mwait_enabled = %d\n", __kmp_mic_type, __kmp_mwait_enabled
); }
              "__kmp_mwait_enabled = %d\n",if (kmp_f_debug >= 30) { __kmp_debug_printf ("__kmp_user_level_mwait_init: __kmp_mic_type = %d, "
 "__kmp_mwait_enabled = %d\n", __kmp_mic_type, __kmp_mwait_enabled
); }
              __kmp_mic_type, __kmp_mwait_enabled))if (kmp_f_debug >= 30) { __kmp_debug_printf ("__kmp_user_level_mwait_init: __kmp_mic_type = %d, "
 "__kmp_mwait_enabled = %d\n", __kmp_mic_type, __kmp_mwait_enabled
); };
6995}
6996#endif /* KMP_HAVE_UMWAIT */

6998static void __kmp_do_serial_initialize(void) {
int i, gtid;
size_t size;

KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize: enter\n"
); };

KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4)if (!(sizeof(kmp_int32) == 4)) { __kmp_debug_assert("sizeof(kmp_int32) == 4"
, "openmp/runtime/src/kmp_runtime.cpp", 7004); };
KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4)if (!(sizeof(kmp_uint32) == 4)) { __kmp_debug_assert("sizeof(kmp_uint32) == 4"
, "openmp/runtime/src/kmp_runtime.cpp", 7005); };
KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8)if (!(sizeof(kmp_int64) == 8)) { __kmp_debug_assert("sizeof(kmp_int64) == 8"
, "openmp/runtime/src/kmp_runtime.cpp", 7006); };
KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8)if (!(sizeof(kmp_uint64) == 8)) { __kmp_debug_assert("sizeof(kmp_uint64) == 8"
, "openmp/runtime/src/kmp_runtime.cpp", 7007); };
KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *))if (!(sizeof(kmp_intptr_t) == sizeof(void *))) { __kmp_debug_assert
("sizeof(kmp_intptr_t) == sizeof(void *)", "openmp/runtime/src/kmp_runtime.cpp"
, 7008); };

7010#if OMPT_SUPPORT1
ompt_pre_init();
7012#endif
7013#if OMPD_SUPPORT1
__kmp_env_dump();
ompd_init();
7016#endif

__kmp_validate_locks();

/* Initialize internal memory allocator */
__kmp_init_allocator();

/* Register the library startup via an environment variable or via mapped
   shared memory file and check to see whether another copy of the library is
   already registered. Since forked child process is often terminated, we
   postpone the registration till middle initialization in the child */
if (__kmp_need_register_serial)
  __kmp_register_library_startup();

/* TODO reinitialization of library */
if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
  KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize: reinitialization of library\n"
); };
}

__kmp_global.g.g_abort = 0;
TCW_SYNC_4(__kmp_global.g.g_done, FALSE)(__kmp_global.g.g_done) = (0);

7038/* initialize the locks */
7039#if KMP_USE_ADAPTIVE_LOCKS(0 || 1) && !0
7040#if KMP_DEBUG_ADAPTIVE_LOCKS0
__kmp_init_speculative_stats();
7042#endif
7043#endif
7044#if KMP_STATS_ENABLED0
__kmp_stats_init();
7046#endif
__kmp_init_lock(&__kmp_global_lock);
__kmp_init_queuing_lock(&__kmp_dispatch_lock);
__kmp_init_lock(&__kmp_debug_lock);
__kmp_init_atomic_lock(&__kmp_atomic_lock);
__kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
__kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
__kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
__kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
__kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
__kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
__kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
__kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
__kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
__kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
__kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
__kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
__kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
__kmp_init_bootstrap_lock(&__kmp_exit_lock);
7065#if KMP_USE_MONITOR
__kmp_init_bootstrap_lock(&__kmp_monitor_lock);
7067#endif
__kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);

/* conduct initialization and initial setup of configuration */

__kmp_runtime_initialize();

7074#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))
__kmp_check_mic_type();
7076#endif

7078// Some global variable initialization moved here from kmp_env_initialize()
7079#ifdef KMP_DEBUG1
kmp_diag = 0;
7081#endif
__kmp_abort_delay = 0;

// From __kmp_init_dflt_team_nth()
/* assume the entire machine will be used */
__kmp_dflt_team_nth_ub = __kmp_xproc;
if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH1) {
  __kmp_dflt_team_nth_ub = KMP_MIN_NTH1;
}
if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
  __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
}
__kmp_max_nth = __kmp_sys_max_nth;
__kmp_cg_max_nth = __kmp_sys_max_nth;
__kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default
if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
  __kmp_teams_max_nth = __kmp_sys_max_nth;
}

// Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME"
// part
__kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME(__kmp_is_hybrid_cpu() ? (0) : (200));
7103#if KMP_USE_MONITOR
__kmp_monitor_wakeups =
    KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
__kmp_bt_intervals =
    KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
7108#endif
// From "KMP_LIBRARY" part of __kmp_env_initialize()
__kmp_library = library_throughput;
// From KMP_SCHEDULE initialization
__kmp_static = kmp_sch_static_balanced;
7113// AC: do not use analytical here, because it is non-monotonous
7114//__kmp_guided = kmp_sch_guided_iterative_chunked;
7115//__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no
7116// need to repeat assignment
7117// Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch
7118// bit control and barrier method control parts
7119#if KMP_FAST_REDUCTION_BARRIER1
7120#define kmp_reduction_barrier_gather_bb ((int)1)
7121#define kmp_reduction_barrier_release_bb ((int)1)
7122#define kmp_reduction_barrier_gather_pat __kmp_barrier_gather_pat_dflt
7123#define kmp_reduction_barrier_release_pat __kmp_barrier_release_pat_dflt
7124#endif // KMP_FAST_REDUCTION_BARRIER
for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
  __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
  __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
  __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
  __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
7130#if KMP_FAST_REDUCTION_BARRIER1
  if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only (
    // lin_64 ): hyper,1
    __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
    __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
    __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
    __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
  }
7138#endif // KMP_FAST_REDUCTION_BARRIER
}
7140#if KMP_FAST_REDUCTION_BARRIER1
7141#undef kmp_reduction_barrier_release_pat
7142#undef kmp_reduction_barrier_gather_pat
7143#undef kmp_reduction_barrier_release_bb
7144#undef kmp_reduction_barrier_gather_bb
7145#endif // KMP_FAST_REDUCTION_BARRIER
7146#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))
if (__kmp_mic_type == mic2) { // KNC
  // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC
  __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather
  __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
      1; // forkjoin release
  __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
  __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
}
7155#if KMP_FAST_REDUCTION_BARRIER1
if (__kmp_mic_type == mic2) { // KNC
  __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
  __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
}
7160#endif // KMP_FAST_REDUCTION_BARRIER
7161#endif // KMP_MIC_SUPPORTED

7163// From KMP_CHECKS initialization
7164#ifdef KMP_DEBUG1
__kmp_env_checks = TRUE(!0); /* development versions have the extra checks */
7166#else
__kmp_env_checks = FALSE0; /* port versions do not have the extra checks */
7168#endif

// From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization
__kmp_foreign_tp = TRUE(!0);

__kmp_global.g.g_dynamic = FALSE0;
__kmp_global.g.g_dynamic_mode = dynamic_default;

__kmp_init_nesting_mode();

__kmp_env_initialize(NULL__null);

7180#if KMP_HAVE_MWAIT((0 || 1) && (1 || 0) && !0) || KMP_HAVE_UMWAIT((0 || 1) && (1 || 0) && !0)
__kmp_user_level_mwait_init();
7182#endif
7183// Print all messages in message catalog for testing purposes.
7184#ifdef KMP_DEBUG1
char const *val = __kmp_env_get("KMP_DUMP_CATALOG");
if (__kmp_str_match_true(val)) {
  kmp_str_buf_t buffer;
  __kmp_str_buf_init(&buffer){ (&buffer)->str = (&buffer)->bulk; (&buffer
)->size = sizeof((&buffer)->bulk); (&buffer)->
used = 0; (&buffer)->bulk[0] = 0; };
  __kmp_i18n_dump_catalog(&buffer);
  __kmp_printf("%s", buffer.str);
  __kmp_str_buf_free(&buffer);
}
__kmp_env_free(&val);
7194#endif

__kmp_threads_capacity =
    __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
// Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part
__kmp_tp_capacity = __kmp_default_tp_capacity(
    __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);

// If the library is shut down properly, both pools must be NULL. Just in
// case, set them to NULL -- some memory may leak, but subsequent code will
// work even if pools are not freed.
KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL)if (!(__kmp_thread_pool == __null)) { __kmp_debug_assert("__kmp_thread_pool == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 7205); };
KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL)if (!(__kmp_thread_pool_insert_pt == __null)) { __kmp_debug_assert
("__kmp_thread_pool_insert_pt == __null", "openmp/runtime/src/kmp_runtime.cpp"
, 7206); };
KMP_DEBUG_ASSERT(__kmp_team_pool == NULL)if (!(__kmp_team_pool == __null)) { __kmp_debug_assert("__kmp_team_pool == __null"
, "openmp/runtime/src/kmp_runtime.cpp", 7207); };
__kmp_thread_pool = NULL__null;
__kmp_thread_pool_insert_pt = NULL__null;
__kmp_team_pool = NULL__null;

/* Allocate all of the variable sized records */
/* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are
 * expandable */
/* Since allocation is cache-aligned, just add extra padding at the end */
size =
    (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity +
    CACHE_LINE64;
__kmp_threads = (kmp_info_t **)__kmp_allocate(size)___kmp_allocate((size), "openmp/runtime/src/kmp_runtime.cpp",
 7219);
__kmp_root = (kmp_root_t **)((char *)__kmp_threads +
                             sizeof(kmp_info_t *) * __kmp_threads_capacity);

/* init thread counts */
KMP_DEBUG_ASSERT(__kmp_all_nth ==if (!(__kmp_all_nth == 0)) { __kmp_debug_assert("__kmp_all_nth == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 7225); }
                 0)if (!(__kmp_all_nth == 0)) { __kmp_debug_assert("__kmp_all_nth == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 7225); }; // Asserts fail if the library is reinitializing and
KMP_DEBUG_ASSERT(__kmp_nth == 0)if (!(__kmp_nth == 0)) { __kmp_debug_assert("__kmp_nth == 0",
 "openmp/runtime/src/kmp_runtime.cpp", 7226); }; // something was wrong in termination.
__kmp_all_nth = 0;
__kmp_nth = 0;

/* setup the uber master thread and hierarchy */
gtid = __kmp_register_root(TRUE(!0));
KA_TRACE(10, ("__kmp_do_serial_initialize  T#%d\n", gtid))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize  T#%d\n"
, gtid); };
KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 7233); };
KMP_ASSERT(KMP_INITIAL_GTID(gtid))if (!((0 == (gtid)))) { __kmp_debug_assert("KMP_INITIAL_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 7234); };

KMP_MB(); /* Flush all pending memory write invalidates.  */

__kmp_common_initialize();

7240#if KMP_OS_UNIX1
/* invoke the child fork handler */
__kmp_register_atfork();
7243#endif

7245#if !KMP_DYNAMIC_LIB1
{
  /* Invoke the exit handler when the program finishes, only for static
     library. For dynamic library, we already have _fini and DllMain. */
  int rc = atexit(__kmp_internal_end_atexit);
  if (rc != 0) {
    __kmp_fatal(KMP_MSG(FunctionError, "atexit()")__kmp_msg_format(kmp_i18n_msg_FunctionError, "atexit()"), KMP_ERR(rc)__kmp_msg_error_code(rc),
                __kmp_msg_null);
  }
}
7255#endif

7257#if KMP_HANDLE_SIGNALS(1 || 0)
7258#if KMP_OS_UNIX1
/* NOTE: make sure that this is called before the user installs their own
   signal handlers so that the user handlers are called first. this way they
   can return false, not call our handler, avoid terminating the library, and
   continue execution where they left off. */
__kmp_install_signals(FALSE0);
7264#endif /* KMP_OS_UNIX */
7265#if KMP_OS_WINDOWS0
__kmp_install_signals(TRUE(!0));
7267#endif /* KMP_OS_WINDOWS */
7268#endif

/* we have finished the serial initialization */
__kmp_init_counter++;

__kmp_init_serial = TRUE(!0);

if (__kmp_settings) {
  __kmp_env_print();
}

if (__kmp_display_env || __kmp_display_env_verbose) {
  __kmp_env_print_2();
}

7283#if OMPT_SUPPORT1
ompt_post_init();
7285#endif

KMP_MB();

KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_serial_initialize: exit\n"
); };
7290}

7292void __kmp_serial_initialize(void) {
if (__kmp_init_serial) {
  return;
}
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
if (__kmp_init_serial) {
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}
__kmp_do_serial_initialize();
__kmp_release_bootstrap_lock(&__kmp_initz_lock);
7303}

7305static void __kmp_do_middle_initialize(void) {
int i, j;
int prev_dflt_team_nth;

if (!__kmp_init_serial) {
  __kmp_do_serial_initialize();
}

KA_TRACE(10, ("__kmp_middle_initialize: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_middle_initialize: enter\n"
); };

if (UNLIKELY(!__kmp_need_register_serial)__builtin_expect(!!(!__kmp_need_register_serial), 0)) {
  // We are in a forked child process. The registration was skipped during
  // serial initialization in __kmp_atfork_child handler. Do it here.
  __kmp_register_library_startup();
}

// Save the previous value for the __kmp_dflt_team_nth so that
// we can avoid some reinitialization if it hasn't changed.
prev_dflt_team_nth = __kmp_dflt_team_nth;

7325#if KMP_AFFINITY_SUPPORTED1
// __kmp_affinity_initialize() will try to set __kmp_ncores to the
// number of cores on the machine.
__kmp_affinity_initialize(__kmp_affinity);

7330#endif /* KMP_AFFINITY_SUPPORTED */

KMP_ASSERT(__kmp_xproc > 0)if (!(__kmp_xproc > 0)) { __kmp_debug_assert("__kmp_xproc > 0"
, "openmp/runtime/src/kmp_runtime.cpp", 7332); };
if (__kmp_avail_proc == 0) {
  __kmp_avail_proc = __kmp_xproc;
}

// If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3),
// correct them now
j = 0;
while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
  __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
      __kmp_avail_proc;
  j++;
}

if (__kmp_dflt_team_nth == 0) {
7347#ifdef KMP_DFLT_NTH_CORES
  // Default #threads = #cores
  __kmp_dflt_team_nth = __kmp_ncores;
  KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
 "__kmp_ncores (%d)\n", __kmp_dflt_team_nth); }
                "__kmp_ncores (%d)\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
 "__kmp_ncores (%d)\n", __kmp_dflt_team_nth); }
                __kmp_dflt_team_nth))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
 "__kmp_ncores (%d)\n", __kmp_dflt_team_nth); };
7353#else
  // Default #threads = #available OS procs
  __kmp_dflt_team_nth = __kmp_avail_proc;
  KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
 "__kmp_avail_proc(%d)\n", __kmp_dflt_team_nth); }
                "__kmp_avail_proc(%d)\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
 "__kmp_avail_proc(%d)\n", __kmp_dflt_team_nth); }
                __kmp_dflt_team_nth))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
 "__kmp_avail_proc(%d)\n", __kmp_dflt_team_nth); };
7359#endif /* KMP_DFLT_NTH_CORES */
}

if (__kmp_dflt_team_nth < KMP_MIN_NTH1) {
  __kmp_dflt_team_nth = KMP_MIN_NTH1;
}
if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
  __kmp_dflt_team_nth = __kmp_sys_max_nth;
}

if (__kmp_nesting_mode > 0)
  __kmp_set_nesting_mode_threads();

// There's no harm in continuing if the following check fails,
// but it indicates an error in the previous logic.
KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub)if (!(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub)) { __kmp_debug_assert
("__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub", "openmp/runtime/src/kmp_runtime.cpp"
, 7374); };

if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
  // Run through the __kmp_threads array and set the num threads icv for each
  // root thread that is currently registered with the RTL (which has not
  // already explicitly set its nthreads-var with a call to
  // omp_set_num_threads()).
  for (i = 0; i < __kmp_threads_capacity; i++) {
    kmp_info_t *thread = __kmp_threads[i];
    if (thread == NULL__null)
      continue;
    if (thread->th.th_current_task->td_icvs.nproc != 0)
      continue;

    set__nproc(__kmp_threads[i], __kmp_dflt_team_nth)(((__kmp_threads[i])->th.th_current_task->td_icvs.nproc
) = (__kmp_dflt_team_nth));
  }
}
KA_TRACE(if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); }
    20,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); }
    ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); }
     __kmp_dflt_team_nth))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n"
, __kmp_dflt_team_nth); };

7396#ifdef KMP_ADJUST_BLOCKTIME1
/* Adjust blocktime to zero if necessary  now that __kmp_avail_proc is set */
if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
  KMP_DEBUG_ASSERT(__kmp_avail_proc > 0)if (!(__kmp_avail_proc > 0)) { __kmp_debug_assert("__kmp_avail_proc > 0"
, "openmp/runtime/src/kmp_runtime.cpp", 7399); };
  if (__kmp_nth > __kmp_avail_proc) {
    __kmp_zero_bt = TRUE(!0);
  }
}
7404#endif /* KMP_ADJUST_BLOCKTIME */

/* we have finished middle initialization */
TCW_SYNC_4(__kmp_init_middle, TRUE)(__kmp_init_middle) = ((!0));

KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_do_middle_initialize: exit\n"
); };
7410}

7412void __kmp_middle_initialize(void) {
if (__kmp_init_middle) {
  return;
}
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
if (__kmp_init_middle) {
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}
__kmp_do_middle_initialize();
__kmp_release_bootstrap_lock(&__kmp_initz_lock);
7423}

7425void __kmp_parallel_initialize(void) {
int gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg(); // this might be a new root

/* synchronize parallel initialization (for sibling) */
if (TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
  return;
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
if (TCR_4(__kmp_init_parallel)(__kmp_init_parallel)) {
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}

/* TODO reinitialization after we have already shut down */
if (TCR_4(__kmp_global.g.g_done)(__kmp_global.g.g_done)) {
  KA_TRACE(if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: attempt to init while shutting down\n"
); }
      10,if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: attempt to init while shutting down\n"
); }
      ("__kmp_parallel_initialize: attempt to init while shutting down\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: attempt to init while shutting down\n"
); };
  __kmp_infinite_loop();
}

/* jc: The lock __kmp_initz_lock is already held, so calling
   __kmp_serial_initialize would cause a deadlock.  So we call
   __kmp_do_serial_initialize directly. */
if (!__kmp_init_middle) {
  __kmp_do_middle_initialize();
}
__kmp_assign_root_init_mask();
__kmp_resume_if_hard_paused();

/* begin initialization */
KA_TRACE(10, ("__kmp_parallel_initialize: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: enter\n"
); };
KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 7456); };

7458#if KMP_ARCH_X860 || KMP_ARCH_X86_641
// Save the FP control regs.
// Worker threads will set theirs to these values at thread startup.
__kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
__kmp_store_mxcsr(&__kmp_init_mxcsr);
__kmp_init_mxcsr &= KMP_X86_MXCSR_MASK0xffffffc0;
7464#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */

7466#if KMP_OS_UNIX1
7467#if KMP_HANDLE_SIGNALS(1 || 0)
/*  must be after __kmp_serial_initialize  */
__kmp_install_signals(TRUE(!0));
7470#endif
7471#endif

__kmp_suspend_initialize();

7475#if defined(USE_LOAD_BALANCE1)
if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
  __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
}
7479#else
if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
  __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
}
7483#endif

if (__kmp_version) {
  __kmp_print_version_2();
}

/* we have finished parallel initialization */
TCW_SYNC_4(__kmp_init_parallel, TRUE)(__kmp_init_parallel) = ((!0));

KMP_MB();
KA_TRACE(10, ("__kmp_parallel_initialize: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_parallel_initialize: exit\n"
); };

__kmp_release_bootstrap_lock(&__kmp_initz_lock);
7496}

7498void __kmp_hidden_helper_initialize() {
if (TCR_4(__kmp_init_hidden_helper)(__kmp_init_hidden_helper))
  return;

// __kmp_parallel_initialize is required before we initialize hidden helper
if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel))
  __kmp_parallel_initialize();

// Double check. Note that this double check should not be placed before
// __kmp_parallel_initialize as it will cause dead lock.
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
if (TCR_4(__kmp_init_hidden_helper)(__kmp_init_hidden_helper)) {
  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
  return;
}

7514#if KMP_AFFINITY_SUPPORTED1
// Initialize hidden helper affinity settings.
// The above __kmp_parallel_initialize() will initialize
// regular affinity (and topology) if not already done.
if (!__kmp_hh_affinity.flags.initialized)
  __kmp_affinity_initialize(__kmp_hh_affinity);
7520#endif

// Set the count of hidden helper tasks to be executed to zero
KMP_ATOMIC_ST_REL(&__kmp_unexecuted_hidden_helper_tasks, 0)(&__kmp_unexecuted_hidden_helper_tasks)->store(0, std::
memory_order_release);

// Set the global variable indicating that we're initializing hidden helper
// team/threads
TCW_SYNC_4(__kmp_init_hidden_helper_threads, TRUE)(__kmp_init_hidden_helper_threads) = ((!0));

// Platform independent initialization
__kmp_do_initialize_hidden_helper_threads();

// Wait here for the finish of initialization of hidden helper teams
__kmp_hidden_helper_threads_initz_wait();

// We have finished hidden helper initialization
TCW_SYNC_4(__kmp_init_hidden_helper, TRUE)(__kmp_init_hidden_helper) = ((!0));

__kmp_release_bootstrap_lock(&__kmp_initz_lock);
7539}

7541/* ------------------------------------------------------------------------ */

7543void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
                                 kmp_team_t *team) {
kmp_disp_t *dispatch;

KMP_MB();

/* none of the threads have encountered any constructs, yet. */
this_thr->th.th_local.this_construct = 0;
7551#if KMP_CACHE_MANAGE
KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
7553#endif /* KMP_CACHE_MANAGE */
dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch)((void *)(this_thr->th.th_dispatch));
KMP_DEBUG_ASSERT(dispatch)if (!(dispatch)) { __kmp_debug_assert("dispatch", "openmp/runtime/src/kmp_runtime.cpp"
, 7555); };
KMP_DEBUG_ASSERT(team->t.t_dispatch)if (!(team->t.t_dispatch)) { __kmp_debug_assert("team->t.t_dispatch"
, "openmp/runtime/src/kmp_runtime.cpp", 7556); };
// KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[
// this_thr->th.th_info.ds.ds_tid ] );

dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */
dispatch->th_doacross_buf_idx = 0; // reset doacross dispatch buffer counter
if (__kmp_env_consistency_check)
  __kmp_push_parallel(gtid, team->t.t_ident);

KMP_MB(); /* Flush all pending memory write invalidates.  */
7566}

7568void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
                                kmp_team_t *team) {
if (__kmp_env_consistency_check)
  __kmp_pop_parallel(gtid, team->t.t_ident);

__kmp_finish_implicit_task(this_thr);
7574}

7576int __kmp_invoke_task_func(int gtid) {
int rc;
int tid = __kmp_tid_from_gtid(gtid);
kmp_info_t *this_thr = __kmp_threads[gtid];
kmp_team_t *team = this_thr->th.th_team;

__kmp_run_before_invoked_task(gtid, tid, this_thr, team);
7583#if USE_ITT_BUILD1
if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
  // inform ittnotify about entering user's code
  if (team->t.t_stack_id != NULL__null) {
    __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id);
  } else {
    KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL)if (!(team->t.t_parent->t.t_stack_id != __null)) { __kmp_debug_assert
("team->t.t_parent->t.t_stack_id != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 7589); };
    __kmp_itt_stack_callee_enter(
        (__itt_caller)team->t.t_parent->t.t_stack_id);
  }
}
7594#endif /* USE_ITT_BUILD */
7595#if INCLUDE_SSC_MARKS(1 && 1)
SSC_MARK_INVOKING()__asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
 ::"i"(0xd695) : "%ebx");
7597#endif

7599#if OMPT_SUPPORT1
void *dummy;
void **exit_frame_p;
ompt_data_t *my_task_data;
ompt_data_t *my_parallel_data;
int ompt_team_size;

if (ompt_enabled.enabled) {
  exit_frame_p = &(team->t.t_implicit_task_taskdata[tid]
                       .ompt_task_info.frame.exit_frame.ptr);
} else {
  exit_frame_p = &dummy;
}

my_task_data =
    &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
my_parallel_data = &(team->t.ompt_team_info.parallel_data);
if (ompt_enabled.ompt_callback_implicit_task) {
  ompt_team_size = team->t.t_nproc;
  ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
      ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
      __kmp_tid_from_gtid(gtid), ompt_task_implicit);
  OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->thread_num = __kmp_tid_from_gtid(gtid);
}
7623#endif

7625#if KMP_STATS_ENABLED0
stats_state_e previous_state = KMP_GET_THREAD_STATE()((void)0);
if (previous_state == stats_state_e::TEAMS_REGION) {
  KMP_PUSH_PARTITIONED_TIMER(OMP_teams)((void)0);
} else {
  KMP_PUSH_PARTITIONED_TIMER(OMP_parallel)((void)0);
}
KMP_SET_THREAD_STATE(IMPLICIT_TASK)((void)0);
7633#endif

rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn)((void *)(team->t.t_pkfn)), gtid,
                            tid, (int)team->t.t_argc, (void **)team->t.t_argv
7637#if OMPT_SUPPORT1
                            ,
                            exit_frame_p
7640#endif
);
7642#if OMPT_SUPPORT1
*exit_frame_p = NULL__null;
this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team;
7645#endif

7647#if KMP_STATS_ENABLED0
if (previous_state == stats_state_e::TEAMS_REGION) {
  KMP_SET_THREAD_STATE(previous_state)((void)0);
}
KMP_POP_PARTITIONED_TIMER()((void)0);
7652#endif

7654#if USE_ITT_BUILD1
if (__itt_stack_caller_create_ptr__kmp_itt_stack_caller_create_ptr__3_0) {
  // inform ittnotify about leaving user's code
  if (team->t.t_stack_id != NULL__null) {
    __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id);
  } else {
    KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL)if (!(team->t.t_parent->t.t_stack_id != __null)) { __kmp_debug_assert
("team->t.t_parent->t.t_stack_id != __null", "openmp/runtime/src/kmp_runtime.cpp"
, 7660); };
    __kmp_itt_stack_callee_leave(
        (__itt_caller)team->t.t_parent->t.t_stack_id);
  }
}
7665#endif /* USE_ITT_BUILD */
__kmp_run_after_invoked_task(gtid, tid, this_thr, team);

return rc;
7669}

7671void __kmp_teams_master(int gtid) {
// This routine is called by all primary threads in teams construct
kmp_info_t *thr = __kmp_threads[gtid];
kmp_team_t *team = thr->th.th_team;
ident_t *loc = team->t.t_ident;
thr->th.th_set_nproc = thr->th.th_teams_size.nth;
KMP_DEBUG_ASSERT(thr->th.th_teams_microtask)if (!(thr->th.th_teams_microtask)) { __kmp_debug_assert("thr->th.th_teams_microtask"
, "openmp/runtime/src/kmp_runtime.cpp", 7677); };
KMP_DEBUG_ASSERT(thr->th.th_set_nproc)if (!(thr->th.th_set_nproc)) { __kmp_debug_assert("thr->th.th_set_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 7678); };
KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n"
, gtid, __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask
); }
              __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n"
, gtid, __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask
); };

// This thread is a new CG root.  Set up the proper variables.
kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t))___kmp_allocate((sizeof(kmp_cg_root_t)), "openmp/runtime/src/kmp_runtime.cpp"
, 7683);
tmp->cg_root = thr; // Make thr the CG root
// Init to thread limit stored when league primary threads were forked
tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;
tmp->cg_nthreads = 1; // Init counter to one active thread, this one
KA_TRACE(100, ("__kmp_teams_master: Thread %p created node %p and init"if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_teams_master: Thread %p created node %p and init"
 " cg_nthreads to 1\n", thr, tmp); }
               " cg_nthreads to 1\n",if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_teams_master: Thread %p created node %p and init"
 " cg_nthreads to 1\n", thr, tmp); }
               thr, tmp))if (kmp_a_debug >= 100) { __kmp_debug_printf ("__kmp_teams_master: Thread %p created node %p and init"
 " cg_nthreads to 1\n", thr, tmp); };
tmp->up = thr->th.th_cg_roots;
thr->th.th_cg_roots = tmp;

7694// Launch league of teams now, but not let workers execute
7695// (they hang on fork barrier until next parallel)
7696#if INCLUDE_SSC_MARKS(1 && 1)
SSC_MARK_FORKING()__asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
 ::"i"(0xd693) : "%ebx");
7698#endif
__kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
                (microtask_t)thr->th.th_teams_microtask, // "wrapped" task
                VOLATILE_CAST(launch_t)(launch_t) __kmp_invoke_task_func, NULL__null);
7702#if INCLUDE_SSC_MARKS(1 && 1)
SSC_MARK_JOINING()__asm__ __volatile__("movl %0, %%ebx; .byte 0x64, 0x67, 0x90 "
 ::"i"(0xd694) : "%ebx");
7704#endif
// If the team size was reduced from the limit, set it to the new size
if (thr->th.th_team_nproc < thr->th.th_teams_size.nth)
  thr->th.th_teams_size.nth = thr->th.th_team_nproc;
// AC: last parameter "1" eliminates join barrier which won't work because
// worker threads are in a fork barrier waiting for more parallel regions
__kmp_join_call(loc, gtid
7711#if OMPT_SUPPORT1
                ,
                fork_context_intel
7714#endif
                ,
                1);
7717}

7719int __kmp_invoke_teams_master(int gtid) {
kmp_info_t *this_thr = __kmp_threads[gtid];
kmp_team_t *team = this_thr->th.th_team;
7722#if KMP_DEBUG1
if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
  KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==if (!((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn
 == (void *)__kmp_teams_master)) { __kmp_debug_assert("(void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == (void *)__kmp_teams_master"
, "openmp/runtime/src/kmp_runtime.cpp", 7725); }
                   (void *)__kmp_teams_master)if (!((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn
 == (void *)__kmp_teams_master)) { __kmp_debug_assert("(void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == (void *)__kmp_teams_master"
, "openmp/runtime/src/kmp_runtime.cpp", 7725); };
7726#endif
__kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7728#if OMPT_SUPPORT1
int tid = __kmp_tid_from_gtid(gtid);
ompt_data_t *task_data =
    &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data;
ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data;
if (ompt_enabled.ompt_callback_implicit_task) {
  ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
      ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid,
      ompt_task_initial);
  OMPT_CUR_TASK_INFO(this_thr)(&(this_thr->th.th_current_task->ompt_task_info))->thread_num = tid;
}
7739#endif
__kmp_teams_master(gtid);
7741#if OMPT_SUPPORT1
this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league;
7743#endif
__kmp_run_after_invoked_task(gtid, 0, this_thr, team);
return 1;
7746}

7748/* this sets the requested number of threads for the next parallel region
 encountered by this team. since this should be enclosed in the forkjoin
 critical section it should avoid race conditions with asymmetrical nested
 parallelism */

7753void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) {
kmp_info_t *thr = __kmp_threads[gtid];

if (num_threads > 0)
  thr->th.th_set_nproc = num_threads;
7758}

7760static void __kmp_push_thread_limit(kmp_info_t *thr, int num_teams,
                                  int num_threads) {
KMP_DEBUG_ASSERT(thr)if (!(thr)) { __kmp_debug_assert("thr", "openmp/runtime/src/kmp_runtime.cpp"
, 7762); };
// Remember the number of threads for inner parallel regions
if (!TCR_4(__kmp_init_middle)(__kmp_init_middle))
  __kmp_middle_initialize(); // get internal globals calculated
__kmp_assign_root_init_mask();
KMP_DEBUG_ASSERT(__kmp_avail_proc)if (!(__kmp_avail_proc)) { __kmp_debug_assert("__kmp_avail_proc"
, "openmp/runtime/src/kmp_runtime.cpp", 7767); };
KMP_DEBUG_ASSERT(__kmp_dflt_team_nth)if (!(__kmp_dflt_team_nth)) { __kmp_debug_assert("__kmp_dflt_team_nth"
, "openmp/runtime/src/kmp_runtime.cpp", 7768); };

if (num_threads == 0) {
  if (__kmp_teams_thread_limit > 0) {
    num_threads = __kmp_teams_thread_limit;
  } else {
    num_threads = __kmp_avail_proc / num_teams;
  }
  // adjust num_threads w/o warning as it is not user setting
  // num_threads = min(num_threads, nthreads-var, thread-limit-var)
  // no thread_limit clause specified -  do not change thread-limit-var ICV
  if (num_threads > __kmp_dflt_team_nth) {
    num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV
  }
  if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) {
    num_threads = thr->th.th_current_task->td_icvs.thread_limit;
  } // prevent team size to exceed thread-limit-var
  if (num_teams * num_threads > __kmp_teams_max_nth) {
    num_threads = __kmp_teams_max_nth / num_teams;
  }
  if (num_threads == 0) {
    num_threads = 1;
  }
} else {
  if (num_threads < 0) {
    __kmp_msg(kmp_ms_warning, KMP_MSG(CantFormThrTeam, num_threads, 1)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, num_threads, 1
),
              __kmp_msg_null);
    num_threads = 1;
  }
  // This thread will be the primary thread of the league primary threads
  // Store new thread limit; old limit is saved in th_cg_roots list
  thr->th.th_current_task->td_icvs.thread_limit = num_threads;
  // num_threads = min(num_threads, nthreads-var)
  if (num_threads > __kmp_dflt_team_nth) {
    num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV
  }
  if (num_teams * num_threads > __kmp_teams_max_nth) {
    int new_threads = __kmp_teams_max_nth / num_teams;
    if (new_threads == 0) {
      new_threads = 1;
    }
    if (new_threads != num_threads) {
      if (!__kmp_reserve_warn) { // user asked for too many threads
        __kmp_reserve_warn = 1; // conflicts with KMP_TEAMS_THREAD_LIMIT
        __kmp_msg(kmp_ms_warning,
                  KMP_MSG(CantFormThrTeam, num_threads, new_threads)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, num_threads, new_threads
),
                  KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
      }
    }
    num_threads = new_threads;
  }
}
thr->th.th_teams_size.nth = num_threads;
7821}

7823/* this sets the requested number of teams for the teams region and/or
 the number of threads for the next parallel region encountered  */
7825void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams,
                        int num_threads) {
kmp_info_t *thr = __kmp_threads[gtid];
if (num_teams < 0) {
  // OpenMP specification requires requested values to be positive,
  // but people can send us any value, so we'd better check
  __kmp_msg(kmp_ms_warning, KMP_MSG(NumTeamsNotPositive, num_teams, 1)__kmp_msg_format(kmp_i18n_msg_NumTeamsNotPositive, num_teams,
 1),
            __kmp_msg_null);
  num_teams = 1;
}
if (num_teams == 0) {
  if (__kmp_nteams > 0) {
    num_teams = __kmp_nteams;
  } else {
    num_teams = 1; // default number of teams is 1.
  }
}
if (num_teams > __kmp_teams_max_nth) { // if too many teams requested?
  if (!__kmp_reserve_warn) {
    __kmp_reserve_warn = 1;
    __kmp_msg(kmp_ms_warning,
              KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, num_teams, __kmp_teams_max_nth
),
              KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
  }
  num_teams = __kmp_teams_max_nth;
}
// Set number of teams (number of threads in the outer "parallel" of the
// teams)
thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;

__kmp_push_thread_limit(thr, num_teams, num_threads);
7856}

7858/* This sets the requested number of teams for the teams region and/or
 the number of threads for the next parallel region encountered  */
7860void __kmp_push_num_teams_51(ident_t *id, int gtid, int num_teams_lb,
                           int num_teams_ub, int num_threads) {
kmp_info_t *thr = __kmp_threads[gtid];
KMP_DEBUG_ASSERT(num_teams_lb >= 0 && num_teams_ub >= 0)if (!(num_teams_lb >= 0 && num_teams_ub >= 0)) {
 __kmp_debug_assert("num_teams_lb >= 0 && num_teams_ub >= 0"
, "openmp/runtime/src/kmp_runtime.cpp", 7863); };
KMP_DEBUG_ASSERT(num_teams_ub >= num_teams_lb)if (!(num_teams_ub >= num_teams_lb)) { __kmp_debug_assert(
"num_teams_ub >= num_teams_lb", "openmp/runtime/src/kmp_runtime.cpp"
, 7864); };
KMP_DEBUG_ASSERT(num_threads >= 0)if (!(num_threads >= 0)) { __kmp_debug_assert("num_threads >= 0"
, "openmp/runtime/src/kmp_runtime.cpp", 7865); };

if (num_teams_lb > num_teams_ub) {
  __kmp_fatal(KMP_MSG(FailedToCreateTeam, num_teams_lb, num_teams_ub)__kmp_msg_format(kmp_i18n_msg_FailedToCreateTeam, num_teams_lb
, num_teams_ub),
              KMP_HNT(SetNewBound, __kmp_teams_max_nth)__kmp_msg_format(kmp_i18n_hnt_SetNewBound, __kmp_teams_max_nth
), __kmp_msg_null);
}

int num_teams = 1; // defalt number of teams is 1.

if (num_teams_lb == 0 && num_teams_ub > 0)
  num_teams_lb = num_teams_ub;

if (num_teams_lb == 0 && num_teams_ub == 0) { // no num_teams clause
  num_teams = (__kmp_nteams > 0) ? __kmp_nteams : num_teams;
  if (num_teams > __kmp_teams_max_nth) {
    if (!__kmp_reserve_warn) {
      __kmp_reserve_warn = 1;
      __kmp_msg(kmp_ms_warning,
                KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth)__kmp_msg_format(kmp_i18n_msg_CantFormThrTeam, num_teams, __kmp_teams_max_nth
),
                KMP_HNT(Unset_ALL_THREADS)__kmp_msg_format(kmp_i18n_hnt_Unset_ALL_THREADS), __kmp_msg_null);
    }
    num_teams = __kmp_teams_max_nth;
  }
} else if (num_teams_lb == num_teams_ub) { // requires exact number of teams
  num_teams = num_teams_ub;
} else { // num_teams_lb <= num_teams <= num_teams_ub
  if (num_threads <= 0) {
    if (num_teams_ub > __kmp_teams_max_nth) {
      num_teams = num_teams_lb;
    } else {
      num_teams = num_teams_ub;
    }
  } else {
    num_teams = (num_threads > __kmp_teams_max_nth)
                    ? num_teams
                    : __kmp_teams_max_nth / num_threads;
    if (num_teams < num_teams_lb) {
      num_teams = num_teams_lb;
    } else if (num_teams > num_teams_ub) {
      num_teams = num_teams_ub;
    }
  }
}
// Set number of teams (number of threads in the outer "parallel" of the
// teams)
thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;

__kmp_push_thread_limit(thr, num_teams, num_threads);
7913}

7915// Set the proc_bind var to use in the following parallel region.
7916void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) {
kmp_info_t *thr = __kmp_threads[gtid];
thr->th.th_set_proc_bind = proc_bind;
7919}

7921/* Launch the worker threads into the microtask. */

7923void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) {
kmp_info_t *this_thr = __kmp_threads[gtid];

7926#ifdef KMP_DEBUG1
int f;
7928#endif /* KMP_DEBUG */

KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 7930); };
KMP_DEBUG_ASSERT(this_thr->th.th_team == team)if (!(this_thr->th.th_team == team)) { __kmp_debug_assert(
"this_thr->th.th_team == team", "openmp/runtime/src/kmp_runtime.cpp"
, 7931); };
KMP_ASSERT(KMP_MASTER_GTID(gtid))if (!((0 == __kmp_tid_from_gtid((gtid))))) { __kmp_debug_assert
("KMP_MASTER_GTID(gtid)", "openmp/runtime/src/kmp_runtime.cpp"
, 7932); };
KMP_MB(); /* Flush all pending memory write invalidates.  */

team->t.t_construct = 0; /* no single directives seen yet */
team->t.t_ordered.dt.t_value =
    0; /* thread 0 enters the ordered section first */

/* Reset the identifiers on the dispatch buffer */
KMP_DEBUG_ASSERT(team->t.t_disp_buffer)if (!(team->t.t_disp_buffer)) { __kmp_debug_assert("team->t.t_disp_buffer"
, "openmp/runtime/src/kmp_runtime.cpp", 7940); };
if (team->t.t_max_nproc > 1) {
  int i;
  for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
    team->t.t_disp_buffer[i].buffer_index = i;
    team->t.t_disp_buffer[i].doacross_buf_idx = i;
  }
} else {
  team->t.t_disp_buffer[0].buffer_index = 0;
  team->t.t_disp_buffer[0].doacross_buf_idx = 0;
}

KMP_MB(); /* Flush all pending memory write invalidates.  */
KMP_ASSERT(this_thr->th.th_team == team)if (!(this_thr->th.th_team == team)) { __kmp_debug_assert(
"this_thr->th.th_team == team", "openmp/runtime/src/kmp_runtime.cpp"
, 7953); };

7955#ifdef KMP_DEBUG1
for (f = 0; f < team->t.t_nproc; f++) {
  KMP_DEBUG_ASSERT(team->t.t_threads[f] &&if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 7958); }
                   team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc)if (!(team->t.t_threads[f] && team->t.t_threads
[f]->th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("team->t.t_threads[f] && team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 7958); };
}
7960#endif /* KMP_DEBUG */

/* release the worker threads so they may begin working */
__kmp_fork_barrier(gtid, 0);
7964}

7966void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) {
kmp_info_t *this_thr = __kmp_threads[gtid];

KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 7969); };
KMP_DEBUG_ASSERT(this_thr->th.th_team == team)if (!(this_thr->th.th_team == team)) { __kmp_debug_assert(
"this_thr->th.th_team == team", "openmp/runtime/src/kmp_runtime.cpp"
, 7970); };
KMP_ASSERT(KMP_MASTER_GTID(gtid))if (!((0 == __kmp_tid_from_gtid((gtid))))) { __kmp_debug_assert
("KMP_MASTER_GTID(gtid)", "openmp/runtime/src/kmp_runtime.cpp"
, 7971); };
KMP_MB(); /* Flush all pending memory write invalidates.  */

/* Join barrier after fork */

7976#ifdef KMP_DEBUG1
if (__kmp_threads[gtid] &&
    __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
  __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
               __kmp_threads[gtid]);
  __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "
               "team->t.t_nproc=%d\n",
               gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
               team->t.t_nproc);
  __kmp_print_structure();
}
KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&if (!(__kmp_threads[gtid] && __kmp_threads[gtid]->
th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("__kmp_threads[gtid] && __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 7988); }
                 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc)if (!(__kmp_threads[gtid] && __kmp_threads[gtid]->
th.th_team_nproc == team->t.t_nproc)) { __kmp_debug_assert
("__kmp_threads[gtid] && __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc"
, "openmp/runtime/src/kmp_runtime.cpp", 7988); };
7989#endif /* KMP_DEBUG */

__kmp_join_barrier(gtid); /* wait for everyone */
7992#if OMPT_SUPPORT1
if (ompt_enabled.enabled &&
    this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
  int ds_tid = this_thr->th.th_info.ds.ds_tid;
  ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr)(&(this_thr->th.th_current_task->ompt_task_info.task_data
));
  this_thr->th.ompt_thread_info.state = ompt_state_overhead;
7998#if OMPT_OPTIONAL1
  void *codeptr = NULL__null;
  if (KMP_MASTER_TID(ds_tid)(0 == (ds_tid)) &&
      (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback ||
       ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback))
    codeptr = OMPT_CUR_TEAM_INFO(this_thr)(&(this_thr->th.th_team->t.ompt_team_info))->master_return_address;

  if (ompt_enabled.ompt_callback_sync_region_wait) {
    ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)ompt_callback_sync_region_wait_callback(
        ompt_sync_region_barrier_implicit, ompt_scope_end, NULL__null, task_data,
        codeptr);
  }
  if (ompt_enabled.ompt_callback_sync_region) {
    ompt_callbacks.ompt_callback(ompt_callback_sync_region)ompt_callback_sync_region_callback(
        ompt_sync_region_barrier_implicit, ompt_scope_end, NULL__null, task_data,
        codeptr);
  }
8015#endif
  if (!KMP_MASTER_TID(ds_tid)(0 == (ds_tid)) && ompt_enabled.ompt_callback_implicit_task) {
    ompt_callbacks.ompt_callback(ompt_callback_implicit_task)ompt_callback_implicit_task_callback(
        ompt_scope_end, NULL__null, task_data, 0, ds_tid,
        ompt_task_implicit); // TODO: Can this be ompt_task_initial?
  }
}
8022#endif

KMP_MB(); /* Flush all pending memory write invalidates.  */
KMP_ASSERT(this_thr->th.th_team == team)if (!(this_thr->th.th_team == team)) { __kmp_debug_assert(
"this_thr->th.th_team == team", "openmp/runtime/src/kmp_runtime.cpp"
, 8025); };
8026}

8028/* ------------------------------------------------------------------------ */

8030#ifdef USE_LOAD_BALANCE1

8032// Return the worker threads actively spinning in the hot team, if we
8033// are at the outermost level of parallelism.  Otherwise, return 0.
8034static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
int i;
int retval;
kmp_team_t *hot_team;

if (root->r.r_active) {
  return 0;
}
hot_team = root->r.r_hot_team;
if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME(2147483647)) {
  return hot_team->t.t_nproc - 1; // Don't count primary thread
}

// Skip the primary thread - it is accounted for elsewhere.
retval = 0;
for (i = 1; i < hot_team->t.t_nproc; i++) {
  if (hot_team->t.t_threads[i]->th.th_active) {
    retval++;
  }
}
return retval;
8055}

8057// Perform an automatic adjustment to the number of
8058// threads used by the next parallel region.
8059static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) {
int retval;
int pool_active;
int hot_team_active;
int team_curr_active;
int system_active;

KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n"
, root, set_nproc); }
              set_nproc))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n"
, root, set_nproc); };
KMP_DEBUG_ASSERT(root)if (!(root)) { __kmp_debug_assert("root", "openmp/runtime/src/kmp_runtime.cpp"
, 8068); };
KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]if (!(root->r.r_root_team->t.t_threads[0] ->th.th_current_task
->td_icvs.dynamic == (!0))) { __kmp_debug_assert("root->r.r_root_team->t.t_threads[0] ->th.th_current_task->td_icvs.dynamic == (!0)"
, "openmp/runtime/src/kmp_runtime.cpp", 8070); }
                     ->th.th_current_task->td_icvs.dynamic == TRUE)if (!(root->r.r_root_team->t.t_threads[0] ->th.th_current_task
->td_icvs.dynamic == (!0))) { __kmp_debug_assert("root->r.r_root_team->t.t_threads[0] ->th.th_current_task->td_icvs.dynamic == (!0)"
, "openmp/runtime/src/kmp_runtime.cpp", 8070); };
KMP_DEBUG_ASSERT(set_nproc > 1)if (!(set_nproc > 1)) { __kmp_debug_assert("set_nproc > 1"
, "openmp/runtime/src/kmp_runtime.cpp", 8071); };

if (set_nproc == 1) {
  KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n"))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: serial execution.\n"
); };
  return 1;
}

// Threads that are active in the thread pool, active in the hot team for this
// particular root (if we are at the outer par level), and the currently
// executing thread (to become the primary thread) are available to add to the
// new team, but are currently contributing to the system load, and must be
// accounted for.
pool_active = __kmp_thread_pool_active_nth;
hot_team_active = __kmp_active_hot_team_nproc(root);
team_curr_active = pool_active + hot_team_active + 1;

// Check the system load.
system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d "if (kmp_b_debug >= 30) { __kmp_debug_printf ("__kmp_load_balance_nproc: system active = %d pool active = %d "
 "hot team active = %d\n", system_active, pool_active, hot_team_active
); }
              "hot team active = %d\n",if (kmp_b_debug >= 30) { __kmp_debug_printf ("__kmp_load_balance_nproc: system active = %d pool active = %d "
 "hot team active = %d\n", system_active, pool_active, hot_team_active
); }
              system_active, pool_active, hot_team_active))if (kmp_b_debug >= 30) { __kmp_debug_printf ("__kmp_load_balance_nproc: system active = %d pool active = %d "
 "hot team active = %d\n", system_active, pool_active, hot_team_active
); };

if (system_active < 0) {
  // There was an error reading the necessary info from /proc, so use the
  // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode
  // = dynamic_thread_limit, we shouldn't wind up getting back here.
  __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
  KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit")__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_CantLoadBalUsing
, "KMP_DYNAMIC_MODE=thread limit"), __kmp_msg_null);

  // Make this call behave like the thread limit algorithm.
  retval = __kmp_avail_proc - __kmp_nth +
           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
  if (retval > set_nproc) {
    retval = set_nproc;
  }
  if (retval < KMP_MIN_NTH1) {
    retval = KMP_MIN_NTH1;
  }

  KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n",if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n"
, retval); }
                retval))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n"
, retval); };
  return retval;
}

// There is a slight delay in the load balance algorithm in detecting new
// running procs. The real system load at this instant should be at least as
// large as the #active omp thread that are available to add to the team.
if (system_active < team_curr_active) {
  system_active = team_curr_active;
}
retval = __kmp_avail_proc - system_active + team_curr_active;
if (retval > set_nproc) {
  retval = set_nproc;
}
if (retval < KMP_MIN_NTH1) {
  retval = KMP_MIN_NTH1;
}

KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval))if (kmp_b_debug >= 20) { __kmp_debug_printf ("__kmp_load_balance_nproc: exit. retval:%d\n"
, retval); };
return retval;
8131} // __kmp_load_balance_nproc()

8133#endif /* USE_LOAD_BALANCE */

8135/* ------------------------------------------------------------------------ */

8137/* NOTE: this is called with the __kmp_init_lock held */
8138void __kmp_cleanup(void) {
int f;

KA_TRACE(10, ("__kmp_cleanup: enter\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_cleanup: enter\n"
); };

if (TCR_4(__kmp_init_parallel)(__kmp_init_parallel)) {
8144#if KMP_HANDLE_SIGNALS(1 || 0)
  __kmp_remove_signals();
8146#endif
  TCW_4(__kmp_init_parallel, FALSE)(__kmp_init_parallel) = (0);
}

if (TCR_4(__kmp_init_middle)(__kmp_init_middle)) {
8151#if KMP_AFFINITY_SUPPORTED1
  __kmp_affinity_uninitialize();
8153#endif /* KMP_AFFINITY_SUPPORTED */
  __kmp_cleanup_hierarchy();
  TCW_4(__kmp_init_middle, FALSE)(__kmp_init_middle) = (0);
}

KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_cleanup: go serial cleanup\n"
); };

if (__kmp_init_serial) {
  __kmp_runtime_destroy();
  __kmp_init_serial = FALSE0;
}

__kmp_cleanup_threadprivate_caches();

for (f = 0; f < __kmp_threads_capacity; f++) {
  if (__kmp_root[f] != NULL__null) {
    __kmp_free(__kmp_root[f])___kmp_free((__kmp_root[f]), "openmp/runtime/src/kmp_runtime.cpp"
, 8169);
    __kmp_root[f] = NULL__null;
  }
}
__kmp_free(__kmp_threads)___kmp_free((__kmp_threads), "openmp/runtime/src/kmp_runtime.cpp"
, 8173);
// __kmp_threads and __kmp_root were allocated at once, as single block, so
// there is no need in freeing __kmp_root.
__kmp_threads = NULL__null;
__kmp_root = NULL__null;
__kmp_threads_capacity = 0;

// Free old __kmp_threads arrays if they exist.
kmp_old_threads_list_t *ptr = __kmp_old_threads_list;
while (ptr) {
  kmp_old_threads_list_t *next = ptr->next;
  __kmp_free(ptr->threads)___kmp_free((ptr->threads), "openmp/runtime/src/kmp_runtime.cpp"
, 8184);
  __kmp_free(ptr)___kmp_free((ptr), "openmp/runtime/src/kmp_runtime.cpp", 8185
);
  ptr = next;
}

8189#if KMP_USE_DYNAMIC_LOCK1
__kmp_cleanup_indirect_user_locks();
8191#else
__kmp_cleanup_user_locks();
8193#endif
8194#if OMPD_SUPPORT1
if (ompd_state) {
  __kmp_free(ompd_env_block)___kmp_free((ompd_env_block), "openmp/runtime/src/kmp_runtime.cpp"
, 8196);
  ompd_env_block = NULL__null;
  ompd_env_block_size = 0;
}
8200#endif

8202#if KMP_AFFINITY_SUPPORTED1
KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file))free(const_cast<char *>(__kmp_cpuinfo_file));
__kmp_cpuinfo_file = NULL__null;
8205#endif /* KMP_AFFINITY_SUPPORTED */

8207#if KMP_USE_ADAPTIVE_LOCKS(0 || 1) && !0
8208#if KMP_DEBUG_ADAPTIVE_LOCKS0
__kmp_print_speculative_stats();
8210#endif
8211#endif
KMP_INTERNAL_FREE(__kmp_nested_nth.nth)free(__kmp_nested_nth.nth);
__kmp_nested_nth.nth = NULL__null;
__kmp_nested_nth.size = 0;
__kmp_nested_nth.used = 0;
KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types)free(__kmp_nested_proc_bind.bind_types);
__kmp_nested_proc_bind.bind_types = NULL__null;
__kmp_nested_proc_bind.size = 0;
__kmp_nested_proc_bind.used = 0;
if (__kmp_affinity_format) {
  KMP_INTERNAL_FREE(__kmp_affinity_format)free(__kmp_affinity_format);
  __kmp_affinity_format = NULL__null;
}

__kmp_i18n_catclose();

8227#if KMP_USE_HIER_SCHED0
__kmp_hier_scheds.deallocate();
8229#endif

8231#if KMP_STATS_ENABLED0
__kmp_stats_fini();
8233#endif

KA_TRACE(10, ("__kmp_cleanup: exit\n"))if (kmp_a_debug >= 10) { __kmp_debug_printf ("__kmp_cleanup: exit\n"
); };
8236}

8238/* ------------------------------------------------------------------------ */

8240int __kmp_ignore_mppbeg(void) {
char *env;

if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL__null) {
  if (__kmp_str_match_false(env))
    return FALSE0;
}
// By default __kmpc_begin() is no-op.
return TRUE(!0);
8249}

8251int __kmp_ignore_mppend(void) {
char *env;

if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL__null) {
  if (__kmp_str_match_false(env))
    return FALSE0;
}
// By default __kmpc_end() is no-op.
return TRUE(!0);
8260}

8262void __kmp_internal_begin(void) {
int gtid;
kmp_root_t *root;

/* this is a very important step as it will register new sibling threads
   and assign these new uber threads a new gtid */
gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
root = __kmp_threads[gtid]->th.th_root;
KMP_ASSERT(KMP_UBER_GTID(gtid))if (!(KMP_UBER_GTID(gtid))) { __kmp_debug_assert("KMP_UBER_GTID(gtid)"
, "openmp/runtime/src/kmp_runtime.cpp", 8270); };

if (root->r.r_begin)
  return;
__kmp_acquire_lock(&root->r.r_begin_lock, gtid);
if (root->r.r_begin) {
  __kmp_release_lock(&root->r.r_begin_lock, gtid);
  return;
}

root->r.r_begin = TRUE(!0);

__kmp_release_lock(&root->r.r_begin_lock, gtid);
8283}

8285/* ------------------------------------------------------------------------ */

8287void __kmp_user_set_library(enum library_type arg) {
int gtid;
kmp_root_t *root;
kmp_info_t *thread;

/* first, make sure we are initialized so we can get our gtid */

gtid = __kmp_entry_gtid()__kmp_get_global_thread_id_reg();
thread = __kmp_threads[gtid];

root = thread->th.th_root;

KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n"
, gtid, arg, library_serial); }
              library_serial))if (kmp_a_debug >= 20) { __kmp_debug_printf ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n"
, gtid, arg, library_serial); };
if (root->r.r_in_parallel) { /* Must be called in serial section of top-level
                                thread */
  KMP_WARNING(SetLibraryIncorrectCall)__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_SetLibraryIncorrectCall
), __kmp_msg_null);
  return;
}

switch (arg) {
case library_serial:
  thread->th.th_set_nproc = 0;
  set__nproc(thread, 1)(((thread)->th.th_current_task->td_icvs.nproc) = (1));
  break;
case library_turnaround:
  thread->th.th_set_nproc = 0;
  set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
 ? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub))
                                         : __kmp_dflt_team_nth_ub)(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
 ? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub));
  break;
case library_throughput:
  thread->th.th_set_nproc = 0;
  set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
 ? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub))
                                         : __kmp_dflt_team_nth_ub)(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_dflt_team_nth
 ? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub));
  break;
default:
  KMP_FATAL(UnknownLibraryType, arg)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownLibraryType,
 arg), __kmp_msg_null);
}

__kmp_aux_set_library(arg);
8327}

8329void __kmp_aux_set_stacksize(size_t arg) {
if (!__kmp_init_serial)
  __kmp_serial_initialize();

8333#if KMP_OS_DARWIN0
if (arg & (0x1000 - 1)) {
  arg &= ~(0x1000 - 1);
  if (arg + 0x1000) /* check for overflow if we round up */
    arg += 0x1000;
}
8339#endif
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);

/* only change the default stacksize before the first parallel region */
if (!TCR_4(__kmp_init_parallel)(__kmp_init_parallel)) {
  size_t value = arg; /* argument is in bytes */

  if (value < __kmp_sys_min_stksize)
    value = __kmp_sys_min_stksize;
  else if (value > KMP_MAX_STKSIZE(~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)
)))
    value = KMP_MAX_STKSIZE(~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)
));

  __kmp_stksize = value;

  __kmp_env_stksize = TRUE(!0); /* was KMP_STACKSIZE specified? */
}

__kmp_release_bootstrap_lock(&__kmp_initz_lock);
8357}

8359/* set the behaviour of the runtime library */
8360/* TODO this can cause some odd behaviour with sibling parallelism... */
8361void __kmp_aux_set_library(enum library_type arg) {
__kmp_library = arg;

switch (__kmp_library) {
case library_serial: {
  KMP_INFORM(LibraryIsSerial)__kmp_msg(kmp_ms_inform, __kmp_msg_format(kmp_i18n_msg_LibraryIsSerial
), __kmp_msg_null);
} break;
case library_turnaround:
  if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set)
    __kmp_use_yield = 2; // only yield when oversubscribed
  break;
case library_throughput:
  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME(2147483647))
    __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME(__kmp_is_hybrid_cpu() ? (0) : (200));
  break;
default:
  KMP_FATAL(UnknownLibraryType, arg)__kmp_fatal(__kmp_msg_format(kmp_i18n_msg_UnknownLibraryType,
 arg), __kmp_msg_null);
}
8379}

8381/* Getting team information common for all team API */
8382// Returns NULL if not in teams construct
8383static kmp_team_t *__kmp_aux_get_team_info(int &teams_serialized) {
kmp_info_t *thr = __kmp_entry_thread();
teams_serialized = 0;
if (thr->th.th_teams_microtask) {
  kmp_team_t *team = thr->th.th_team;
  int tlevel = thr->th.th_teams_level; // the level of the teams construct
  int ii = team->t.t_level;
  teams_serialized = team->t.t_serialized;
  int level = tlevel + 1;
  KMP_DEBUG_ASSERT(ii >= tlevel)if (!(ii >= tlevel)) { __kmp_debug_assert("ii >= tlevel"
, "openmp/runtime/src/kmp_runtime.cpp", 8392); };
  while (ii > level) {
    for (teams_serialized = team->t.t_serialized;
         (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {
    }
    if (team->t.t_serialized && (!teams_serialized)) {
      team = team->t.t_parent;
      continue;
    }
    if (ii > level) {
      team = team->t.t_parent;
      ii--;
    }
  }
  return team;
}
return NULL__null;
8409}

8411int __kmp_aux_get_team_num() {
int serialized;
kmp_team_t *team = __kmp_aux_get_team_info(serialized);
if (team) {
  if (serialized > 1) {
    return 0; // teams region is serialized ( 1 team of 1 thread ).
  } else {
    return team->t.t_master_tid;
  }
}
return 0;
8422}

8424int __kmp_aux_get_num_teams() {
int serialized;
kmp_team_t *team = __kmp_aux_get_team_info(serialized);
if (team) {
  if (serialized > 1) {
    return 1;
  } else {
    return team->t.t_parent->t.t_nproc;
  }
}
return 1;
8435}

8437/* ------------------------------------------------------------------------ */

8439/*
* Affinity Format Parser
*
* Field is in form of: %[[[0].]size]type
* % and type are required (%% means print a literal '%')
* type is either single char or long name surrounded by {},
* e.g., N or {num_threads}
* 0 => leading zeros
* . => right justified when size is specified
* by default output is left justified
* size is the *minimum* field length
* All other characters are printed as is
*
* Available field types:
* L {thread_level}      - omp_get_level()
* n {thread_num}        - omp_get_thread_num()
* h {host}              - name of host machine
* P {process_id}        - process id (integer)
* T {thread_identifier} - native thread identifier (integer)
* N {num_threads}       - omp_get_num_threads()
* A {ancestor_tnum}     - omp_get_ancestor_thread_num(omp_get_level()-1)
* a {thread_affinity}   - comma separated list of integers or integer ranges
*                         (values of affinity mask)
*
* Implementation-specific field types can be added
* If a type is unknown, print "undefined"
*/

8467// Structure holding the short name, long name, and corresponding data type
8468// for snprintf.  A table of these will represent the entire valid keyword
8469// field types.
8470typedef struct kmp_affinity_format_field_t {
char short_name; // from spec e.g., L -> thread level
const char *long_name; // from spec thread_level -> thread level
char field_format; // data type for snprintf (typically 'd' or 's'
// for integer or string)
8475} kmp_affinity_format_field_t;

8477static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = {
8478#if KMP_AFFINITY_SUPPORTED1
  {'A', "thread_affinity", 's'},
8480#endif
  {'t', "team_num", 'd'},
  {'T', "num_teams", 'd'},
  {'L', "nesting_level", 'd'},
  {'n', "thread_num", 'd'},
  {'N', "num_threads", 'd'},
  {'a', "ancestor_tnum", 'd'},
  {'H', "host", 's'},
  {'P', "process_id", 'd'},
  {'i', "native_thread_id", 'd'}};

8491// Return the number of characters it takes to hold field
8492static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th,
                                          const char **ptr,
                                          kmp_str_buf_t *field_buffer) {
int rc, format_index, field_value;
const char *width_left, *width_right;
bool pad_zeros, right_justify, parse_long_name, found_valid_name;
static const int FORMAT_SIZE = 20;
char format[FORMAT_SIZE] = {0};
char absolute_short_name = 0;

KMP_DEBUG_ASSERT(gtid >= 0)if (!(gtid >= 0)) { __kmp_debug_assert("gtid >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 8502); };
KMP_DEBUG_ASSERT(th)if (!(th)) { __kmp_debug_assert("th", "openmp/runtime/src/kmp_runtime.cpp"
, 8503); };
KMP_DEBUG_ASSERT(**ptr == '%')if (!(**ptr == '%')) { __kmp_debug_assert("**ptr == '%'", "openmp/runtime/src/kmp_runtime.cpp"
, 8504); };
KMP_DEBUG_ASSERT(field_buffer)if (!(field_buffer)) { __kmp_debug_assert("field_buffer", "openmp/runtime/src/kmp_runtime.cpp"
, 8505); };

__kmp_str_buf_clear(field_buffer);

// Skip the initial %
(*ptr)++;

// Check for %% first
if (**ptr == '%') {
  __kmp_str_buf_cat(field_buffer, "%", 1);
  (*ptr)++; // skip over the second %
  return 1;
}

// Parse field modifiers if they are present
pad_zeros = false;
if (**ptr == '0') {
  pad_zeros = true;
  (*ptr)++; // skip over 0
}
right_justify = false;
if (**ptr == '.') {
  right_justify = true;
  (*ptr)++; // skip over .
}
// Parse width of field: [width_left, width_right)
width_left = width_right = NULL__null;
if (**ptr >= '0' && **ptr <= '9') {
  width_left = *ptr;
  SKIP_DIGITS(*ptr){ while (*(*ptr) >= '0' && *(*ptr) <= '9') (*ptr
)++; };
  width_right = *ptr;
}

// Create the format for KMP_SNPRINTF based on flags parsed above
format_index = 0;
format[format_index++] = '%';
if (!right_justify)
  format[format_index++] = '-';
if (pad_zeros)
  format[format_index++] = '0';
if (width_left && width_right) {
  int i = 0;
  // Only allow 8 digit number widths.
  // This also prevents overflowing format variable
  while (i < 8 && width_left < width_right) {
    format[format_index++] = *width_left;
    width_left++;
    i++;
  }
}

// Parse a name (long or short)
// Canonicalize the name into absolute_short_name
found_valid_name = false;
parse_long_name = (**ptr == '{');
if (parse_long_name)
  (*ptr)++; // skip initial left brace
for (size_t i = 0; i < sizeof(__kmp_affinity_format_table) /
                           sizeof(__kmp_affinity_format_table[0]);
     ++i) {
  char short_name = __kmp_affinity_format_table[i].short_name;
  const char *long_name = __kmp_affinity_format_table[i].long_name;
  char field_format = __kmp_affinity_format_table[i].field_format;
  if (parse_long_name) {
    size_t length = KMP_STRLENstrlen(long_name);
    if (strncmp(*ptr, long_name, length) == 0) {
      found_valid_name = true;
      (*ptr) += length; // skip the long name
    }
  } else if (**ptr == short_name) {
    found_valid_name = true;
    (*ptr)++; // skip the short name
  }
  if (found_valid_name) {
    format[format_index++] = field_format;
    format[format_index++] = '\0';
    absolute_short_name = short_name;
    break;
  }
}
if (parse_long_name) {
  if (**ptr != '}') {
    absolute_short_name = 0;
  } else {
    (*ptr)++; // skip over the right brace
  }
}

// Attempt to fill the buffer with the requested
// value using snprintf within __kmp_str_buf_print()
switch (absolute_short_name) {
case 't':
  rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num());
  break;
case 'T':
  rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams());
  break;
case 'L':
  rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);
  break;
case 'n':
  rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid));
  break;
case 'H': {
  static const int BUFFER_SIZE = 256;
  char buf[BUFFER_SIZE];
  __kmp_expand_host_name(buf, BUFFER_SIZE);
  rc = __kmp_str_buf_print(field_buffer, format, buf);
} break;
case 'P':
  rc = __kmp_str_buf_print(field_buffer, format, getpid());
  break;
case 'i':
  rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid()syscall(186));
  break;
case 'N':
  rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);
  break;
case 'a':
  field_value =
      __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);
  rc = __kmp_str_buf_print(field_buffer, format, field_value);
  break;
8628#if KMP_AFFINITY_SUPPORTED1
case 'A': {
  kmp_str_buf_t buf;
  __kmp_str_buf_init(&buf){ (&buf)->str = (&buf)->bulk; (&buf)->size
 = sizeof((&buf)->bulk); (&buf)->used = 0; (&
buf)->bulk[0] = 0; };
  __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);
  rc = __kmp_str_buf_print(field_buffer, format, buf.str);
  __kmp_str_buf_free(&buf);
} break;
8636#endif
default:
  // According to spec, If an implementation does not have info for field
  // type, then "undefined" is printed
  rc = __kmp_str_buf_print(field_buffer, "%s", "undefined");
  // Skip the field
  if (parse_long_name) {
    SKIP_TOKEN(*ptr){ while ((*(*ptr) >= '0' && *(*ptr) <= '9') || (
*(*ptr) >= 'a' && *(*ptr) <= 'z') || (*(*ptr) >=
 'A' && *(*ptr) <= 'Z') || *(*ptr) == '_') (*ptr)++
; };
    if (**ptr == '}')
      (*ptr)++;
  } else {
    (*ptr)++;
  }
}

KMP_ASSERT(format_index <= FORMAT_SIZE)if (!(format_index <= FORMAT_SIZE)) { __kmp_debug_assert("format_index <= FORMAT_SIZE"
, "openmp/runtime/src/kmp_runtime.cpp", 8651); };
return rc;
8653}

8655/*
* Return number of characters needed to hold the affinity string
* (not including null byte character)
* The resultant string is printed to buffer, which the caller can then
* handle afterwards
*/
8661size_t __kmp_aux_capture_affinity(int gtid, const char *format,
                                kmp_str_buf_t *buffer) {
const char *parse_ptr;
size_t retval;
const kmp_info_t *th;
kmp_str_buf_t field;

KMP_DEBUG_ASSERT(buffer)if (!(buffer)) { __kmp_debug_assert("buffer", "openmp/runtime/src/kmp_runtime.cpp"
, 8668); };
KMP_DEBUG_ASSERT(gtid >= 0)if (!(gtid >= 0)) { __kmp_debug_assert("gtid >= 0", "openmp/runtime/src/kmp_runtime.cpp"
, 8669); };

__kmp_str_buf_init(&field){ (&field)->str = (&field)->bulk; (&field)->
size = sizeof((&field)->bulk); (&field)->used =
 0; (&field)->bulk[0] = 0; };
__kmp_str_buf_clear(buffer);

th = __kmp_threads[gtid];
retval = 0;

// If format is NULL or zero-length string, then we use
// affinity-format-var ICV
parse_ptr = format;
if (parse_ptr == NULL__null || *parse_ptr == '\0') {
  parse_ptr = __kmp_affinity_format;
}
KMP_DEBUG_ASSERT(parse_ptr)if (!(parse_ptr)) { __kmp_debug_assert("parse_ptr", "openmp/runtime/src/kmp_runtime.cpp"
, 8683); };

while (*parse_ptr != '\0') {
  // Parse a field
  if (*parse_ptr == '%') {
    // Put field in the buffer
    int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field);
    __kmp_str_buf_catbuf(buffer, &field);
    retval += rc;
  } else {
    // Put literal character in buffer
    __kmp_str_buf_cat(buffer, parse_ptr, 1);
    retval++;
    parse_ptr++;
  }
}
__kmp_str_buf_free(&field);
return retval;
8701}

8703// Displays the affinity string to stdout
8704void __kmp_aux_display_affinity(int gtid, const char *format) {
kmp_str_buf_t buf;
__kmp_str_buf_init(&buf){ (&buf)->str = (&buf)->bulk; (&buf)->size
 = sizeof((&buf)->bulk); (&buf)->used = 0; (&
buf)->bulk[0] = 0; };
__kmp_aux_capture_affinity(gtid, format, &buf);
__kmp_fprintf(kmp_out, "%s" KMP_END_OF_LINE"\n", buf.str);
__kmp_str_buf_free(&buf);
8710}

8712/* ------------------------------------------------------------------------ */

8714void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) {
int blocktime = arg; /* argument is in milliseconds */
8716#if KMP_USE_MONITOR
int bt_intervals;
8718#endif
kmp_int8 bt_set;

__kmp_save_internal_controls(thread);

/* Normalize and set blocktime for the teams */
if (blocktime < KMP_MIN_BLOCKTIME(0))
  blocktime = KMP_MIN_BLOCKTIME(0);
else if (blocktime > KMP_MAX_BLOCKTIME(2147483647))
  blocktime = KMP_MAX_BLOCKTIME(2147483647);

set__blocktime_team(thread->th.th_team, tid, blocktime)(((thread->th.th_team)->t.t_threads[(tid)]->th.th_current_task
->td_icvs.blocktime) = (blocktime));
set__blocktime_team(thread->th.th_serial_team, 0, blocktime)(((thread->th.th_serial_team)->t.t_threads[(0)]->th.
th_current_task->td_icvs.blocktime) = (blocktime));

8732#if KMP_USE_MONITOR
/* Calculate and set blocktime intervals for the teams */
bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);

set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
8738#endif

/* Set whether blocktime has been set to "TRUE" */
bt_set = TRUE(!0);

set__bt_set_team(thread->th.th_team, tid, bt_set)(((thread->th.th_team)->t.t_threads[(tid)]->th.th_current_task
->td_icvs.bt_set) = (bt_set));
set__bt_set_team(thread->th.th_serial_team, 0, bt_set)(((thread->th.th_serial_team)->t.t_threads[(0)]->th.
th_current_task->td_icvs.bt_set) = (bt_set));
8745#if KMP_USE_MONITOR
KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
 "bt_intervals=%d, monitor_updates=%d\n", __kmp_gtid_from_tid
(tid, thread->th.th_team), thread->th.th_team->t.t_id
, tid, blocktime, bt_intervals, __kmp_monitor_wakeups); }
              "bt_intervals=%d, monitor_updates=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
 "bt_intervals=%d, monitor_updates=%d\n", __kmp_gtid_from_tid
(tid, thread->th.th_team), thread->th.th_team->t.t_id
, tid, blocktime, bt_intervals, __kmp_monitor_wakeups); }
              __kmp_gtid_from_tid(tid, thread->th.th_team),if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
 "bt_intervals=%d, monitor_updates=%d\n", __kmp_gtid_from_tid
(tid, thread->th.th_team), thread->th.th_team->t.t_id
, tid, blocktime, bt_intervals, __kmp_monitor_wakeups); }
              thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
 "bt_intervals=%d, monitor_updates=%d\n", __kmp_gtid_from_tid
(tid, thread->th.th_team), thread->th.th_team->t.t_id
, tid, blocktime, bt_intervals, __kmp_monitor_wakeups); }
              __kmp_monitor_wakeups))if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
 "bt_intervals=%d, monitor_updates=%d\n", __kmp_gtid_from_tid
(tid, thread->th.th_team), thread->th.th_team->t.t_id
, tid, blocktime, bt_intervals, __kmp_monitor_wakeups); };
8751#else
KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n"
, __kmp_gtid_from_tid(tid, thread->th.th_team), thread->
th.th_team->t.t_id, tid, blocktime); }
              __kmp_gtid_from_tid(tid, thread->th.th_team),if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n"
, __kmp_gtid_from_tid(tid, thread->th.th_team), thread->
th.th_team->t.t_id, tid, blocktime); }
              thread->th.th_team->t.t_id, tid, blocktime))if (kmp_f_debug >= 10) { __kmp_debug_printf ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n"
, __kmp_gtid_from_tid(tid, thread->th.th_team), thread->
th.th_team->t.t_id, tid, blocktime); };
8755#endif
8756}

8758void __kmp_aux_set_defaults(char const *str, size_t len) {
if (!__kmp_init_serial) {
  __kmp_serial_initialize();
}
__kmp_env_initialize(str);

if (__kmp_settings || __kmp_display_env || __kmp_display_env_verbose) {
  __kmp_env_print();
}
8767} // __kmp_aux_set_defaults

8769/* ------------------------------------------------------------------------ */
8770/* internal fast reduction routines */

8772PACKED_REDUCTION_METHOD_T
8773__kmp_determine_reduction_method(
  ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
  void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
  kmp_critical_name *lck) {

// Default reduction method: critical construct ( lck != NULL, like in current
// PAROPT )
// If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method
// can be selected by RTL
// If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method
// can be selected by RTL
// Finally, it's up to OpenMP RTL to make a decision on which method to select
// among generated by PAROPT.

PACKED_REDUCTION_METHOD_T retval;

int team_size;

KMP_DEBUG_ASSERT(loc)if (!(loc)) { __kmp_debug_assert("loc", "openmp/runtime/src/kmp_runtime.cpp"
, 8791); }; // it would be nice to test ( loc != 0 )
KMP_DEBUG_ASSERT(lck)if (!(lck)) { __kmp_debug_assert("lck", "openmp/runtime/src/kmp_runtime.cpp"
, 8792); }; // it would be nice to test ( lck != 0 )

8794#define FAST_REDUCTION_ATOMIC_METHOD_GENERATED                                 \
(loc &&                                                                      \
 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)))
8797#define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func))

retval = critical_reduce_block;

// another choice of getting a team size (with 1 dynamic deference) is slower
team_size = __kmp_get_team_num_threads(global_tid)(__kmp_threads[(global_tid)]->th.th_team->t.t_nproc);
if (team_size == 1) {

  retval = empty_reduce_block;

} else {

  int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;

8811#if KMP_ARCH_X86_641 || KMP_ARCH_PPC64(0 || 0) || KMP_ARCH_AARCH640 ||                   \
  KMP_ARCH_MIPS640 || KMP_ARCH_RISCV640 || KMP_ARCH_LOONGARCH640

8814#if KMP_OS_LINUX1 || KMP_OS_DRAGONFLY0 || KMP_OS_FREEBSD0 || KMP_OS_NETBSD0 ||     \
  KMP_OS_OPENBSD0 || KMP_OS_WINDOWS0 || KMP_OS_DARWIN0 || KMP_OS_HURD0

  int teamsize_cutoff = 4;

8819#if KMP_MIC_SUPPORTED((0 || 1) && (1 || 0))
  if (__kmp_mic_type != non_mic) {
    teamsize_cutoff = 8;
  }
8823#endif
  int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
  if (tree_available) {
    if (team_size <= teamsize_cutoff) {
      if (atomic_available) {
        retval = atomic_reduce_block;
      }
    } else {
      retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER(((tree_reduce_block) | (bs_reduction_barrier)));
    }
  } else if (atomic_available) {
    retval = atomic_reduce_block;
  }
8836#else
8837#error "Unknown or unsupported OS"
8838#endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||
     // KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD

8841#elif KMP_ARCH_X860 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS0

8843#if KMP_OS_LINUX1 || KMP_OS_FREEBSD0 || KMP_OS_WINDOWS0 || KMP_OS_HURD0

  // basic tuning

  if (atomic_available) {
    if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ???
      retval = atomic_reduce_block;
    }
  } // otherwise: use critical section

8853#elif KMP_OS_DARWIN0

  int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
  if (atomic_available && (num_vars <= 3)) {
    retval = atomic_reduce_block;
  } else if (tree_available) {
    if ((reduce_size > (9 * sizeof(kmp_real64))) &&
        (reduce_size < (2000 * sizeof(kmp_real64)))) {
      retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER(((tree_reduce_block) | (bs_plain_barrier)));
    }
  } // otherwise: use critical section

8865#else
8866#error "Unknown or unsupported OS"
8867#endif

8869#else
8870#error "Unknown or unsupported architecture"
8871#endif
}

// KMP_FORCE_REDUCTION

// If the team is serialized (team_size == 1), ignore the forced reduction
// method and stay with the unsynchronized method (empty_reduce_block)
if (__kmp_force_reduction_method != reduction_method_not_defined &&
    team_size != 1) {

  PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;

  int atomic_available, tree_available;

  switch ((forced_retval = __kmp_force_reduction_method)) {
  case critical_reduce_block:
    KMP_ASSERT(lck)if (!(lck)) { __kmp_debug_assert("lck", "openmp/runtime/src/kmp_runtime.cpp"
, 8887); }; // lck should be != 0
    break;

  case atomic_reduce_block:
    atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
    if (!atomic_available) {
      KMP_WARNING(RedMethodNotSupported, "atomic")__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_RedMethodNotSupported
, "atomic"), __kmp_msg_null);
      forced_retval = critical_reduce_block;
    }
    break;

  case tree_reduce_block:
    tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
    if (!tree_available) {
      KMP_WARNING(RedMethodNotSupported, "tree")__kmp_msg(kmp_ms_warning, __kmp_msg_format(kmp_i18n_msg_RedMethodNotSupported
, "tree"), __kmp_msg_null);
      forced_retval = critical_reduce_block;
    } else {
8904#if KMP_FAST_REDUCTION_BARRIER1
      forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER(((tree_reduce_block) | (bs_reduction_barrier)));
8906#endif
    }
    break;

  default:
    KMP_ASSERT(0)if (!(0)) { __kmp_debug_assert("0", "openmp/runtime/src/kmp_runtime.cpp"
, 8911); }; // "unsupported method specified"
  }

  retval = forced_retval;
}

KA_TRACE(10, ("reduction method selected=%08x\n", retval))if (kmp_a_debug >= 10) { __kmp_debug_printf ("reduction method selected=%08x\n"
, retval); };

8919#undef FAST_REDUCTION_TREE_METHOD_GENERATED
8920#undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED

return (retval);
8923}
8924// this function is for testing set/get/determine reduce method
8925kmp_int32 __kmp_get_reduce_method(void) {
return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
8927}

8929// Soft pause sets up threads to ignore blocktime and just go to sleep.
8930// Spin-wait code checks __kmp_pause_status and reacts accordingly.
8931void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }

8933// Hard pause shuts down the runtime completely.  Resume happens naturally when
8934// OpenMP is used subsequently.
8935void __kmp_hard_pause() {
__kmp_pause_status = kmp_hard_paused;
__kmp_internal_end_thread(-1);
8938}

8940// Soft resume sets __kmp_pause_status, and wakes up all threads.
8941void __kmp_resume_if_soft_paused() {
if (__kmp_pause_status == kmp_soft_paused) {
  __kmp_pause_status = kmp_not_paused;

  for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
    kmp_info_t *thread = __kmp_threads[gtid];
    if (thread) { // Wake it if sleeping
      kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
                       thread);
      if (fl.is_sleeping())
        fl.resume(gtid);
      else if (__kmp_try_suspend_mx(thread)) { // got suspend lock
        __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep
      } else { // thread holds the lock and may sleep soon
        do { // until either the thread sleeps, or we can get the lock
          if (fl.is_sleeping()) {
            fl.resume(gtid);
            break;
          } else if (__kmp_try_suspend_mx(thread)) {
            __kmp_unlock_suspend_mx(thread);
            break;
          }
        } while (1);
      }
    }
  }
}
8968}

8970// This function is called via __kmpc_pause_resource. Returns 0 if successful.
8971// TODO: add warning messages
8972int __kmp_pause_resource(kmp_pause_status_t level) {
if (level == kmp_not_paused) { // requesting resume
  if (__kmp_pause_status == kmp_not_paused) {
    // error message about runtime not being paused, so can't resume
    return 1;
  } else {
    KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||if (!(__kmp_pause_status == kmp_soft_paused || __kmp_pause_status
 == kmp_hard_paused)) { __kmp_debug_assert("__kmp_pause_status == kmp_soft_paused || __kmp_pause_status == kmp_hard_paused"
, "openmp/runtime/src/kmp_runtime.cpp", 8979); }
                     __kmp_pause_status == kmp_hard_paused)if (!(__kmp_pause_status == kmp_soft_paused || __kmp_pause_status
 == kmp_hard_paused)) { __kmp_debug_assert("__kmp_pause_status == kmp_soft_paused || __kmp_pause_status == kmp_hard_paused"
, "openmp/runtime/src/kmp_runtime.cpp", 8979); };
    __kmp_pause_status = kmp_not_paused;
    return 0;
  }
} else if (level == kmp_soft_paused) { // requesting soft pause
  if (__kmp_pause_status != kmp_not_paused) {
    // error message about already being paused
    return 1;
  } else {
    __kmp_soft_pause();
    return 0;
  }
} else if (level == kmp_hard_paused) { // requesting hard pause
  if (__kmp_pause_status != kmp_not_paused) {
    // error message about already being paused
    return 1;
  } else {
    __kmp_hard_pause();
    return 0;
  }
} else {
  // error message about invalid level
  return 1;
}
9003}

9005void __kmp_omp_display_env(int verbose) {
__kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
if (__kmp_init_serial == 0)
  __kmp_do_serial_initialize();
__kmp_display_env_impl(!verbose, verbose);
__kmp_release_bootstrap_lock(&__kmp_initz_lock);
9011}

9013// The team size is changing, so distributed barrier must be modified
9014void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads,
                             int new_nthreads) {
KMP_DEBUG_ASSERT(__kmp_barrier_release_pattern[bs_forkjoin_barrier] ==if (!(__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar
)) { __kmp_debug_assert("__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar"
, "openmp/runtime/src/kmp_runtime.cpp", 9017); }
                 bp_dist_bar)if (!(__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar
)) { __kmp_debug_assert("__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar"
, "openmp/runtime/src/kmp_runtime.cpp", 9017); };
kmp_info_t **other_threads = team->t.t_threads;

// We want all the workers to stop waiting on the barrier while we adjust the
// size of the team.
for (int f = 1; f < old_nthreads; ++f) {
  KMP_DEBUG_ASSERT(other_threads[f] != NULL)if (!(other_threads[f] != __null)) { __kmp_debug_assert("other_threads[f] != __null"
, "openmp/runtime/src/kmp_runtime.cpp", 9023); };
  // Ignore threads that are already inactive or not present in the team
  if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) {
    // teams construct causes thread_limit to get passed in, and some of
    // those could be inactive; just ignore them
    continue;
  }
  // If thread is transitioning still to in_use state, wait for it
  if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) {
    while (team->t.t_threads[f]->th.th_used_in_team.load() == 3)
      KMP_CPU_PAUSE()__kmp_x86_pause();
  }
  // The thread should be in_use now
  KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1)if (!(team->t.t_threads[f]->th.th_used_in_team.load() ==
 1)) { __kmp_debug_assert("team->t.t_threads[f]->th.th_used_in_team.load() == 1"
, "openmp/runtime/src/kmp_runtime.cpp", 9036); };
  // Transition to unused state
  team->t.t_threads[f]->th.th_used_in_team.store(2);
  KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2)if (!(team->t.t_threads[f]->th.th_used_in_team.load() ==
 2)) { __kmp_debug_assert("team->t.t_threads[f]->th.th_used_in_team.load() == 2"
, "openmp/runtime/src/kmp_runtime.cpp", 9039); };
}
// Release all the workers
team->t.b->go_release();

KMP_MFENCE()if (__builtin_expect(!!(!__kmp_cpuinfo.initialized), 0)) { __kmp_query_cpuid
(&__kmp_cpuinfo); } if (__kmp_cpuinfo.flags.sse2) { __sync_synchronize
(); };

// Workers should see transition status 2 and move to 0; but may need to be
// woken up first
int count = old_nthreads - 1;
while (count > 0) {
  count = old_nthreads - 1;
  for (int f = 1; f < old_nthreads; ++f) {
    if (other_threads[f]->th.th_used_in_team.load() != 0) {
      if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647)) { // Wake up the workers
        kmp_atomic_flag_64<> *flag = (kmp_atomic_flag_64<> *)CCAST(const_cast<void *>(other_threads[f]->th.th_sleep_loc
)
            void *, other_threads[f]->th.th_sleep_loc)const_cast<void *>(other_threads[f]->th.th_sleep_loc
);
        __kmp_atomic_resume_64(other_threads[f]->th.th_info.ds.ds_gtid, flag);
      }
    } else {
      KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0)if (!(team->t.t_threads[f]->th.th_used_in_team.load() ==
 0)) { __kmp_debug_assert("team->t.t_threads[f]->th.th_used_in_team.load() == 0"
, "openmp/runtime/src/kmp_runtime.cpp", 9059); };
      count--;
    }
  }
}
// Now update the barrier size
team->t.b->update_num_threads(new_nthreads);
team->t.b->go_reset();
9067}

9069void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads) {
// Add the threads back to the team
KMP_DEBUG_ASSERT(team)if (!(team)) { __kmp_debug_assert("team", "openmp/runtime/src/kmp_runtime.cpp"
, 9071); };
// Threads were paused and pointed at th_used_in_team temporarily during a
// resize of the team. We're going to set th_used_in_team to 3 to indicate to
// the thread that it should transition itself back into the team. Then, if
// blocktime isn't infinite, the thread could be sleeping, so we send a resume
// to wake it up.
for (int f = 1; f < new_nthreads; ++f) {
  KMP_DEBUG_ASSERT(team->t.t_threads[f])if (!(team->t.t_threads[f])) { __kmp_debug_assert("team->t.t_threads[f]"
, "openmp/runtime/src/kmp_runtime.cpp", 9078); };
  KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0,__sync_bool_compare_and_swap((volatile kmp_uint32 *)(&(team
->t.t_threads[f]->th.th_used_in_team)), (kmp_uint32)(0)
, (kmp_uint32)(3))
                              3)__sync_bool_compare_and_swap((volatile kmp_uint32 *)(&(team
->t.t_threads[f]->th.th_used_in_team)), (kmp_uint32)(0)
, (kmp_uint32)(3));
  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME(2147483647)) { // Wake up sleeping threads
    __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid,
                    (kmp_flag_32<false, false> *)NULL__null);
  }
}
// The threads should be transitioning to the team; when they are done, they
// should have set th_used_in_team to 1. This loop forces master to wait until
// all threads have moved into the team and are waiting in the barrier.
int count = new_nthreads - 1;
while (count > 0) {
  count = new_nthreads - 1;
  for (int f = 1; f < new_nthreads; ++f) {
    if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) {
      count--;
    }
  }
}
9098}

9100// Globals and functions for hidden helper task
9101kmp_info_t **__kmp_hidden_helper_threads;
9102kmp_info_t *__kmp_hidden_helper_main_thread;
9103std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks;
9104#if KMP_OS_LINUX1
9105kmp_int32 __kmp_hidden_helper_threads_num = 8;
9106kmp_int32 __kmp_enable_hidden_helper = TRUE(!0);
9107#else
9108kmp_int32 __kmp_hidden_helper_threads_num = 0;
9109kmp_int32 __kmp_enable_hidden_helper = FALSE0;
9110#endif

9112namespace {
9113std::atomic<kmp_int32> __kmp_hit_hidden_helper_threads_num;

9115void __kmp_hidden_helper_wrapper_fn(int *gtid, int *, ...) {
// This is an explicit synchronization on all hidden helper threads in case
// that when a regular thread pushes a hidden helper task to one hidden
// helper thread, the thread has not been awaken once since they're released
// by the main thread after creating the team.
KMP_ATOMIC_INC(&__kmp_hit_hidden_helper_threads_num)(&__kmp_hit_hidden_helper_threads_num)->fetch_add(1, std
::memory_order_acq_rel);
while (KMP_ATOMIC_LD_ACQ(&__kmp_hit_hidden_helper_threads_num)(&__kmp_hit_hidden_helper_threads_num)->load(std::memory_order_acquire
) !=
       __kmp_hidden_helper_threads_num)
  ;

// If main thread, then wait for signal
if (__kmpc_master(nullptr, *gtid)) {
  // First, unset the initial state and release the initial thread
  TCW_4(__kmp_init_hidden_helper_threads, FALSE)(__kmp_init_hidden_helper_threads) = (0);
  __kmp_hidden_helper_initz_release();
  __kmp_hidden_helper_main_thread_wait();
  // Now wake up all worker threads
  for (int i = 1; i < __kmp_hit_hidden_helper_threads_num; ++i) {
    __kmp_hidden_helper_worker_thread_signal();
  }
}
9136}
9137} // namespace

9139void __kmp_hidden_helper_threads_initz_routine() {
// Create a new root for hidden helper team/threads
const int gtid = __kmp_register_root(TRUE(!0));
__kmp_hidden_helper_main_thread = __kmp_threads[gtid];
__kmp_hidden_helper_threads = &__kmp_threads[gtid];
__kmp_hidden_helper_main_thread->th.th_set_nproc =
    __kmp_hidden_helper_threads_num;

KMP_ATOMIC_ST_REL(&__kmp_hit_hidden_helper_threads_num, 0)(&__kmp_hit_hidden_helper_threads_num)->store(0, std::
memory_order_release);

__kmpc_fork_call(nullptr, 0, __kmp_hidden_helper_wrapper_fn);

// Set the initialization flag to FALSE
TCW_SYNC_4(__kmp_init_hidden_helper, FALSE)(__kmp_init_hidden_helper) = (0);

__kmp_hidden_helper_threads_deinitz_release();
9155}

9157/* Nesting Mode:
 Set via KMP_NESTING_MODE, which takes an integer.
 Note: we skip duplicate topology levels, and skip levels with only
    one entity.
 KMP_NESTING_MODE=0 is the default, and doesn't use nesting mode.
 KMP_NESTING_MODE=1 sets as many nesting levels as there are distinct levels
    in the topology, and initializes the number of threads at each of those
    levels to the number of entities at each level, respectively, below the
    entity at the parent level.
 KMP_NESTING_MODE=N, where N>1, attempts to create up to N nesting levels,
    but starts with nesting OFF -- max-active-levels-var is 1 -- and requires
    the user to turn nesting on explicitly. This is an even more experimental
    option to this experimental feature, and may change or go away in the
    future.
9171*/

9173// Allocate space to store nesting levels
9174void __kmp_init_nesting_mode() {
int levels = KMP_HW_LAST;
__kmp_nesting_mode_nlevels = levels;
__kmp_nesting_nth_level = (int *)KMP_INTERNAL_MALLOC(levels * sizeof(int))malloc(levels * sizeof(int));
for (int i = 0; i < levels; ++i)
  __kmp_nesting_nth_level[i] = 0;
if (__kmp_nested_nth.size < levels) {
  __kmp_nested_nth.nth =
      (int *)KMP_INTERNAL_REALLOC(__kmp_nested_nth.nth, levels * sizeof(int))realloc((__kmp_nested_nth.nth), (levels * sizeof(int)));
  __kmp_nested_nth.size = levels;
}
9185}

9187// Set # threads for top levels of nesting; must be called after topology set
9188void __kmp_set_nesting_mode_threads() {
kmp_info_t *thread = __kmp_threads[__kmp_entry_gtid()__kmp_get_global_thread_id_reg()];

if (__kmp_nesting_mode == 1)
  __kmp_nesting_mode_nlevels = KMP_MAX_ACTIVE_LEVELS_LIMIT2147483647;
else if (__kmp_nesting_mode > 1)
  __kmp_nesting_mode_nlevels = __kmp_nesting_mode;

if (__kmp_topology) { // use topology info
  int loc, hw_level;
  for (loc = 0, hw_level = 0; hw_level < __kmp_topology->get_depth() &&
                              loc < __kmp_nesting_mode_nlevels;
       loc++, hw_level++) {
    __kmp_nesting_nth_level[loc] = __kmp_topology->get_ratio(hw_level);
    if (__kmp_nesting_nth_level[loc] == 1)
      loc--;
  }
  // Make sure all cores are used
  if (__kmp_nesting_mode > 1 && loc > 1) {
    int core_level = __kmp_topology->get_level(KMP_HW_CORE);
    int num_cores = __kmp_topology->get_count(core_level);
    int upper_levels = 1;
    for (int level = 0; level < loc - 1; ++level)
      upper_levels *= __kmp_nesting_nth_level[level];
    if (upper_levels * __kmp_nesting_nth_level[loc - 1] < num_cores)
      __kmp_nesting_nth_level[loc - 1] =
          num_cores / __kmp_nesting_nth_level[loc - 2];
  }
  __kmp_nesting_mode_nlevels = loc;
  __kmp_nested_nth.used = __kmp_nesting_mode_nlevels;
} else { // no topology info available; provide a reasonable guesstimation
  if (__kmp_avail_proc >= 4) {
    __kmp_nesting_nth_level[0] = __kmp_avail_proc / 2;
    __kmp_nesting_nth_level[1] = 2;
    __kmp_nesting_mode_nlevels = 2;
  } else {
    __kmp_nesting_nth_level[0] = __kmp_avail_proc;
    __kmp_nesting_mode_nlevels = 1;
  }
  __kmp_nested_nth.used = __kmp_nesting_mode_nlevels;
}
for (int i = 0; i < __kmp_nesting_mode_nlevels; ++i) {
  __kmp_nested_nth.nth[i] = __kmp_nesting_nth_level[i];
}
set__nproc(thread, __kmp_nesting_nth_level[0])(((thread)->th.th_current_task->td_icvs.nproc) = (__kmp_nesting_nth_level
[0]));
if (__kmp_nesting_mode > 1 && __kmp_nesting_mode_nlevels > __kmp_nesting_mode)
  __kmp_nesting_mode_nlevels = __kmp_nesting_mode;
if (get__max_active_levels(thread)((thread)->th.th_current_task->td_icvs.max_active_levels
) > 1) {
  // if max levels was set, set nesting mode levels to same
  __kmp_nesting_mode_nlevels = get__max_active_levels(thread)((thread)->th.th_current_task->td_icvs.max_active_levels
);
}
if (__kmp_nesting_mode == 1) // turn on nesting for this case only
  set__max_active_levels(thread, __kmp_nesting_mode_nlevels)(((thread)->th.th_current_task->td_icvs.max_active_levels
) = (__kmp_nesting_mode_nlevels));
9241}

9243// Empty symbols to export (see exports_so.txt) when feature is disabled
9244extern "C" {
9245#if !KMP_STATS_ENABLED0
9246void __kmp_reset_stats() {}
9247#endif
9248#if !USE_DEBUGGER0
9249int __kmp_omp_debug_struct_info = FALSE0;
9250int __kmp_debugging = FALSE0;
9251#endif
9252#if !USE_ITT_BUILD1 || !USE_ITT_NOTIFY1
9253void __kmp_itt_fini_ittlib() {}
9254void __kmp_itt_init_ittlib() {}
9255#endif
9256}

9258// end of file