14 #include "kmp_affinity.h"
18 #include "kmp_wrapper_getpid.h"
19 #if KMP_USE_HIER_SCHED
20 #include "kmp_dispatch_hier.h"
24 static hierarchy_info machine_hierarchy;
26 void __kmp_cleanup_hierarchy() { machine_hierarchy.fini(); }
28 void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar) {
32 if (TCR_1(machine_hierarchy.uninitialized))
33 machine_hierarchy.init(NULL, nproc);
36 if (nproc > machine_hierarchy.base_num_threads)
37 machine_hierarchy.resize(nproc);
39 depth = machine_hierarchy.depth;
40 KMP_DEBUG_ASSERT(depth > 0);
42 thr_bar->depth = depth;
43 thr_bar->base_leaf_kids = (kmp_uint8)machine_hierarchy.numPerLevel[0] - 1;
44 thr_bar->skip_per_level = machine_hierarchy.skipPerLevel;
47 #if KMP_AFFINITY_SUPPORTED
49 bool KMPAffinity::picked_api =
false;
51 void *KMPAffinity::Mask::operator
new(
size_t n) {
return __kmp_allocate(n); }
52 void *KMPAffinity::Mask::operator
new[](
size_t n) {
return __kmp_allocate(n); }
53 void KMPAffinity::Mask::operator
delete(
void *p) { __kmp_free(p); }
54 void KMPAffinity::Mask::operator
delete[](
void *p) { __kmp_free(p); }
55 void *KMPAffinity::operator
new(
size_t n) {
return __kmp_allocate(n); }
56 void KMPAffinity::operator
delete(
void *p) { __kmp_free(p); }
58 void KMPAffinity::pick_api() {
59 KMPAffinity *affinity_dispatch;
65 if (__kmp_affinity_top_method == affinity_top_method_hwloc &&
66 __kmp_affinity_type != affinity_disabled) {
67 affinity_dispatch =
new KMPHwlocAffinity();
71 affinity_dispatch =
new KMPNativeAffinity();
73 __kmp_affinity_dispatch = affinity_dispatch;
77 void KMPAffinity::destroy_api() {
78 if (__kmp_affinity_dispatch != NULL) {
79 delete __kmp_affinity_dispatch;
80 __kmp_affinity_dispatch = NULL;
85 #define KMP_ADVANCE_SCAN(scan) \
86 while (*scan != '\0') { \
94 char *__kmp_affinity_print_mask(
char *buf,
int buf_len,
95 kmp_affin_mask_t *mask) {
96 int start = 0, finish = 0, previous = 0;
99 KMP_ASSERT(buf_len >= 40);
102 char *end = buf + buf_len - 1;
105 if (mask->begin() == mask->end()) {
106 KMP_SNPRINTF(scan, end - scan + 1,
"{<empty>}");
107 KMP_ADVANCE_SCAN(scan);
108 KMP_ASSERT(scan <= end);
113 start = mask->begin();
117 for (finish = mask->next(start), previous = start;
118 finish == previous + 1 && finish != mask->end();
119 finish = mask->next(finish)) {
126 KMP_SNPRINTF(scan, end - scan + 1,
"%s",
",");
127 KMP_ADVANCE_SCAN(scan);
132 if (previous - start > 1) {
133 KMP_SNPRINTF(scan, end - scan + 1,
"%d-%d", static_cast<int>(start),
134 static_cast<int>(previous));
137 KMP_SNPRINTF(scan, end - scan + 1,
"%d", static_cast<int>(start));
138 KMP_ADVANCE_SCAN(scan);
139 if (previous - start > 0) {
140 KMP_SNPRINTF(scan, end - scan + 1,
",%d", static_cast<int>(previous));
143 KMP_ADVANCE_SCAN(scan);
146 if (start == mask->end())
154 KMP_ASSERT(scan <= end);
157 #undef KMP_ADVANCE_SCAN
163 kmp_str_buf_t *__kmp_affinity_str_buf_mask(kmp_str_buf_t *buf,
164 kmp_affin_mask_t *mask) {
165 int start = 0, finish = 0, previous = 0;
170 __kmp_str_buf_clear(buf);
173 if (mask->begin() == mask->end()) {
174 __kmp_str_buf_print(buf,
"%s",
"{<empty>}");
179 start = mask->begin();
183 for (finish = mask->next(start), previous = start;
184 finish == previous + 1 && finish != mask->end();
185 finish = mask->next(finish)) {
192 __kmp_str_buf_print(buf,
"%s",
",");
197 if (previous - start > 1) {
198 __kmp_str_buf_print(buf,
"%d-%d", static_cast<int>(start),
199 static_cast<int>(previous));
202 __kmp_str_buf_print(buf,
"%d", static_cast<int>(start));
203 if (previous - start > 0) {
204 __kmp_str_buf_print(buf,
",%d", static_cast<int>(previous));
209 if (start == mask->end())
215 void __kmp_affinity_entire_machine_mask(kmp_affin_mask_t *mask) {
218 #if KMP_GROUP_AFFINITY
220 if (__kmp_num_proc_groups > 1) {
222 KMP_DEBUG_ASSERT(__kmp_GetActiveProcessorCount != NULL);
223 for (group = 0; group < __kmp_num_proc_groups; group++) {
225 int num = __kmp_GetActiveProcessorCount(group);
226 for (i = 0; i < num; i++) {
227 KMP_CPU_SET(i + group * (CHAR_BIT *
sizeof(DWORD_PTR)), mask);
236 for (proc = 0; proc < __kmp_xproc; proc++) {
237 KMP_CPU_SET(proc, mask);
253 static void __kmp_affinity_assign_child_nums(AddrUnsPair *address2os,
255 KMP_DEBUG_ASSERT(numAddrs > 0);
256 int depth = address2os->first.depth;
257 unsigned *counts = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
258 unsigned *lastLabel = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
260 for (labCt = 0; labCt < depth; labCt++) {
261 address2os[0].first.childNums[labCt] = counts[labCt] = 0;
262 lastLabel[labCt] = address2os[0].first.labels[labCt];
265 for (i = 1; i < numAddrs; i++) {
266 for (labCt = 0; labCt < depth; labCt++) {
267 if (address2os[i].first.labels[labCt] != lastLabel[labCt]) {
269 for (labCt2 = labCt + 1; labCt2 < depth; labCt2++) {
271 lastLabel[labCt2] = address2os[i].first.labels[labCt2];
274 lastLabel[labCt] = address2os[i].first.labels[labCt];
278 for (labCt = 0; labCt < depth; labCt++) {
279 address2os[i].first.childNums[labCt] = counts[labCt];
281 for (; labCt < (int)Address::maxDepth; labCt++) {
282 address2os[i].first.childNums[labCt] = 0;
285 __kmp_free(lastLabel);
300 kmp_affin_mask_t *__kmp_affin_fullMask = NULL;
302 static int nCoresPerPkg, nPackages;
303 static int __kmp_nThreadsPerCore;
304 #ifndef KMP_DFLT_NTH_CORES
305 static int __kmp_ncores;
307 static int *__kmp_pu_os_idx = NULL;
313 inline static bool __kmp_affinity_uniform_topology() {
314 return __kmp_avail_proc == (__kmp_nThreadsPerCore * nCoresPerPkg * nPackages);
319 static void __kmp_affinity_print_topology(AddrUnsPair *address2os,
int len,
320 int depth,
int pkgLevel,
321 int coreLevel,
int threadLevel) {
324 KMP_INFORM(OSProcToPhysicalThreadMap,
"KMP_AFFINITY");
325 for (proc = 0; proc < len; proc++) {
328 __kmp_str_buf_init(&buf);
329 for (level = 0; level < depth; level++) {
330 if (level == threadLevel) {
331 __kmp_str_buf_print(&buf,
"%s ", KMP_I18N_STR(Thread));
332 }
else if (level == coreLevel) {
333 __kmp_str_buf_print(&buf,
"%s ", KMP_I18N_STR(Core));
334 }
else if (level == pkgLevel) {
335 __kmp_str_buf_print(&buf,
"%s ", KMP_I18N_STR(Package));
336 }
else if (level > pkgLevel) {
337 __kmp_str_buf_print(&buf,
"%s_%d ", KMP_I18N_STR(Node),
338 level - pkgLevel - 1);
340 __kmp_str_buf_print(&buf,
"L%d ", level);
342 __kmp_str_buf_print(&buf,
"%d ", address2os[proc].first.labels[level]);
344 KMP_INFORM(OSProcMapToPack,
"KMP_AFFINITY", address2os[proc].second,
346 __kmp_str_buf_free(&buf);
352 static void __kmp_affinity_print_hwloc_tp(AddrUnsPair *addrP,
int len,
353 int depth,
int *levels) {
356 __kmp_str_buf_init(&buf);
357 KMP_INFORM(OSProcToPhysicalThreadMap,
"KMP_AFFINITY");
358 for (proc = 0; proc < len; proc++) {
359 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Package),
360 addrP[proc].first.labels[0]);
364 if (__kmp_numa_detected)
366 if (levels[level++] > 0)
367 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Node),
368 addrP[proc].first.labels[label++]);
369 if (__kmp_tile_depth > 0)
371 if (levels[level++] > 0)
372 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Tile),
373 addrP[proc].first.labels[label++]);
374 if (levels[level++] > 0)
376 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Core),
377 addrP[proc].first.labels[label++]);
378 if (levels[level++] > 0)
380 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Thread),
381 addrP[proc].first.labels[label++]);
382 KMP_DEBUG_ASSERT(label == depth);
384 KMP_INFORM(OSProcMapToPack,
"KMP_AFFINITY", addrP[proc].second, buf.str);
385 __kmp_str_buf_clear(&buf);
387 __kmp_str_buf_free(&buf);
390 static int nNodePerPkg, nTilePerPkg, nTilePerNode, nCorePerNode, nCorePerTile;
397 static int __kmp_affinity_remove_radix_one_levels(AddrUnsPair *addrP,
int nTh,
398 int depth,
int *levels) {
402 int new_depth = depth;
403 for (level = depth - 1; level > 0; --level) {
406 for (i = 1; i < nTh; ++i) {
407 if (addrP[0].first.labels[level] != addrP[i].first.labels[level]) {
413 if (!radix1_detected)
418 if (level == new_depth) {
421 for (i = 0; i < nTh; ++i) {
422 addrP[i].first.depth--;
427 for (j = level; j < new_depth; ++j) {
428 for (i = 0; i < nTh; ++i) {
429 addrP[i].first.labels[j] = addrP[i].first.labels[j + 1];
430 addrP[i].first.depth--;
443 static int __kmp_hwloc_get_nobjs_under_obj(hwloc_obj_t obj,
444 hwloc_obj_type_t type) {
447 for (first = hwloc_get_obj_below_by_type(__kmp_hwloc_topology, obj->type,
448 obj->logical_index, type, 0);
450 hwloc_get_ancestor_obj_by_type(__kmp_hwloc_topology, obj->type, first) ==
452 first = hwloc_get_next_obj_by_type(__kmp_hwloc_topology, first->type,
459 static int __kmp_hwloc_count_children_by_depth(hwloc_topology_t t,
461 kmp_hwloc_depth_t depth,
463 if (o->depth == depth) {
469 for (
unsigned i = 0; i < o->arity; i++)
470 sum += __kmp_hwloc_count_children_by_depth(t, o->children[i], depth, f);
474 static int __kmp_hwloc_count_children_by_type(hwloc_topology_t t, hwloc_obj_t o,
475 hwloc_obj_type_t type,
477 if (!hwloc_compare_types(o->type, type)) {
483 for (
unsigned i = 0; i < o->arity; i++)
484 sum += __kmp_hwloc_count_children_by_type(t, o->children[i], type, f);
488 static int __kmp_hwloc_process_obj_core_pu(AddrUnsPair *addrPair,
490 int &num_active_cores,
491 hwloc_obj_t obj,
int depth,
493 hwloc_obj_t core = NULL;
494 hwloc_topology_t &tp = __kmp_hwloc_topology;
495 int NC = __kmp_hwloc_count_children_by_type(tp, obj, HWLOC_OBJ_CORE, &core);
496 for (
int core_id = 0; core_id < NC; ++core_id, core = core->next_cousin) {
497 hwloc_obj_t pu = NULL;
498 KMP_DEBUG_ASSERT(core != NULL);
499 int num_active_threads = 0;
500 int NT = __kmp_hwloc_count_children_by_type(tp, core, HWLOC_OBJ_PU, &pu);
502 for (
int pu_id = 0; pu_id < NT; ++pu_id, pu = pu->next_cousin) {
503 KMP_DEBUG_ASSERT(pu != NULL);
504 if (!KMP_CPU_ISSET(pu->os_index, __kmp_affin_fullMask))
506 Address addr(depth + 2);
507 KA_TRACE(20, (
"Hwloc inserting %d (%d) %d (%d) %d (%d) into address2os\n",
508 obj->os_index, obj->logical_index, core->os_index,
509 core->logical_index, pu->os_index, pu->logical_index));
510 for (
int i = 0; i < depth; ++i)
511 addr.labels[i] = labels[i];
512 addr.labels[depth] = core_id;
513 addr.labels[depth + 1] = pu_id;
514 addrPair[nActiveThreads] = AddrUnsPair(addr, pu->os_index);
515 __kmp_pu_os_idx[nActiveThreads] = pu->os_index;
517 ++num_active_threads;
519 if (num_active_threads) {
522 if (num_active_threads > __kmp_nThreadsPerCore)
523 __kmp_nThreadsPerCore = num_active_threads;
531 static int __kmp_hwloc_check_numa() {
532 hwloc_topology_t &tp = __kmp_hwloc_topology;
533 hwloc_obj_t hT, hC, hL, hN, hS;
534 int depth, l2cache_depth, package_depth;
537 hT = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PU, 0);
542 hN = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hT);
543 hS = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hT);
544 KMP_DEBUG_ASSERT(hS != NULL);
545 if (hN != NULL && hN->depth > hS->depth) {
546 __kmp_numa_detected = TRUE;
547 if (__kmp_affinity_gran == affinity_gran_node) {
548 __kmp_affinity_gran = affinity_gran_numa;
552 package_depth = hwloc_get_type_depth(tp, HWLOC_OBJ_PACKAGE);
553 l2cache_depth = hwloc_get_cache_type_depth(tp, 2, HWLOC_OBJ_CACHE_UNIFIED);
555 depth = (l2cache_depth < package_depth) ? package_depth : l2cache_depth;
556 hL = hwloc_get_ancestor_obj_by_depth(tp, depth, hT);
559 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC) > 1)
560 __kmp_tile_depth = depth;
564 static int __kmp_affinity_create_hwloc_map(AddrUnsPair **address2os,
565 kmp_i18n_id_t *
const msg_id) {
566 hwloc_topology_t &tp = __kmp_hwloc_topology;
568 *msg_id = kmp_i18n_null;
571 kmp_affin_mask_t *oldMask;
572 KMP_CPU_ALLOC(oldMask);
573 __kmp_get_system_affinity(oldMask, TRUE);
574 __kmp_hwloc_check_numa();
576 if (!KMP_AFFINITY_CAPABLE()) {
579 KMP_ASSERT(__kmp_affinity_type == affinity_none);
581 nCoresPerPkg = __kmp_hwloc_get_nobjs_under_obj(
582 hwloc_get_obj_by_type(tp, HWLOC_OBJ_PACKAGE, 0), HWLOC_OBJ_CORE);
583 __kmp_nThreadsPerCore = __kmp_hwloc_get_nobjs_under_obj(
584 hwloc_get_obj_by_type(tp, HWLOC_OBJ_CORE, 0), HWLOC_OBJ_PU);
585 __kmp_ncores = __kmp_xproc / __kmp_nThreadsPerCore;
586 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
587 if (__kmp_affinity_verbose) {
588 KMP_INFORM(AffNotCapableUseLocCpuidL11,
"KMP_AFFINITY");
589 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
590 if (__kmp_affinity_uniform_topology()) {
591 KMP_INFORM(Uniform,
"KMP_AFFINITY");
593 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
595 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
596 __kmp_nThreadsPerCore, __kmp_ncores);
598 KMP_CPU_FREE(oldMask);
603 int levels[5] = {0, 1, 2, 3, 4};
605 if (__kmp_numa_detected)
607 if (__kmp_tile_depth)
611 AddrUnsPair *retval =
612 (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * __kmp_avail_proc);
613 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
614 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
621 hwloc_obj_t socket, node, tile;
622 int nActiveThreads = 0;
625 __kmp_ncores = nPackages = nCoresPerPkg = __kmp_nThreadsPerCore = 0;
626 nNodePerPkg = nTilePerPkg = nTilePerNode = nCorePerNode = nCorePerTile = 0;
627 for (socket = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PACKAGE, 0); socket != NULL;
628 socket = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PACKAGE, socket),
630 labels[0] = socket_id;
631 if (__kmp_numa_detected) {
633 int n_active_nodes = 0;
635 NN = __kmp_hwloc_count_children_by_type(tp, socket, HWLOC_OBJ_NUMANODE,
637 for (
int node_id = 0; node_id < NN; ++node_id, node = node->next_cousin) {
639 if (__kmp_tile_depth) {
642 int n_active_tiles = 0;
644 NT = __kmp_hwloc_count_children_by_depth(tp, node, __kmp_tile_depth,
646 for (
int tl_id = 0; tl_id < NT; ++tl_id, tile = tile->next_cousin) {
648 int n_active_cores = 0;
649 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
650 n_active_cores, tile, 3, labels);
651 if (n_active_cores) {
653 if (n_active_cores > nCorePerTile)
654 nCorePerTile = n_active_cores;
657 if (n_active_tiles) {
659 if (n_active_tiles > nTilePerNode)
660 nTilePerNode = n_active_tiles;
664 int n_active_cores = 0;
665 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
666 n_active_cores, node, 2, labels);
667 if (n_active_cores) {
669 if (n_active_cores > nCorePerNode)
670 nCorePerNode = n_active_cores;
674 if (n_active_nodes) {
676 if (n_active_nodes > nNodePerPkg)
677 nNodePerPkg = n_active_nodes;
680 if (__kmp_tile_depth) {
683 int n_active_tiles = 0;
685 NT = __kmp_hwloc_count_children_by_depth(tp, socket, __kmp_tile_depth,
687 for (
int tl_id = 0; tl_id < NT; ++tl_id, tile = tile->next_cousin) {
689 int n_active_cores = 0;
690 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
691 n_active_cores, tile, 2, labels);
692 if (n_active_cores) {
694 if (n_active_cores > nCorePerTile)
695 nCorePerTile = n_active_cores;
698 if (n_active_tiles) {
700 if (n_active_tiles > nTilePerPkg)
701 nTilePerPkg = n_active_tiles;
705 int n_active_cores = 0;
706 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads, n_active_cores,
708 if (n_active_cores) {
710 if (n_active_cores > nCoresPerPkg)
711 nCoresPerPkg = n_active_cores;
718 KMP_DEBUG_ASSERT(nActiveThreads == __kmp_avail_proc);
719 KMP_ASSERT(nActiveThreads > 0);
720 if (nActiveThreads == 1) {
721 __kmp_ncores = nPackages = 1;
722 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
723 if (__kmp_affinity_verbose) {
724 char buf[KMP_AFFIN_MASK_PRINT_LEN];
725 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
727 KMP_INFORM(AffUsingHwloc,
"KMP_AFFINITY");
728 if (__kmp_affinity_respect_mask) {
729 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
731 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
733 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
734 KMP_INFORM(Uniform,
"KMP_AFFINITY");
735 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
736 __kmp_nThreadsPerCore, __kmp_ncores);
739 if (__kmp_affinity_type == affinity_none) {
741 KMP_CPU_FREE(oldMask);
747 addr.labels[0] = retval[0].first.labels[0];
748 retval[0].first = addr;
750 if (__kmp_affinity_gran_levels < 0) {
751 __kmp_affinity_gran_levels = 0;
754 if (__kmp_affinity_verbose) {
755 __kmp_affinity_print_topology(retval, 1, 1, 0, -1, -1);
758 *address2os = retval;
759 KMP_CPU_FREE(oldMask);
764 qsort(retval, nActiveThreads,
sizeof(*retval),
765 __kmp_affinity_cmp_Address_labels);
768 int nPUs = nPackages * __kmp_nThreadsPerCore;
769 if (__kmp_numa_detected) {
770 if (__kmp_tile_depth) {
771 nPUs *= (nNodePerPkg * nTilePerNode * nCorePerTile);
773 nPUs *= (nNodePerPkg * nCorePerNode);
776 if (__kmp_tile_depth) {
777 nPUs *= (nTilePerPkg * nCorePerTile);
779 nPUs *= nCoresPerPkg;
782 unsigned uniform = (nPUs == nActiveThreads);
785 if (__kmp_affinity_verbose) {
786 char mask[KMP_AFFIN_MASK_PRINT_LEN];
787 __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
788 if (__kmp_affinity_respect_mask) {
789 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", mask);
791 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", mask);
793 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
795 KMP_INFORM(Uniform,
"KMP_AFFINITY");
797 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
799 if (__kmp_numa_detected) {
800 if (__kmp_tile_depth) {
801 KMP_INFORM(TopologyExtraNoTi,
"KMP_AFFINITY", nPackages, nNodePerPkg,
802 nTilePerNode, nCorePerTile, __kmp_nThreadsPerCore,
805 KMP_INFORM(TopologyExtraNode,
"KMP_AFFINITY", nPackages, nNodePerPkg,
806 nCorePerNode, __kmp_nThreadsPerCore, __kmp_ncores);
807 nPUs *= (nNodePerPkg * nCorePerNode);
810 if (__kmp_tile_depth) {
811 KMP_INFORM(TopologyExtraTile,
"KMP_AFFINITY", nPackages, nTilePerPkg,
812 nCorePerTile, __kmp_nThreadsPerCore, __kmp_ncores);
815 __kmp_str_buf_init(&buf);
816 __kmp_str_buf_print(&buf,
"%d", nPackages);
817 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, nCoresPerPkg,
818 __kmp_nThreadsPerCore, __kmp_ncores);
819 __kmp_str_buf_free(&buf);
824 if (__kmp_affinity_type == affinity_none) {
826 KMP_CPU_FREE(oldMask);
830 int depth_full = depth;
833 depth = __kmp_affinity_remove_radix_one_levels(retval, nActiveThreads, depth,
835 KMP_DEBUG_ASSERT(__kmp_affinity_gran != affinity_gran_default);
836 if (__kmp_affinity_gran_levels < 0) {
839 __kmp_affinity_gran_levels = 0;
840 if (__kmp_affinity_gran > affinity_gran_thread) {
841 for (
int i = 1; i <= depth_full; ++i) {
842 if (__kmp_affinity_gran <= i)
844 if (levels[depth_full - i] > 0)
845 __kmp_affinity_gran_levels++;
848 if (__kmp_affinity_gran > affinity_gran_package)
849 __kmp_affinity_gran_levels++;
852 if (__kmp_affinity_verbose)
853 __kmp_affinity_print_hwloc_tp(retval, nActiveThreads, depth, levels);
855 KMP_CPU_FREE(oldMask);
856 *address2os = retval;
859 #endif // KMP_USE_HWLOC
864 static int __kmp_affinity_create_flat_map(AddrUnsPair **address2os,
865 kmp_i18n_id_t *
const msg_id) {
867 *msg_id = kmp_i18n_null;
872 if (!KMP_AFFINITY_CAPABLE()) {
873 KMP_ASSERT(__kmp_affinity_type == affinity_none);
874 __kmp_ncores = nPackages = __kmp_xproc;
875 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
876 if (__kmp_affinity_verbose) {
877 KMP_INFORM(AffFlatTopology,
"KMP_AFFINITY");
878 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
879 KMP_INFORM(Uniform,
"KMP_AFFINITY");
880 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
881 __kmp_nThreadsPerCore, __kmp_ncores);
890 __kmp_ncores = nPackages = __kmp_avail_proc;
891 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
892 if (__kmp_affinity_verbose) {
893 char buf[KMP_AFFIN_MASK_PRINT_LEN];
894 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
895 __kmp_affin_fullMask);
897 KMP_INFORM(AffCapableUseFlat,
"KMP_AFFINITY");
898 if (__kmp_affinity_respect_mask) {
899 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
901 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
903 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
904 KMP_INFORM(Uniform,
"KMP_AFFINITY");
905 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
906 __kmp_nThreadsPerCore, __kmp_ncores);
908 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
909 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
910 if (__kmp_affinity_type == affinity_none) {
913 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
914 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask))
916 __kmp_pu_os_idx[avail_ct++] = i;
923 (AddrUnsPair *)__kmp_allocate(
sizeof(**address2os) * __kmp_avail_proc);
926 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
928 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
931 __kmp_pu_os_idx[avail_ct] = i;
934 (*address2os)[avail_ct++] = AddrUnsPair(addr, i);
936 if (__kmp_affinity_verbose) {
937 KMP_INFORM(OSProcToPackage,
"KMP_AFFINITY");
940 if (__kmp_affinity_gran_levels < 0) {
943 if (__kmp_affinity_gran > affinity_gran_package) {
944 __kmp_affinity_gran_levels = 1;
946 __kmp_affinity_gran_levels = 0;
952 #if KMP_GROUP_AFFINITY
958 static int __kmp_affinity_create_proc_group_map(AddrUnsPair **address2os,
959 kmp_i18n_id_t *
const msg_id) {
961 *msg_id = kmp_i18n_null;
965 if (!KMP_AFFINITY_CAPABLE()) {
972 (AddrUnsPair *)__kmp_allocate(
sizeof(**address2os) * __kmp_avail_proc);
973 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
974 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
977 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
979 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
982 __kmp_pu_os_idx[avail_ct] = i;
984 addr.labels[0] = i / (CHAR_BIT *
sizeof(DWORD_PTR));
985 addr.labels[1] = i % (CHAR_BIT *
sizeof(DWORD_PTR));
986 (*address2os)[avail_ct++] = AddrUnsPair(addr, i);
988 if (__kmp_affinity_verbose) {
989 KMP_INFORM(AffOSProcToGroup,
"KMP_AFFINITY", i, addr.labels[0],
994 if (__kmp_affinity_gran_levels < 0) {
995 if (__kmp_affinity_gran == affinity_gran_group) {
996 __kmp_affinity_gran_levels = 1;
997 }
else if ((__kmp_affinity_gran == affinity_gran_fine) ||
998 (__kmp_affinity_gran == affinity_gran_thread)) {
999 __kmp_affinity_gran_levels = 0;
1001 const char *gran_str = NULL;
1002 if (__kmp_affinity_gran == affinity_gran_core) {
1004 }
else if (__kmp_affinity_gran == affinity_gran_package) {
1005 gran_str =
"package";
1006 }
else if (__kmp_affinity_gran == affinity_gran_node) {
1014 __kmp_affinity_gran_levels = 0;
1022 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1024 static int __kmp_cpuid_mask_width(
int count) {
1027 while ((1 << r) < count)
1032 class apicThreadInfo {
1036 unsigned maxCoresPerPkg;
1037 unsigned maxThreadsPerPkg;
1043 static int __kmp_affinity_cmp_apicThreadInfo_phys_id(
const void *a,
1045 const apicThreadInfo *aa = (
const apicThreadInfo *)a;
1046 const apicThreadInfo *bb = (
const apicThreadInfo *)b;
1047 if (aa->pkgId < bb->pkgId)
1049 if (aa->pkgId > bb->pkgId)
1051 if (aa->coreId < bb->coreId)
1053 if (aa->coreId > bb->coreId)
1055 if (aa->threadId < bb->threadId)
1057 if (aa->threadId > bb->threadId)
1066 static int __kmp_affinity_create_apicid_map(AddrUnsPair **address2os,
1067 kmp_i18n_id_t *
const msg_id) {
1070 *msg_id = kmp_i18n_null;
1073 __kmp_x86_cpuid(0, 0, &buf);
1075 *msg_id = kmp_i18n_str_NoLeaf4Support;
1084 if (!KMP_AFFINITY_CAPABLE()) {
1087 KMP_ASSERT(__kmp_affinity_type == affinity_none);
1093 __kmp_x86_cpuid(1, 0, &buf);
1094 int maxThreadsPerPkg = (buf.ebx >> 16) & 0xff;
1095 if (maxThreadsPerPkg == 0) {
1096 maxThreadsPerPkg = 1;
1110 __kmp_x86_cpuid(0, 0, &buf);
1112 __kmp_x86_cpuid(4, 0, &buf);
1113 nCoresPerPkg = ((buf.eax >> 26) & 0x3f) + 1;
1131 __kmp_ncores = __kmp_xproc;
1132 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
1133 __kmp_nThreadsPerCore = 1;
1134 if (__kmp_affinity_verbose) {
1135 KMP_INFORM(AffNotCapableUseLocCpuid,
"KMP_AFFINITY");
1136 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1137 if (__kmp_affinity_uniform_topology()) {
1138 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1140 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1142 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1143 __kmp_nThreadsPerCore, __kmp_ncores);
1153 kmp_affin_mask_t *oldMask;
1154 KMP_CPU_ALLOC(oldMask);
1155 KMP_ASSERT(oldMask != NULL);
1156 __kmp_get_system_affinity(oldMask, TRUE);
1184 apicThreadInfo *threadInfo = (apicThreadInfo *)__kmp_allocate(
1185 __kmp_avail_proc *
sizeof(apicThreadInfo));
1186 unsigned nApics = 0;
1187 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
1189 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
1192 KMP_DEBUG_ASSERT((
int)nApics < __kmp_avail_proc);
1194 __kmp_affinity_dispatch->bind_thread(i);
1195 threadInfo[nApics].osId = i;
1198 __kmp_x86_cpuid(1, 0, &buf);
1199 if (((buf.edx >> 9) & 1) == 0) {
1200 __kmp_set_system_affinity(oldMask, TRUE);
1201 __kmp_free(threadInfo);
1202 KMP_CPU_FREE(oldMask);
1203 *msg_id = kmp_i18n_str_ApicNotPresent;
1206 threadInfo[nApics].apicId = (buf.ebx >> 24) & 0xff;
1207 threadInfo[nApics].maxThreadsPerPkg = (buf.ebx >> 16) & 0xff;
1208 if (threadInfo[nApics].maxThreadsPerPkg == 0) {
1209 threadInfo[nApics].maxThreadsPerPkg = 1;
1218 __kmp_x86_cpuid(0, 0, &buf);
1220 __kmp_x86_cpuid(4, 0, &buf);
1221 threadInfo[nApics].maxCoresPerPkg = ((buf.eax >> 26) & 0x3f) + 1;
1223 threadInfo[nApics].maxCoresPerPkg = 1;
1227 int widthCT = __kmp_cpuid_mask_width(threadInfo[nApics].maxThreadsPerPkg);
1228 threadInfo[nApics].pkgId = threadInfo[nApics].apicId >> widthCT;
1230 int widthC = __kmp_cpuid_mask_width(threadInfo[nApics].maxCoresPerPkg);
1231 int widthT = widthCT - widthC;
1236 __kmp_set_system_affinity(oldMask, TRUE);
1237 __kmp_free(threadInfo);
1238 KMP_CPU_FREE(oldMask);
1239 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1243 int maskC = (1 << widthC) - 1;
1244 threadInfo[nApics].coreId = (threadInfo[nApics].apicId >> widthT) & maskC;
1246 int maskT = (1 << widthT) - 1;
1247 threadInfo[nApics].threadId = threadInfo[nApics].apicId & maskT;
1254 __kmp_set_system_affinity(oldMask, TRUE);
1263 KMP_ASSERT(nApics > 0);
1265 __kmp_ncores = nPackages = 1;
1266 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
1267 if (__kmp_affinity_verbose) {
1268 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1269 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1271 KMP_INFORM(AffUseGlobCpuid,
"KMP_AFFINITY");
1272 if (__kmp_affinity_respect_mask) {
1273 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
1275 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
1277 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1278 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1279 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1280 __kmp_nThreadsPerCore, __kmp_ncores);
1283 if (__kmp_affinity_type == affinity_none) {
1284 __kmp_free(threadInfo);
1285 KMP_CPU_FREE(oldMask);
1289 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair));
1291 addr.labels[0] = threadInfo[0].pkgId;
1292 (*address2os)[0] = AddrUnsPair(addr, threadInfo[0].osId);
1294 if (__kmp_affinity_gran_levels < 0) {
1295 __kmp_affinity_gran_levels = 0;
1298 if (__kmp_affinity_verbose) {
1299 __kmp_affinity_print_topology(*address2os, 1, 1, 0, -1, -1);
1302 __kmp_free(threadInfo);
1303 KMP_CPU_FREE(oldMask);
1308 qsort(threadInfo, nApics,
sizeof(*threadInfo),
1309 __kmp_affinity_cmp_apicThreadInfo_phys_id);
1326 __kmp_nThreadsPerCore = 1;
1327 unsigned nCores = 1;
1330 unsigned lastPkgId = threadInfo[0].pkgId;
1331 unsigned coreCt = 1;
1332 unsigned lastCoreId = threadInfo[0].coreId;
1333 unsigned threadCt = 1;
1334 unsigned lastThreadId = threadInfo[0].threadId;
1337 unsigned prevMaxCoresPerPkg = threadInfo[0].maxCoresPerPkg;
1338 unsigned prevMaxThreadsPerPkg = threadInfo[0].maxThreadsPerPkg;
1340 for (i = 1; i < nApics; i++) {
1341 if (threadInfo[i].pkgId != lastPkgId) {
1344 lastPkgId = threadInfo[i].pkgId;
1345 if ((
int)coreCt > nCoresPerPkg)
1346 nCoresPerPkg = coreCt;
1348 lastCoreId = threadInfo[i].coreId;
1349 if ((
int)threadCt > __kmp_nThreadsPerCore)
1350 __kmp_nThreadsPerCore = threadCt;
1352 lastThreadId = threadInfo[i].threadId;
1356 prevMaxCoresPerPkg = threadInfo[i].maxCoresPerPkg;
1357 prevMaxThreadsPerPkg = threadInfo[i].maxThreadsPerPkg;
1361 if (threadInfo[i].coreId != lastCoreId) {
1364 lastCoreId = threadInfo[i].coreId;
1365 if ((
int)threadCt > __kmp_nThreadsPerCore)
1366 __kmp_nThreadsPerCore = threadCt;
1368 lastThreadId = threadInfo[i].threadId;
1369 }
else if (threadInfo[i].threadId != lastThreadId) {
1371 lastThreadId = threadInfo[i].threadId;
1373 __kmp_free(threadInfo);
1374 KMP_CPU_FREE(oldMask);
1375 *msg_id = kmp_i18n_str_LegacyApicIDsNotUnique;
1381 if ((prevMaxCoresPerPkg != threadInfo[i].maxCoresPerPkg) ||
1382 (prevMaxThreadsPerPkg != threadInfo[i].maxThreadsPerPkg)) {
1383 __kmp_free(threadInfo);
1384 KMP_CPU_FREE(oldMask);
1385 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1390 if ((
int)coreCt > nCoresPerPkg)
1391 nCoresPerPkg = coreCt;
1392 if ((
int)threadCt > __kmp_nThreadsPerCore)
1393 __kmp_nThreadsPerCore = threadCt;
1399 __kmp_ncores = nCores;
1400 if (__kmp_affinity_verbose) {
1401 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1402 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1404 KMP_INFORM(AffUseGlobCpuid,
"KMP_AFFINITY");
1405 if (__kmp_affinity_respect_mask) {
1406 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
1408 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
1410 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1411 if (__kmp_affinity_uniform_topology()) {
1412 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1414 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1416 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1417 __kmp_nThreadsPerCore, __kmp_ncores);
1419 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
1420 KMP_DEBUG_ASSERT(nApics == (
unsigned)__kmp_avail_proc);
1421 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
1422 for (i = 0; i < nApics; ++i) {
1423 __kmp_pu_os_idx[i] = threadInfo[i].osId;
1425 if (__kmp_affinity_type == affinity_none) {
1426 __kmp_free(threadInfo);
1427 KMP_CPU_FREE(oldMask);
1435 int coreLevel = (nCoresPerPkg <= 1) ? -1 : 1;
1437 (__kmp_nThreadsPerCore <= 1) ? -1 : ((coreLevel >= 0) ? 2 : 1);
1438 unsigned depth = (pkgLevel >= 0) + (coreLevel >= 0) + (threadLevel >= 0);
1440 KMP_ASSERT(depth > 0);
1441 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * nApics);
1443 for (i = 0; i < nApics; ++i) {
1444 Address addr(depth);
1445 unsigned os = threadInfo[i].osId;
1448 if (pkgLevel >= 0) {
1449 addr.labels[d++] = threadInfo[i].pkgId;
1451 if (coreLevel >= 0) {
1452 addr.labels[d++] = threadInfo[i].coreId;
1454 if (threadLevel >= 0) {
1455 addr.labels[d++] = threadInfo[i].threadId;
1457 (*address2os)[i] = AddrUnsPair(addr, os);
1460 if (__kmp_affinity_gran_levels < 0) {
1463 __kmp_affinity_gran_levels = 0;
1464 if ((threadLevel >= 0) && (__kmp_affinity_gran > affinity_gran_thread)) {
1465 __kmp_affinity_gran_levels++;
1467 if ((coreLevel >= 0) && (__kmp_affinity_gran > affinity_gran_core)) {
1468 __kmp_affinity_gran_levels++;
1470 if ((pkgLevel >= 0) && (__kmp_affinity_gran > affinity_gran_package)) {
1471 __kmp_affinity_gran_levels++;
1475 if (__kmp_affinity_verbose) {
1476 __kmp_affinity_print_topology(*address2os, nApics, depth, pkgLevel,
1477 coreLevel, threadLevel);
1480 __kmp_free(threadInfo);
1481 KMP_CPU_FREE(oldMask);
1488 static int __kmp_affinity_create_x2apicid_map(AddrUnsPair **address2os,
1489 kmp_i18n_id_t *
const msg_id) {
1492 *msg_id = kmp_i18n_null;
1495 __kmp_x86_cpuid(0, 0, &buf);
1497 *msg_id = kmp_i18n_str_NoLeaf11Support;
1500 __kmp_x86_cpuid(11, 0, &buf);
1502 *msg_id = kmp_i18n_str_NoLeaf11Support;
1511 int threadLevel = -1;
1514 __kmp_nThreadsPerCore = nCoresPerPkg = nPackages = 1;
1516 for (level = 0;; level++) {
1527 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1530 __kmp_x86_cpuid(11, level, &buf);
1539 int kind = (buf.ecx >> 8) & 0xff;
1542 threadLevel = level;
1545 __kmp_nThreadsPerCore = buf.ebx & 0xffff;
1546 if (__kmp_nThreadsPerCore == 0) {
1547 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1550 }
else if (kind == 2) {
1554 nCoresPerPkg = buf.ebx & 0xffff;
1555 if (nCoresPerPkg == 0) {
1556 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1561 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1564 if (pkgLevel >= 0) {
1568 nPackages = buf.ebx & 0xffff;
1569 if (nPackages == 0) {
1570 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1581 if (threadLevel >= 0) {
1582 threadLevel = depth - threadLevel - 1;
1584 if (coreLevel >= 0) {
1585 coreLevel = depth - coreLevel - 1;
1587 KMP_DEBUG_ASSERT(pkgLevel >= 0);
1588 pkgLevel = depth - pkgLevel - 1;
1595 if (!KMP_AFFINITY_CAPABLE()) {
1598 KMP_ASSERT(__kmp_affinity_type == affinity_none);
1600 __kmp_ncores = __kmp_xproc / __kmp_nThreadsPerCore;
1601 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
1602 if (__kmp_affinity_verbose) {
1603 KMP_INFORM(AffNotCapableUseLocCpuidL11,
"KMP_AFFINITY");
1604 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1605 if (__kmp_affinity_uniform_topology()) {
1606 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1608 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1610 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1611 __kmp_nThreadsPerCore, __kmp_ncores);
1621 kmp_affin_mask_t *oldMask;
1622 KMP_CPU_ALLOC(oldMask);
1623 __kmp_get_system_affinity(oldMask, TRUE);
1626 AddrUnsPair *retval =
1627 (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * __kmp_avail_proc);
1633 KMP_CPU_SET_ITERATE(proc, __kmp_affin_fullMask) {
1635 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
1638 KMP_DEBUG_ASSERT(nApics < __kmp_avail_proc);
1640 __kmp_affinity_dispatch->bind_thread(proc);
1643 Address addr(depth);
1646 for (level = 0; level < depth; level++) {
1647 __kmp_x86_cpuid(11, level, &buf);
1648 unsigned apicId = buf.edx;
1650 if (level != depth - 1) {
1651 KMP_CPU_FREE(oldMask);
1652 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1655 addr.labels[depth - level - 1] = apicId >> prev_shift;
1659 int shift = buf.eax & 0x1f;
1660 int mask = (1 << shift) - 1;
1661 addr.labels[depth - level - 1] = (apicId & mask) >> prev_shift;
1664 if (level != depth) {
1665 KMP_CPU_FREE(oldMask);
1666 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1670 retval[nApics] = AddrUnsPair(addr, proc);
1676 __kmp_set_system_affinity(oldMask, TRUE);
1679 KMP_ASSERT(nApics > 0);
1681 __kmp_ncores = nPackages = 1;
1682 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
1683 if (__kmp_affinity_verbose) {
1684 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1685 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1687 KMP_INFORM(AffUseGlobCpuidL11,
"KMP_AFFINITY");
1688 if (__kmp_affinity_respect_mask) {
1689 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
1691 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
1693 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1694 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1695 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1696 __kmp_nThreadsPerCore, __kmp_ncores);
1699 if (__kmp_affinity_type == affinity_none) {
1701 KMP_CPU_FREE(oldMask);
1707 addr.labels[0] = retval[0].first.labels[pkgLevel];
1708 retval[0].first = addr;
1710 if (__kmp_affinity_gran_levels < 0) {
1711 __kmp_affinity_gran_levels = 0;
1714 if (__kmp_affinity_verbose) {
1715 __kmp_affinity_print_topology(retval, 1, 1, 0, -1, -1);
1718 *address2os = retval;
1719 KMP_CPU_FREE(oldMask);
1724 qsort(retval, nApics,
sizeof(*retval), __kmp_affinity_cmp_Address_labels);
1727 unsigned *totals = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1728 unsigned *counts = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1729 unsigned *maxCt = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1730 unsigned *last = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1731 for (level = 0; level < depth; level++) {
1735 last[level] = retval[0].first.labels[level];
1742 for (proc = 1; (int)proc < nApics; proc++) {
1744 for (level = 0; level < depth; level++) {
1745 if (retval[proc].first.labels[level] != last[level]) {
1747 for (j = level + 1; j < depth; j++) {
1758 last[j] = retval[proc].first.labels[j];
1762 if (counts[level] > maxCt[level]) {
1763 maxCt[level] = counts[level];
1765 last[level] = retval[proc].first.labels[level];
1767 }
else if (level == depth - 1) {
1773 KMP_CPU_FREE(oldMask);
1774 *msg_id = kmp_i18n_str_x2ApicIDsNotUnique;
1784 if (threadLevel >= 0) {
1785 __kmp_nThreadsPerCore = maxCt[threadLevel];
1787 __kmp_nThreadsPerCore = 1;
1789 nPackages = totals[pkgLevel];
1791 if (coreLevel >= 0) {
1792 __kmp_ncores = totals[coreLevel];
1793 nCoresPerPkg = maxCt[coreLevel];
1795 __kmp_ncores = nPackages;
1800 unsigned prod = maxCt[0];
1801 for (level = 1; level < depth; level++) {
1802 prod *= maxCt[level];
1804 bool uniform = (prod == totals[level - 1]);
1807 if (__kmp_affinity_verbose) {
1808 char mask[KMP_AFFIN_MASK_PRINT_LEN];
1809 __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1811 KMP_INFORM(AffUseGlobCpuidL11,
"KMP_AFFINITY");
1812 if (__kmp_affinity_respect_mask) {
1813 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", mask);
1815 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", mask);
1817 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1819 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1821 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1825 __kmp_str_buf_init(&buf);
1827 __kmp_str_buf_print(&buf,
"%d", totals[0]);
1828 for (level = 1; level <= pkgLevel; level++) {
1829 __kmp_str_buf_print(&buf,
" x %d", maxCt[level]);
1831 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, nCoresPerPkg,
1832 __kmp_nThreadsPerCore, __kmp_ncores);
1834 __kmp_str_buf_free(&buf);
1836 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
1837 KMP_DEBUG_ASSERT(nApics == __kmp_avail_proc);
1838 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
1839 for (proc = 0; (int)proc < nApics; ++proc) {
1840 __kmp_pu_os_idx[proc] = retval[proc].second;
1842 if (__kmp_affinity_type == affinity_none) {
1848 KMP_CPU_FREE(oldMask);
1855 for (level = 0; level < depth; level++) {
1856 if ((maxCt[level] == 1) && (level != pkgLevel)) {
1864 if (new_depth != depth) {
1865 AddrUnsPair *new_retval =
1866 (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * nApics);
1867 for (proc = 0; (int)proc < nApics; proc++) {
1868 Address addr(new_depth);
1869 new_retval[proc] = AddrUnsPair(addr, retval[proc].second);
1872 int newPkgLevel = -1;
1873 int newCoreLevel = -1;
1874 int newThreadLevel = -1;
1875 for (level = 0; level < depth; level++) {
1876 if ((maxCt[level] == 1) && (level != pkgLevel)) {
1880 if (level == pkgLevel) {
1881 newPkgLevel = new_level;
1883 if (level == coreLevel) {
1884 newCoreLevel = new_level;
1886 if (level == threadLevel) {
1887 newThreadLevel = new_level;
1889 for (proc = 0; (int)proc < nApics; proc++) {
1890 new_retval[proc].first.labels[new_level] =
1891 retval[proc].first.labels[level];
1897 retval = new_retval;
1899 pkgLevel = newPkgLevel;
1900 coreLevel = newCoreLevel;
1901 threadLevel = newThreadLevel;
1904 if (__kmp_affinity_gran_levels < 0) {
1907 __kmp_affinity_gran_levels = 0;
1908 if ((threadLevel >= 0) && (__kmp_affinity_gran > affinity_gran_thread)) {
1909 __kmp_affinity_gran_levels++;
1911 if ((coreLevel >= 0) && (__kmp_affinity_gran > affinity_gran_core)) {
1912 __kmp_affinity_gran_levels++;
1914 if (__kmp_affinity_gran > affinity_gran_package) {
1915 __kmp_affinity_gran_levels++;
1919 if (__kmp_affinity_verbose) {
1920 __kmp_affinity_print_topology(retval, nApics, depth, pkgLevel, coreLevel,
1928 KMP_CPU_FREE(oldMask);
1929 *address2os = retval;
1936 #define threadIdIndex 1
1937 #define coreIdIndex 2
1938 #define pkgIdIndex 3
1939 #define nodeIdIndex 4
1941 typedef unsigned *ProcCpuInfo;
1942 static unsigned maxIndex = pkgIdIndex;
1944 static int __kmp_affinity_cmp_ProcCpuInfo_phys_id(
const void *a,
1947 const unsigned *aa = *(
unsigned *
const *)a;
1948 const unsigned *bb = *(
unsigned *
const *)b;
1949 for (i = maxIndex;; i--) {
1960 #if KMP_USE_HIER_SCHED
1962 static void __kmp_dispatch_set_hierarchy_values() {
1968 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1] =
1969 nPackages * nCoresPerPkg * __kmp_nThreadsPerCore;
1970 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L1 + 1] = __kmp_ncores;
1971 #if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
1972 if (__kmp_mic_type >= mic3)
1973 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L2 + 1] = __kmp_ncores / 2;
1975 #endif // KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
1976 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L2 + 1] = __kmp_ncores;
1977 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_L3 + 1] = nPackages;
1978 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_NUMA + 1] = nPackages;
1979 __kmp_hier_max_units[kmp_hier_layer_e::LAYER_LOOP + 1] = 1;
1982 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_THREAD + 1] = 1;
1983 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L1 + 1] =
1984 __kmp_nThreadsPerCore;
1985 #if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
1986 if (__kmp_mic_type >= mic3)
1987 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L2 + 1] =
1988 2 * __kmp_nThreadsPerCore;
1990 #endif // KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
1991 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L2 + 1] =
1992 __kmp_nThreadsPerCore;
1993 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_L3 + 1] =
1994 nCoresPerPkg * __kmp_nThreadsPerCore;
1995 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_NUMA + 1] =
1996 nCoresPerPkg * __kmp_nThreadsPerCore;
1997 __kmp_hier_threads_per[kmp_hier_layer_e::LAYER_LOOP + 1] =
1998 nPackages * nCoresPerPkg * __kmp_nThreadsPerCore;
2003 int __kmp_dispatch_get_index(
int tid, kmp_hier_layer_e type) {
2004 int index = type + 1;
2005 int num_hw_threads = __kmp_hier_max_units[kmp_hier_layer_e::LAYER_THREAD + 1];
2006 KMP_DEBUG_ASSERT(type != kmp_hier_layer_e::LAYER_LAST);
2007 if (type == kmp_hier_layer_e::LAYER_THREAD)
2009 else if (type == kmp_hier_layer_e::LAYER_LOOP)
2011 KMP_DEBUG_ASSERT(__kmp_hier_max_units[index] != 0);
2012 if (tid >= num_hw_threads)
2013 tid = tid % num_hw_threads;
2014 return (tid / __kmp_hier_threads_per[index]) % __kmp_hier_max_units[index];
2018 int __kmp_dispatch_get_t1_per_t2(kmp_hier_layer_e t1, kmp_hier_layer_e t2) {
2021 KMP_DEBUG_ASSERT(i1 <= i2);
2022 KMP_DEBUG_ASSERT(t1 != kmp_hier_layer_e::LAYER_LAST);
2023 KMP_DEBUG_ASSERT(t2 != kmp_hier_layer_e::LAYER_LAST);
2024 KMP_DEBUG_ASSERT(__kmp_hier_threads_per[i1] != 0);
2026 return __kmp_hier_threads_per[i2] / __kmp_hier_threads_per[i1];
2028 #endif // KMP_USE_HIER_SCHED
2032 static int __kmp_affinity_create_cpuinfo_map(AddrUnsPair **address2os,
2034 kmp_i18n_id_t *
const msg_id,
2037 *msg_id = kmp_i18n_null;
2042 unsigned num_records = 0;
2044 buf[
sizeof(buf) - 1] = 1;
2045 if (!fgets(buf,
sizeof(buf), f)) {
2050 char s1[] =
"processor";
2051 if (strncmp(buf, s1,
sizeof(s1) - 1) == 0) {
2058 if (KMP_SSCANF(buf,
"node_%u id", &level) == 1) {
2059 if (nodeIdIndex + level >= maxIndex) {
2060 maxIndex = nodeIdIndex + level;
2068 if (num_records == 0) {
2070 *msg_id = kmp_i18n_str_NoProcRecords;
2073 if (num_records > (
unsigned)__kmp_xproc) {
2075 *msg_id = kmp_i18n_str_TooManyProcRecords;
2084 if (fseek(f, 0, SEEK_SET) != 0) {
2086 *msg_id = kmp_i18n_str_CantRewindCpuinfo;
2092 unsigned **threadInfo =
2093 (
unsigned **)__kmp_allocate((num_records + 1) *
sizeof(
unsigned *));
2095 for (i = 0; i <= num_records; i++) {
2097 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2100 #define CLEANUP_THREAD_INFO \
2101 for (i = 0; i <= num_records; i++) { \
2102 __kmp_free(threadInfo[i]); \
2104 __kmp_free(threadInfo);
2109 #define INIT_PROC_INFO(p) \
2110 for (__index = 0; __index <= maxIndex; __index++) { \
2111 (p)[__index] = UINT_MAX; \
2114 for (i = 0; i <= num_records; i++) {
2115 INIT_PROC_INFO(threadInfo[i]);
2118 unsigned num_avail = 0;
2125 buf[
sizeof(buf) - 1] = 1;
2126 bool long_line =
false;
2127 if (!fgets(buf,
sizeof(buf), f)) {
2132 for (i = 0; i <= maxIndex; i++) {
2133 if (threadInfo[num_avail][i] != UINT_MAX) {
2141 }
else if (!buf[
sizeof(buf) - 1]) {
2146 #define CHECK_LINE \
2148 CLEANUP_THREAD_INFO; \
2149 *msg_id = kmp_i18n_str_LongLineCpuinfo; \
2155 char s1[] =
"processor";
2156 if (strncmp(buf, s1,
sizeof(s1) - 1) == 0) {
2158 char *p = strchr(buf +
sizeof(s1) - 1,
':');
2160 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2162 if (threadInfo[num_avail][osIdIndex] != UINT_MAX)
2163 #if KMP_ARCH_AARCH64
2172 threadInfo[num_avail][osIdIndex] = val;
2173 #if KMP_OS_LINUX && !(KMP_ARCH_X86 || KMP_ARCH_X86_64)
2177 "/sys/devices/system/cpu/cpu%u/topology/physical_package_id",
2178 threadInfo[num_avail][osIdIndex]);
2179 __kmp_read_from_file(path,
"%u", &threadInfo[num_avail][pkgIdIndex]);
2181 KMP_SNPRINTF(path,
sizeof(path),
2182 "/sys/devices/system/cpu/cpu%u/topology/core_id",
2183 threadInfo[num_avail][osIdIndex]);
2184 __kmp_read_from_file(path,
"%u", &threadInfo[num_avail][coreIdIndex]);
2188 char s2[] =
"physical id";
2189 if (strncmp(buf, s2,
sizeof(s2) - 1) == 0) {
2191 char *p = strchr(buf +
sizeof(s2) - 1,
':');
2193 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2195 if (threadInfo[num_avail][pkgIdIndex] != UINT_MAX)
2197 threadInfo[num_avail][pkgIdIndex] = val;
2200 char s3[] =
"core id";
2201 if (strncmp(buf, s3,
sizeof(s3) - 1) == 0) {
2203 char *p = strchr(buf +
sizeof(s3) - 1,
':');
2205 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2207 if (threadInfo[num_avail][coreIdIndex] != UINT_MAX)
2209 threadInfo[num_avail][coreIdIndex] = val;
2211 #endif // KMP_OS_LINUX && USE_SYSFS_INFO
2213 char s4[] =
"thread id";
2214 if (strncmp(buf, s4,
sizeof(s4) - 1) == 0) {
2216 char *p = strchr(buf +
sizeof(s4) - 1,
':');
2218 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2220 if (threadInfo[num_avail][threadIdIndex] != UINT_MAX)
2222 threadInfo[num_avail][threadIdIndex] = val;
2226 if (KMP_SSCANF(buf,
"node_%u id", &level) == 1) {
2228 char *p = strchr(buf +
sizeof(s4) - 1,
':');
2230 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2232 KMP_ASSERT(nodeIdIndex + level <= maxIndex);
2233 if (threadInfo[num_avail][nodeIdIndex + level] != UINT_MAX)
2235 threadInfo[num_avail][nodeIdIndex + level] = val;
2242 if ((*buf != 0) && (*buf !=
'\n')) {
2247 while (((ch = fgetc(f)) != EOF) && (ch !=
'\n'))
2255 if ((
int)num_avail == __kmp_xproc) {
2256 CLEANUP_THREAD_INFO;
2257 *msg_id = kmp_i18n_str_TooManyEntries;
2263 if (threadInfo[num_avail][osIdIndex] == UINT_MAX) {
2264 CLEANUP_THREAD_INFO;
2265 *msg_id = kmp_i18n_str_MissingProcField;
2268 if (threadInfo[0][pkgIdIndex] == UINT_MAX) {
2269 CLEANUP_THREAD_INFO;
2270 *msg_id = kmp_i18n_str_MissingPhysicalIDField;
2275 if (!KMP_CPU_ISSET(threadInfo[num_avail][osIdIndex],
2276 __kmp_affin_fullMask)) {
2277 INIT_PROC_INFO(threadInfo[num_avail]);
2284 KMP_ASSERT(num_avail <= num_records);
2285 INIT_PROC_INFO(threadInfo[num_avail]);
2290 CLEANUP_THREAD_INFO;
2291 *msg_id = kmp_i18n_str_MissingValCpuinfo;
2295 CLEANUP_THREAD_INFO;
2296 *msg_id = kmp_i18n_str_DuplicateFieldCpuinfo;
2301 #if KMP_MIC && REDUCE_TEAM_SIZE
2302 unsigned teamSize = 0;
2303 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2314 KMP_ASSERT(num_avail > 0);
2315 KMP_ASSERT(num_avail <= num_records);
2316 if (num_avail == 1) {
2318 __kmp_nThreadsPerCore = nCoresPerPkg = nPackages = 1;
2319 if (__kmp_affinity_verbose) {
2320 if (!KMP_AFFINITY_CAPABLE()) {
2321 KMP_INFORM(AffNotCapableUseCpuinfo,
"KMP_AFFINITY");
2322 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2323 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2325 char buf[KMP_AFFIN_MASK_PRINT_LEN];
2326 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
2327 __kmp_affin_fullMask);
2328 KMP_INFORM(AffCapableUseCpuinfo,
"KMP_AFFINITY");
2329 if (__kmp_affinity_respect_mask) {
2330 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
2332 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
2334 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2335 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2339 __kmp_str_buf_init(&buf);
2340 __kmp_str_buf_print(&buf,
"1");
2341 for (index = maxIndex - 1; index > pkgIdIndex; index--) {
2342 __kmp_str_buf_print(&buf,
" x 1");
2344 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, 1, 1, 1);
2345 __kmp_str_buf_free(&buf);
2348 if (__kmp_affinity_type == affinity_none) {
2349 CLEANUP_THREAD_INFO;
2353 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair));
2355 addr.labels[0] = threadInfo[0][pkgIdIndex];
2356 (*address2os)[0] = AddrUnsPair(addr, threadInfo[0][osIdIndex]);
2358 if (__kmp_affinity_gran_levels < 0) {
2359 __kmp_affinity_gran_levels = 0;
2362 if (__kmp_affinity_verbose) {
2363 __kmp_affinity_print_topology(*address2os, 1, 1, 0, -1, -1);
2366 CLEANUP_THREAD_INFO;
2371 qsort(threadInfo, num_avail,
sizeof(*threadInfo),
2372 __kmp_affinity_cmp_ProcCpuInfo_phys_id);
2384 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2386 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2388 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2390 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2392 bool assign_thread_ids =
false;
2393 unsigned threadIdCt;
2396 restart_radix_check:
2400 if (assign_thread_ids) {
2401 if (threadInfo[0][threadIdIndex] == UINT_MAX) {
2402 threadInfo[0][threadIdIndex] = threadIdCt++;
2403 }
else if (threadIdCt <= threadInfo[0][threadIdIndex]) {
2404 threadIdCt = threadInfo[0][threadIdIndex] + 1;
2407 for (index = 0; index <= maxIndex; index++) {
2411 lastId[index] = threadInfo[0][index];
2416 for (i = 1; i < num_avail; i++) {
2419 for (index = maxIndex; index >= threadIdIndex; index--) {
2420 if (assign_thread_ids && (index == threadIdIndex)) {
2422 if (threadInfo[i][threadIdIndex] == UINT_MAX) {
2423 threadInfo[i][threadIdIndex] = threadIdCt++;
2427 else if (threadIdCt <= threadInfo[i][threadIdIndex]) {
2428 threadIdCt = threadInfo[i][threadIdIndex] + 1;
2431 if (threadInfo[i][index] != lastId[index]) {
2436 for (index2 = threadIdIndex; index2 < index; index2++) {
2438 if (counts[index2] > maxCt[index2]) {
2439 maxCt[index2] = counts[index2];
2442 lastId[index2] = threadInfo[i][index2];
2446 lastId[index] = threadInfo[i][index];
2448 if (assign_thread_ids && (index > threadIdIndex)) {
2450 #if KMP_MIC && REDUCE_TEAM_SIZE
2453 teamSize += (threadIdCt <= 2) ? (threadIdCt) : (threadIdCt - 1);
2454 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2460 if (threadInfo[i][threadIdIndex] == UINT_MAX) {
2461 threadInfo[i][threadIdIndex] = threadIdCt++;
2467 else if (threadIdCt <= threadInfo[i][threadIdIndex]) {
2468 threadIdCt = threadInfo[i][threadIdIndex] + 1;
2474 if (index < threadIdIndex) {
2478 if ((threadInfo[i][threadIdIndex] != UINT_MAX) || assign_thread_ids) {
2483 CLEANUP_THREAD_INFO;
2484 *msg_id = kmp_i18n_str_PhysicalIDsNotUnique;
2490 assign_thread_ids =
true;
2491 goto restart_radix_check;
2495 #if KMP_MIC && REDUCE_TEAM_SIZE
2498 teamSize += (threadIdCt <= 2) ? (threadIdCt) : (threadIdCt - 1);
2499 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2501 for (index = threadIdIndex; index <= maxIndex; index++) {
2502 if (counts[index] > maxCt[index]) {
2503 maxCt[index] = counts[index];
2507 __kmp_nThreadsPerCore = maxCt[threadIdIndex];
2508 nCoresPerPkg = maxCt[coreIdIndex];
2509 nPackages = totals[pkgIdIndex];
2512 unsigned prod = totals[maxIndex];
2513 for (index = threadIdIndex; index < maxIndex; index++) {
2514 prod *= maxCt[index];
2516 bool uniform = (prod == totals[threadIdIndex]);
2522 __kmp_ncores = totals[coreIdIndex];
2524 if (__kmp_affinity_verbose) {
2525 if (!KMP_AFFINITY_CAPABLE()) {
2526 KMP_INFORM(AffNotCapableUseCpuinfo,
"KMP_AFFINITY");
2527 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2529 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2531 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
2534 char buf[KMP_AFFIN_MASK_PRINT_LEN];
2535 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
2536 __kmp_affin_fullMask);
2537 KMP_INFORM(AffCapableUseCpuinfo,
"KMP_AFFINITY");
2538 if (__kmp_affinity_respect_mask) {
2539 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
2541 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
2543 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2545 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2547 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
2551 __kmp_str_buf_init(&buf);
2553 __kmp_str_buf_print(&buf,
"%d", totals[maxIndex]);
2554 for (index = maxIndex - 1; index >= pkgIdIndex; index--) {
2555 __kmp_str_buf_print(&buf,
" x %d", maxCt[index]);
2557 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, maxCt[coreIdIndex],
2558 maxCt[threadIdIndex], __kmp_ncores);
2560 __kmp_str_buf_free(&buf);
2563 #if KMP_MIC && REDUCE_TEAM_SIZE
2565 if ((__kmp_dflt_team_nth == 0) && (teamSize > 0)) {
2566 __kmp_dflt_team_nth = teamSize;
2567 KA_TRACE(20, (
"__kmp_affinity_create_cpuinfo_map: setting "
2568 "__kmp_dflt_team_nth = %d\n",
2569 __kmp_dflt_team_nth));
2571 #endif // KMP_MIC && REDUCE_TEAM_SIZE
2573 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
2574 KMP_DEBUG_ASSERT(num_avail == (
unsigned)__kmp_avail_proc);
2575 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
2576 for (i = 0; i < num_avail; ++i) {
2577 __kmp_pu_os_idx[i] = threadInfo[i][osIdIndex];
2580 if (__kmp_affinity_type == affinity_none) {
2585 CLEANUP_THREAD_INFO;
2594 bool *inMap = (
bool *)__kmp_allocate((maxIndex + 1) *
sizeof(bool));
2595 for (index = threadIdIndex; index < maxIndex; index++) {
2596 KMP_ASSERT(totals[index] >= totals[index + 1]);
2597 inMap[index] = (totals[index] > totals[index + 1]);
2599 inMap[maxIndex] = (totals[maxIndex] > 1);
2600 inMap[pkgIdIndex] =
true;
2603 for (index = threadIdIndex; index <= maxIndex; index++) {
2608 KMP_ASSERT(depth > 0);
2611 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * num_avail);
2614 int threadLevel = -1;
2616 for (i = 0; i < num_avail; ++i) {
2617 Address addr(depth);
2618 unsigned os = threadInfo[i][osIdIndex];
2622 for (src_index = maxIndex; src_index >= threadIdIndex; src_index--) {
2623 if (!inMap[src_index]) {
2626 addr.labels[dst_index] = threadInfo[i][src_index];
2627 if (src_index == pkgIdIndex) {
2628 pkgLevel = dst_index;
2629 }
else if (src_index == coreIdIndex) {
2630 coreLevel = dst_index;
2631 }
else if (src_index == threadIdIndex) {
2632 threadLevel = dst_index;
2636 (*address2os)[i] = AddrUnsPair(addr, os);
2639 if (__kmp_affinity_gran_levels < 0) {
2643 __kmp_affinity_gran_levels = 0;
2644 for (src_index = threadIdIndex; src_index <= maxIndex; src_index++) {
2645 if (!inMap[src_index]) {
2648 switch (src_index) {
2650 if (__kmp_affinity_gran > affinity_gran_thread) {
2651 __kmp_affinity_gran_levels++;
2656 if (__kmp_affinity_gran > affinity_gran_core) {
2657 __kmp_affinity_gran_levels++;
2662 if (__kmp_affinity_gran > affinity_gran_package) {
2663 __kmp_affinity_gran_levels++;
2670 if (__kmp_affinity_verbose) {
2671 __kmp_affinity_print_topology(*address2os, num_avail, depth, pkgLevel,
2672 coreLevel, threadLevel);
2680 CLEANUP_THREAD_INFO;
2687 static kmp_affin_mask_t *__kmp_create_masks(
unsigned *maxIndex,
2688 unsigned *numUnique,
2689 AddrUnsPair *address2os,
2690 unsigned numAddrs) {
2696 KMP_ASSERT(numAddrs > 0);
2697 depth = address2os[0].first.depth;
2700 for (i = numAddrs - 1;; --i) {
2701 unsigned osId = address2os[i].second;
2702 if (osId > maxOsId) {
2708 kmp_affin_mask_t *osId2Mask;
2709 KMP_CPU_ALLOC_ARRAY(osId2Mask, (maxOsId + 1));
2713 qsort(address2os, numAddrs,
sizeof(*address2os),
2714 __kmp_affinity_cmp_Address_labels);
2716 KMP_ASSERT(__kmp_affinity_gran_levels >= 0);
2717 if (__kmp_affinity_verbose && (__kmp_affinity_gran_levels > 0)) {
2718 KMP_INFORM(ThreadsMigrate,
"KMP_AFFINITY", __kmp_affinity_gran_levels);
2720 if (__kmp_affinity_gran_levels >= (
int)depth) {
2721 if (__kmp_affinity_verbose ||
2722 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
2723 KMP_WARNING(AffThreadsMayMigrate);
2731 unsigned unique = 0;
2733 unsigned leader = 0;
2734 Address *leaderAddr = &(address2os[0].first);
2735 kmp_affin_mask_t *sum;
2736 KMP_CPU_ALLOC_ON_STACK(sum);
2738 KMP_CPU_SET(address2os[0].second, sum);
2739 for (i = 1; i < numAddrs; i++) {
2743 if (leaderAddr->isClose(address2os[i].first, __kmp_affinity_gran_levels)) {
2744 KMP_CPU_SET(address2os[i].second, sum);
2750 for (; j < i; j++) {
2751 unsigned osId = address2os[j].second;
2752 KMP_DEBUG_ASSERT(osId <= maxOsId);
2753 kmp_affin_mask_t *mask = KMP_CPU_INDEX(osId2Mask, osId);
2754 KMP_CPU_COPY(mask, sum);
2755 address2os[j].first.leader = (j == leader);
2761 leaderAddr = &(address2os[i].first);
2763 KMP_CPU_SET(address2os[i].second, sum);
2768 for (; j < i; j++) {
2769 unsigned osId = address2os[j].second;
2770 KMP_DEBUG_ASSERT(osId <= maxOsId);
2771 kmp_affin_mask_t *mask = KMP_CPU_INDEX(osId2Mask, osId);
2772 KMP_CPU_COPY(mask, sum);
2773 address2os[j].first.leader = (j == leader);
2776 KMP_CPU_FREE_FROM_STACK(sum);
2778 *maxIndex = maxOsId;
2779 *numUnique = unique;
2786 static kmp_affin_mask_t *newMasks;
2787 static int numNewMasks;
2788 static int nextNewMask;
2790 #define ADD_MASK(_mask) \
2792 if (nextNewMask >= numNewMasks) { \
2795 kmp_affin_mask_t *temp; \
2796 KMP_CPU_INTERNAL_ALLOC_ARRAY(temp, numNewMasks); \
2797 for (i = 0; i < numNewMasks / 2; i++) { \
2798 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i); \
2799 kmp_affin_mask_t *dest = KMP_CPU_INDEX(temp, i); \
2800 KMP_CPU_COPY(dest, src); \
2802 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks / 2); \
2805 KMP_CPU_COPY(KMP_CPU_INDEX(newMasks, nextNewMask), (_mask)); \
2809 #define ADD_MASK_OSID(_osId, _osId2Mask, _maxOsId) \
2811 if (((_osId) > _maxOsId) || \
2812 (!KMP_CPU_ISSET((_osId), KMP_CPU_INDEX((_osId2Mask), (_osId))))) { \
2813 if (__kmp_affinity_verbose || \
2814 (__kmp_affinity_warnings && \
2815 (__kmp_affinity_type != affinity_none))) { \
2816 KMP_WARNING(AffIgnoreInvalidProcID, _osId); \
2819 ADD_MASK(KMP_CPU_INDEX(_osId2Mask, (_osId))); \
2825 static void __kmp_affinity_process_proclist(kmp_affin_mask_t **out_masks,
2826 unsigned int *out_numMasks,
2827 const char *proclist,
2828 kmp_affin_mask_t *osId2Mask,
2831 const char *scan = proclist;
2832 const char *next = proclist;
2837 KMP_CPU_INTERNAL_ALLOC_ARRAY(newMasks, numNewMasks);
2839 kmp_affin_mask_t *sumMask;
2840 KMP_CPU_ALLOC(sumMask);
2844 int start, end, stride;
2848 if (*next ==
'\0') {
2860 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad proclist");
2862 num = __kmp_str_to_int(scan, *next);
2863 KMP_ASSERT2(num >= 0,
"bad explicit proc list");
2866 if ((num > maxOsId) ||
2867 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
2868 if (__kmp_affinity_verbose ||
2869 (__kmp_affinity_warnings &&
2870 (__kmp_affinity_type != affinity_none))) {
2871 KMP_WARNING(AffIgnoreInvalidProcID, num);
2873 KMP_CPU_ZERO(sumMask);
2875 KMP_CPU_COPY(sumMask, KMP_CPU_INDEX(osId2Mask, num));
2895 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2898 num = __kmp_str_to_int(scan, *next);
2899 KMP_ASSERT2(num >= 0,
"bad explicit proc list");
2902 if ((num > maxOsId) ||
2903 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
2904 if (__kmp_affinity_verbose ||
2905 (__kmp_affinity_warnings &&
2906 (__kmp_affinity_type != affinity_none))) {
2907 KMP_WARNING(AffIgnoreInvalidProcID, num);
2910 KMP_CPU_UNION(sumMask, KMP_CPU_INDEX(osId2Mask, num));
2927 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2929 start = __kmp_str_to_int(scan, *next);
2930 KMP_ASSERT2(start >= 0,
"bad explicit proc list");
2935 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2949 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2951 end = __kmp_str_to_int(scan, *next);
2952 KMP_ASSERT2(end >= 0,
"bad explicit proc list");
2969 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2971 stride = __kmp_str_to_int(scan, *next);
2972 KMP_ASSERT2(stride >= 0,
"bad explicit proc list");
2977 KMP_ASSERT2(stride != 0,
"bad explicit proc list");
2979 KMP_ASSERT2(start <= end,
"bad explicit proc list");
2981 KMP_ASSERT2(start >= end,
"bad explicit proc list");
2983 KMP_ASSERT2((end - start) / stride <= 65536,
"bad explicit proc list");
2988 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2990 }
while (start <= end);
2993 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2995 }
while (start >= end);
3006 *out_numMasks = nextNewMask;
3007 if (nextNewMask == 0) {
3009 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3012 KMP_CPU_ALLOC_ARRAY((*out_masks), nextNewMask);
3013 for (i = 0; i < nextNewMask; i++) {
3014 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i);
3015 kmp_affin_mask_t *dest = KMP_CPU_INDEX((*out_masks), i);
3016 KMP_CPU_COPY(dest, src);
3018 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3019 KMP_CPU_FREE(sumMask);
3045 static void __kmp_process_subplace_list(
const char **scan,
3046 kmp_affin_mask_t *osId2Mask,
3047 int maxOsId, kmp_affin_mask_t *tempMask,
3052 int start, count, stride, i;
3056 KMP_ASSERT2((**scan >=
'0') && (**scan <=
'9'),
"bad explicit places list");
3059 start = __kmp_str_to_int(*scan, *next);
3060 KMP_ASSERT(start >= 0);
3065 if (**scan ==
'}' || **scan ==
',') {
3066 if ((start > maxOsId) ||
3067 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
3068 if (__kmp_affinity_verbose ||
3069 (__kmp_affinity_warnings &&
3070 (__kmp_affinity_type != affinity_none))) {
3071 KMP_WARNING(AffIgnoreInvalidProcID, start);
3074 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
3077 if (**scan ==
'}') {
3083 KMP_ASSERT2(**scan ==
':',
"bad explicit places list");
3088 KMP_ASSERT2((**scan >=
'0') && (**scan <=
'9'),
"bad explicit places list");
3091 count = __kmp_str_to_int(*scan, *next);
3092 KMP_ASSERT(count >= 0);
3097 if (**scan ==
'}' || **scan ==
',') {
3098 for (i = 0; i < count; i++) {
3099 if ((start > maxOsId) ||
3100 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
3101 if (__kmp_affinity_verbose ||
3102 (__kmp_affinity_warnings &&
3103 (__kmp_affinity_type != affinity_none))) {
3104 KMP_WARNING(AffIgnoreInvalidProcID, start);
3108 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
3113 if (**scan ==
'}') {
3119 KMP_ASSERT2(**scan ==
':',
"bad explicit places list");
3126 if (**scan ==
'+') {
3130 if (**scan ==
'-') {
3138 KMP_ASSERT2((**scan >=
'0') && (**scan <=
'9'),
"bad explicit places list");
3141 stride = __kmp_str_to_int(*scan, *next);
3142 KMP_ASSERT(stride >= 0);
3148 if (**scan ==
'}' || **scan ==
',') {
3149 for (i = 0; i < count; i++) {
3150 if ((start > maxOsId) ||
3151 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
3152 if (__kmp_affinity_verbose ||
3153 (__kmp_affinity_warnings &&
3154 (__kmp_affinity_type != affinity_none))) {
3155 KMP_WARNING(AffIgnoreInvalidProcID, start);
3159 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
3164 if (**scan ==
'}') {
3171 KMP_ASSERT2(0,
"bad explicit places list");
3175 static void __kmp_process_place(
const char **scan, kmp_affin_mask_t *osId2Mask,
3176 int maxOsId, kmp_affin_mask_t *tempMask,
3182 if (**scan ==
'{') {
3184 __kmp_process_subplace_list(scan, osId2Mask, maxOsId, tempMask, setSize);
3185 KMP_ASSERT2(**scan ==
'}',
"bad explicit places list");
3187 }
else if (**scan ==
'!') {
3189 __kmp_process_place(scan, osId2Mask, maxOsId, tempMask, setSize);
3190 KMP_CPU_COMPLEMENT(maxOsId, tempMask);
3191 }
else if ((**scan >=
'0') && (**scan <=
'9')) {
3194 int num = __kmp_str_to_int(*scan, *next);
3195 KMP_ASSERT(num >= 0);
3196 if ((num > maxOsId) ||
3197 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
3198 if (__kmp_affinity_verbose ||
3199 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
3200 KMP_WARNING(AffIgnoreInvalidProcID, num);
3203 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, num));
3208 KMP_ASSERT2(0,
"bad explicit places list");
3213 void __kmp_affinity_process_placelist(kmp_affin_mask_t **out_masks,
3214 unsigned int *out_numMasks,
3215 const char *placelist,
3216 kmp_affin_mask_t *osId2Mask,
3218 int i, j, count, stride, sign;
3219 const char *scan = placelist;
3220 const char *next = placelist;
3223 KMP_CPU_INTERNAL_ALLOC_ARRAY(newMasks, numNewMasks);
3229 kmp_affin_mask_t *tempMask;
3230 kmp_affin_mask_t *previousMask;
3231 KMP_CPU_ALLOC(tempMask);
3232 KMP_CPU_ZERO(tempMask);
3233 KMP_CPU_ALLOC(previousMask);
3234 KMP_CPU_ZERO(previousMask);
3238 __kmp_process_place(&scan, osId2Mask, maxOsId, tempMask, &setSize);
3242 if (*scan ==
'\0' || *scan ==
',') {
3246 KMP_CPU_ZERO(tempMask);
3248 if (*scan ==
'\0') {
3255 KMP_ASSERT2(*scan ==
':',
"bad explicit places list");
3260 KMP_ASSERT2((*scan >=
'0') && (*scan <=
'9'),
"bad explicit places list");
3263 count = __kmp_str_to_int(scan, *next);
3264 KMP_ASSERT(count >= 0);
3269 if (*scan ==
'\0' || *scan ==
',') {
3272 KMP_ASSERT2(*scan ==
':',
"bad explicit places list");
3291 KMP_ASSERT2((*scan >=
'0') && (*scan <=
'9'),
"bad explicit places list");
3294 stride = __kmp_str_to_int(scan, *next);
3295 KMP_DEBUG_ASSERT(stride >= 0);
3301 for (i = 0; i < count; i++) {
3306 KMP_CPU_COPY(previousMask, tempMask);
3307 ADD_MASK(previousMask);
3308 KMP_CPU_ZERO(tempMask);
3310 KMP_CPU_SET_ITERATE(j, previousMask) {
3311 if (!KMP_CPU_ISSET(j, previousMask)) {
3314 if ((j + stride > maxOsId) || (j + stride < 0) ||
3315 (!KMP_CPU_ISSET(j, __kmp_affin_fullMask)) ||
3316 (!KMP_CPU_ISSET(j + stride,
3317 KMP_CPU_INDEX(osId2Mask, j + stride)))) {
3318 if ((__kmp_affinity_verbose ||
3319 (__kmp_affinity_warnings &&
3320 (__kmp_affinity_type != affinity_none))) &&
3322 KMP_WARNING(AffIgnoreInvalidProcID, j + stride);
3326 KMP_CPU_SET(j + stride, tempMask);
3330 KMP_CPU_ZERO(tempMask);
3335 if (*scan ==
'\0') {
3343 KMP_ASSERT2(0,
"bad explicit places list");
3346 *out_numMasks = nextNewMask;
3347 if (nextNewMask == 0) {
3349 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3352 KMP_CPU_ALLOC_ARRAY((*out_masks), nextNewMask);
3353 KMP_CPU_FREE(tempMask);
3354 KMP_CPU_FREE(previousMask);
3355 for (i = 0; i < nextNewMask; i++) {
3356 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i);
3357 kmp_affin_mask_t *dest = KMP_CPU_INDEX((*out_masks), i);
3358 KMP_CPU_COPY(dest, src);
3360 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3366 #undef ADD_MASK_OSID
3369 static int __kmp_hwloc_skip_PUs_obj(hwloc_topology_t t, hwloc_obj_t o) {
3372 hwloc_obj_t hT = NULL;
3373 int N = __kmp_hwloc_count_children_by_type(t, o, HWLOC_OBJ_PU, &hT);
3374 for (
int i = 0; i < N; ++i) {
3375 KMP_DEBUG_ASSERT(hT);
3376 unsigned idx = hT->os_index;
3377 if (KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3378 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3379 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3382 hT = hwloc_get_next_obj_by_type(t, HWLOC_OBJ_PU, hT);
3387 static int __kmp_hwloc_obj_has_PUs(hwloc_topology_t t, hwloc_obj_t o) {
3389 hwloc_obj_t hT = NULL;
3390 int N = __kmp_hwloc_count_children_by_type(t, o, HWLOC_OBJ_PU, &hT);
3391 for (
int i = 0; i < N; ++i) {
3392 KMP_DEBUG_ASSERT(hT);
3393 unsigned idx = hT->os_index;
3394 if (KMP_CPU_ISSET(idx, __kmp_affin_fullMask))
3396 hT = hwloc_get_next_obj_by_type(t, HWLOC_OBJ_PU, hT);
3400 #endif // KMP_USE_HWLOC
3402 static void __kmp_apply_thread_places(AddrUnsPair **pAddr,
int depth) {
3403 AddrUnsPair *newAddr;
3404 if (__kmp_hws_requested == 0)
3407 if (__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) {
3411 hwloc_topology_t tp = __kmp_hwloc_topology;
3412 int nS = 0, nN = 0, nL = 0, nC = 0,
3414 int nCr = 0, nTr = 0;
3415 int nPkg = 0, nCo = 0, n_new = 0, n_old = 0, nCpP = 0, nTpC = 0;
3416 hwloc_obj_t hT, hC, hL, hN, hS;
3420 int numa_support = 0, tile_support = 0;
3421 if (__kmp_pu_os_idx)
3422 hT = hwloc_get_pu_obj_by_os_index(tp,
3423 __kmp_pu_os_idx[__kmp_avail_proc - 1]);
3425 hT = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PU, __kmp_avail_proc - 1);
3427 KMP_WARNING(AffHWSubsetUnsupported);
3431 hN = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hT);
3432 hS = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hT);
3433 if (hN != NULL && hN->depth > hS->depth) {
3435 }
else if (__kmp_hws_node.num > 0) {
3437 KMP_WARNING(AffHWSubsetUnsupported);
3441 L2depth = hwloc_get_cache_type_depth(tp, 2, HWLOC_OBJ_CACHE_UNIFIED);
3442 hL = hwloc_get_ancestor_obj_by_depth(tp, L2depth, hT);
3444 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC) > 1) {
3446 }
else if (__kmp_hws_tile.num > 0) {
3447 if (__kmp_hws_core.num == 0) {
3448 __kmp_hws_core = __kmp_hws_tile;
3449 __kmp_hws_tile.num = 0;
3452 KMP_WARNING(AffHWSubsetInvalid);
3459 if (__kmp_hws_socket.num == 0)
3460 __kmp_hws_socket.num = nPackages;
3461 if (__kmp_hws_socket.offset >= nPackages) {
3462 KMP_WARNING(AffHWSubsetManySockets);
3467 int NN = __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_NUMANODE,
3469 if (__kmp_hws_node.num == 0)
3470 __kmp_hws_node.num = NN;
3471 if (__kmp_hws_node.offset >= NN) {
3472 KMP_WARNING(AffHWSubsetManyNodes);
3477 int NL = __kmp_hwloc_count_children_by_depth(tp, hN, L2depth, &hL);
3478 if (__kmp_hws_tile.num == 0) {
3479 __kmp_hws_tile.num = NL + 1;
3481 if (__kmp_hws_tile.offset >= NL) {
3482 KMP_WARNING(AffHWSubsetManyTiles);
3485 int NC = __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE,
3487 if (__kmp_hws_core.num == 0)
3488 __kmp_hws_core.num = NC;
3489 if (__kmp_hws_core.offset >= NC) {
3490 KMP_WARNING(AffHWSubsetManyCores);
3494 int NC = __kmp_hwloc_count_children_by_type(tp, hN, HWLOC_OBJ_CORE,
3496 if (__kmp_hws_core.num == 0)
3497 __kmp_hws_core.num = NC;
3498 if (__kmp_hws_core.offset >= NC) {
3499 KMP_WARNING(AffHWSubsetManyCores);
3506 int NL = __kmp_hwloc_count_children_by_depth(tp, hS, L2depth, &hL);
3507 if (__kmp_hws_tile.num == 0)
3508 __kmp_hws_tile.num = NL;
3509 if (__kmp_hws_tile.offset >= NL) {
3510 KMP_WARNING(AffHWSubsetManyTiles);
3513 int NC = __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE,
3515 if (__kmp_hws_core.num == 0)
3516 __kmp_hws_core.num = NC;
3517 if (__kmp_hws_core.offset >= NC) {
3518 KMP_WARNING(AffHWSubsetManyCores);
3522 int NC = __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_CORE,
3524 if (__kmp_hws_core.num == 0)
3525 __kmp_hws_core.num = NC;
3526 if (__kmp_hws_core.offset >= NC) {
3527 KMP_WARNING(AffHWSubsetManyCores);
3532 if (__kmp_hws_proc.num == 0)
3533 __kmp_hws_proc.num = __kmp_nThreadsPerCore;
3534 if (__kmp_hws_proc.offset >= __kmp_nThreadsPerCore) {
3535 KMP_WARNING(AffHWSubsetManyProcs);
3541 newAddr = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) *
3545 int NP = hwloc_get_nbobjs_by_type(tp, HWLOC_OBJ_PACKAGE);
3546 for (
int s = 0; s < NP; ++s) {
3548 hS = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hS);
3549 if (!__kmp_hwloc_obj_has_PUs(tp, hS))
3552 if (nS <= __kmp_hws_socket.offset ||
3553 nS > __kmp_hws_socket.num + __kmp_hws_socket.offset) {
3554 n_old += __kmp_hwloc_skip_PUs_obj(tp, hS);
3565 __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_NUMANODE, &hN);
3566 for (
int n = 0; n < NN; ++n) {
3568 if (!__kmp_hwloc_obj_has_PUs(tp, hN)) {
3569 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3573 if (nN <= __kmp_hws_node.offset ||
3574 nN > __kmp_hws_node.num + __kmp_hws_node.offset) {
3576 n_old += __kmp_hwloc_skip_PUs_obj(tp, hN);
3577 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3584 int NL = __kmp_hwloc_count_children_by_depth(tp, hN, L2depth, &hL);
3585 for (
int l = 0; l < NL; ++l) {
3587 if (!__kmp_hwloc_obj_has_PUs(tp, hL)) {
3588 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3592 if (nL <= __kmp_hws_tile.offset ||
3593 nL > __kmp_hws_tile.num + __kmp_hws_tile.offset) {
3595 n_old += __kmp_hwloc_skip_PUs_obj(tp, hL);
3596 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3603 int NC = __kmp_hwloc_count_children_by_type(tp, hL,
3604 HWLOC_OBJ_CORE, &hC);
3605 for (
int c = 0; c < NC; ++c) {
3607 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3608 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3612 if (nC <= __kmp_hws_core.offset ||
3613 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3615 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3616 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3624 int NT = __kmp_hwloc_count_children_by_type(tp, hC,
3626 for (
int t = 0; t < NT; ++t) {
3629 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3630 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3634 if (nT <= __kmp_hws_proc.offset ||
3635 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3637 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3639 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3640 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3645 newAddr[n_new] = (*pAddr)[n_old];
3648 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3656 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3658 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3666 __kmp_hwloc_count_children_by_type(tp, hN, HWLOC_OBJ_CORE, &hC);
3667 for (
int c = 0; c < NC; ++c) {
3669 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3670 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3674 if (nC <= __kmp_hws_core.offset ||
3675 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3677 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3678 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3686 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3687 for (
int t = 0; t < NT; ++t) {
3690 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3691 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3695 if (nT <= __kmp_hws_proc.offset ||
3696 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3698 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3700 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3701 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3706 newAddr[n_new] = (*pAddr)[n_old];
3709 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3717 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3720 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3728 int NL = __kmp_hwloc_count_children_by_depth(tp, hS, L2depth, &hL);
3729 for (
int l = 0; l < NL; ++l) {
3731 if (!__kmp_hwloc_obj_has_PUs(tp, hL)) {
3732 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3736 if (nL <= __kmp_hws_tile.offset ||
3737 nL > __kmp_hws_tile.num + __kmp_hws_tile.offset) {
3739 n_old += __kmp_hwloc_skip_PUs_obj(tp, hL);
3740 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3748 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC);
3749 for (
int c = 0; c < NC; ++c) {
3751 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3752 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3756 if (nC <= __kmp_hws_core.offset ||
3757 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3759 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3760 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3769 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3770 for (
int t = 0; t < NT; ++t) {
3773 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3774 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3778 if (nT <= __kmp_hws_proc.offset ||
3779 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3781 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3783 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3784 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3789 newAddr[n_new] = (*pAddr)[n_old];
3792 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3800 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3802 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3810 __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_CORE, &hC);
3811 for (
int c = 0; c < NC; ++c) {
3813 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3814 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3818 if (nC <= __kmp_hws_core.offset ||
3819 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3821 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3822 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3831 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3832 for (
int t = 0; t < NT; ++t) {
3835 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3836 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3840 if (nT <= __kmp_hws_proc.offset ||
3841 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3843 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3845 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3846 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3851 newAddr[n_new] = (*pAddr)[n_old];
3854 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3862 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3874 KMP_DEBUG_ASSERT(n_old == __kmp_avail_proc);
3875 KMP_DEBUG_ASSERT(nPkg > 0);
3876 KMP_DEBUG_ASSERT(nCpP > 0);
3877 KMP_DEBUG_ASSERT(nTpC > 0);
3878 KMP_DEBUG_ASSERT(nCo > 0);
3879 KMP_DEBUG_ASSERT(nPkg <= nPackages);
3880 KMP_DEBUG_ASSERT(nCpP <= nCoresPerPkg);
3881 KMP_DEBUG_ASSERT(nTpC <= __kmp_nThreadsPerCore);
3882 KMP_DEBUG_ASSERT(nCo <= __kmp_ncores);
3885 nCoresPerPkg = nCpP;
3886 __kmp_nThreadsPerCore = nTpC;
3887 __kmp_avail_proc = n_new;
3891 #endif // KMP_USE_HWLOC
3893 int n_old = 0, n_new = 0, proc_num = 0;
3894 if (__kmp_hws_node.num > 0 || __kmp_hws_tile.num > 0) {
3895 KMP_WARNING(AffHWSubsetNoHWLOC);
3898 if (__kmp_hws_socket.num == 0)
3899 __kmp_hws_socket.num = nPackages;
3900 if (__kmp_hws_core.num == 0)
3901 __kmp_hws_core.num = nCoresPerPkg;
3902 if (__kmp_hws_proc.num == 0 || __kmp_hws_proc.num > __kmp_nThreadsPerCore)
3903 __kmp_hws_proc.num = __kmp_nThreadsPerCore;
3904 if (!__kmp_affinity_uniform_topology()) {
3905 KMP_WARNING(AffHWSubsetNonUniform);
3909 KMP_WARNING(AffHWSubsetNonThreeLevel);
3912 if (__kmp_hws_socket.offset + __kmp_hws_socket.num > nPackages) {
3913 KMP_WARNING(AffHWSubsetManySockets);
3916 if (__kmp_hws_core.offset + __kmp_hws_core.num > nCoresPerPkg) {
3917 KMP_WARNING(AffHWSubsetManyCores);
3922 newAddr = (AddrUnsPair *)__kmp_allocate(
3923 sizeof(AddrUnsPair) * __kmp_hws_socket.num * __kmp_hws_core.num *
3924 __kmp_hws_proc.num);
3925 for (
int i = 0; i < nPackages; ++i) {
3926 if (i < __kmp_hws_socket.offset ||
3927 i >= __kmp_hws_socket.offset + __kmp_hws_socket.num) {
3929 n_old += nCoresPerPkg * __kmp_nThreadsPerCore;
3930 if (__kmp_pu_os_idx != NULL) {
3932 for (
int j = 0; j < nCoresPerPkg; ++j) {
3933 for (
int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3934 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3941 for (
int j = 0; j < nCoresPerPkg; ++j) {
3942 if (j < __kmp_hws_core.offset ||
3943 j >= __kmp_hws_core.offset +
3944 __kmp_hws_core.num) {
3945 n_old += __kmp_nThreadsPerCore;
3946 if (__kmp_pu_os_idx != NULL) {
3947 for (
int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3948 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3954 for (
int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3955 if (k < __kmp_hws_proc.num) {
3957 newAddr[n_new] = (*pAddr)[n_old];
3960 if (__kmp_pu_os_idx != NULL)
3961 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3970 KMP_DEBUG_ASSERT(n_old == nPackages * nCoresPerPkg * __kmp_nThreadsPerCore);
3971 KMP_DEBUG_ASSERT(n_new ==
3972 __kmp_hws_socket.num * __kmp_hws_core.num *
3973 __kmp_hws_proc.num);
3974 nPackages = __kmp_hws_socket.num;
3975 nCoresPerPkg = __kmp_hws_core.num;
3976 __kmp_nThreadsPerCore = __kmp_hws_proc.num;
3977 __kmp_avail_proc = n_new;
3978 __kmp_ncores = nPackages * __kmp_hws_core.num;
3984 if (__kmp_affinity_verbose) {
3985 char m[KMP_AFFIN_MASK_PRINT_LEN];
3986 __kmp_affinity_print_mask(m, KMP_AFFIN_MASK_PRINT_LEN,
3987 __kmp_affin_fullMask);
3988 if (__kmp_affinity_respect_mask) {
3989 KMP_INFORM(InitOSProcSetRespect,
"KMP_HW_SUBSET", m);
3991 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_HW_SUBSET", m);
3993 KMP_INFORM(AvailableOSProc,
"KMP_HW_SUBSET", __kmp_avail_proc);
3995 __kmp_str_buf_init(&buf);
3996 __kmp_str_buf_print(&buf,
"%d", nPackages);
3997 KMP_INFORM(TopologyExtra,
"KMP_HW_SUBSET", buf.str, nCoresPerPkg,
3998 __kmp_nThreadsPerCore, __kmp_ncores);
3999 __kmp_str_buf_free(&buf);
4002 if (__kmp_pu_os_idx != NULL) {
4003 __kmp_free(__kmp_pu_os_idx);
4004 __kmp_pu_os_idx = NULL;
4010 static int __kmp_affinity_find_core_level(
const AddrUnsPair *address2os,
4011 int nprocs,
int bottom_level) {
4014 for (
int i = 0; i < nprocs; i++) {
4015 for (
int j = bottom_level; j > 0; j--) {
4016 if (address2os[i].first.labels[j] > 0) {
4017 if (core_level < (j - 1)) {
4027 static int __kmp_affinity_compute_ncores(
const AddrUnsPair *address2os,
4028 int nprocs,
int bottom_level,
4034 for (i = 0; i < nprocs; i++) {
4035 for (j = bottom_level; j > core_level; j--) {
4036 if ((i + 1) < nprocs) {
4037 if (address2os[i + 1].first.labels[j] > 0) {
4042 if (j == core_level) {
4046 if (j > core_level) {
4055 static int __kmp_affinity_find_core(
const AddrUnsPair *address2os,
int proc,
4056 int bottom_level,
int core_level) {
4057 return __kmp_affinity_compute_ncores(address2os, proc + 1, bottom_level,
4064 static int __kmp_affinity_max_proc_per_core(
const AddrUnsPair *address2os,
4065 int nprocs,
int bottom_level,
4067 int maxprocpercore = 0;
4069 if (core_level < bottom_level) {
4070 for (
int i = 0; i < nprocs; i++) {
4071 int percore = address2os[i].first.labels[core_level + 1] + 1;
4073 if (percore > maxprocpercore) {
4074 maxprocpercore = percore;
4080 return maxprocpercore;
4083 static AddrUnsPair *address2os = NULL;
4084 static int *procarr = NULL;
4085 static int __kmp_aff_depth = 0;
4087 #if KMP_USE_HIER_SCHED
4088 #define KMP_EXIT_AFF_NONE \
4089 KMP_ASSERT(__kmp_affinity_type == affinity_none); \
4090 KMP_ASSERT(address2os == NULL); \
4091 __kmp_apply_thread_places(NULL, 0); \
4092 __kmp_create_affinity_none_places(); \
4093 __kmp_dispatch_set_hierarchy_values(); \
4096 #define KMP_EXIT_AFF_NONE \
4097 KMP_ASSERT(__kmp_affinity_type == affinity_none); \
4098 KMP_ASSERT(address2os == NULL); \
4099 __kmp_apply_thread_places(NULL, 0); \
4100 __kmp_create_affinity_none_places(); \
4106 static void __kmp_create_affinity_none_places() {
4107 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4108 KMP_ASSERT(__kmp_affinity_type == affinity_none);
4109 __kmp_affinity_num_masks = 1;
4110 KMP_CPU_ALLOC_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4111 kmp_affin_mask_t *dest = KMP_CPU_INDEX(__kmp_affinity_masks, 0);
4112 KMP_CPU_COPY(dest, __kmp_affin_fullMask);
4115 static int __kmp_affinity_cmp_Address_child_num(
const void *a,
const void *b) {
4116 const Address *aa = &(((
const AddrUnsPair *)a)->first);
4117 const Address *bb = &(((
const AddrUnsPair *)b)->first);
4118 unsigned depth = aa->depth;
4120 KMP_DEBUG_ASSERT(depth == bb->depth);
4121 KMP_DEBUG_ASSERT((
unsigned)__kmp_affinity_compact <= depth);
4122 KMP_DEBUG_ASSERT(__kmp_affinity_compact >= 0);
4123 for (i = 0; i < (unsigned)__kmp_affinity_compact; i++) {
4124 int j = depth - i - 1;
4125 if (aa->childNums[j] < bb->childNums[j])
4127 if (aa->childNums[j] > bb->childNums[j])
4130 for (; i < depth; i++) {
4131 int j = i - __kmp_affinity_compact;
4132 if (aa->childNums[j] < bb->childNums[j])
4134 if (aa->childNums[j] > bb->childNums[j])
4140 static void __kmp_aux_affinity_initialize(
void) {
4141 if (__kmp_affinity_masks != NULL) {
4142 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4150 if (__kmp_affin_fullMask == NULL) {
4151 KMP_CPU_ALLOC(__kmp_affin_fullMask);
4153 if (KMP_AFFINITY_CAPABLE()) {
4154 if (__kmp_affinity_respect_mask) {
4155 __kmp_get_system_affinity(__kmp_affin_fullMask, TRUE);
4159 __kmp_avail_proc = 0;
4160 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
4161 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
4166 if (__kmp_avail_proc > __kmp_xproc) {
4167 if (__kmp_affinity_verbose ||
4168 (__kmp_affinity_warnings &&
4169 (__kmp_affinity_type != affinity_none))) {
4170 KMP_WARNING(ErrorInitializeAffinity);
4172 __kmp_affinity_type = affinity_none;
4173 KMP_AFFINITY_DISABLE();
4177 __kmp_affinity_entire_machine_mask(__kmp_affin_fullMask);
4178 __kmp_avail_proc = __kmp_xproc;
4182 if (__kmp_affinity_gran == affinity_gran_tile &&
4184 __kmp_affinity_dispatch->get_api_type() == KMPAffinity::NATIVE_OS) {
4185 KMP_WARNING(AffTilesNoHWLOC,
"KMP_AFFINITY");
4186 __kmp_affinity_gran = affinity_gran_package;
4190 kmp_i18n_id_t msg_id = kmp_i18n_null;
4194 if ((__kmp_cpuinfo_file != NULL) &&
4195 (__kmp_affinity_top_method == affinity_top_method_all)) {
4196 __kmp_affinity_top_method = affinity_top_method_cpuinfo;
4199 if (__kmp_affinity_top_method == affinity_top_method_all) {
4203 const char *file_name = NULL;
4207 __kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) {
4208 if (__kmp_affinity_verbose) {
4209 KMP_INFORM(AffUsingHwloc,
"KMP_AFFINITY");
4211 if (!__kmp_hwloc_error) {
4212 depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
4215 }
else if (depth < 0 && __kmp_affinity_verbose) {
4216 KMP_INFORM(AffIgnoringHwloc,
"KMP_AFFINITY");
4218 }
else if (__kmp_affinity_verbose) {
4219 KMP_INFORM(AffIgnoringHwloc,
"KMP_AFFINITY");
4224 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
4227 if (__kmp_affinity_verbose) {
4228 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY", KMP_I18N_STR(Decodingx2APIC));
4232 depth = __kmp_affinity_create_x2apicid_map(&address2os, &msg_id);
4238 if (__kmp_affinity_verbose) {
4239 if (msg_id != kmp_i18n_null) {
4240 KMP_INFORM(AffInfoStrStr,
"KMP_AFFINITY",
4241 __kmp_i18n_catgets(msg_id),
4242 KMP_I18N_STR(DecodingLegacyAPIC));
4244 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY",
4245 KMP_I18N_STR(DecodingLegacyAPIC));
4250 depth = __kmp_affinity_create_apicid_map(&address2os, &msg_id);
4262 if (__kmp_affinity_verbose) {
4263 if (msg_id != kmp_i18n_null) {
4264 KMP_INFORM(AffStrParseFilename,
"KMP_AFFINITY",
4265 __kmp_i18n_catgets(msg_id),
"/proc/cpuinfo");
4267 KMP_INFORM(AffParseFilename,
"KMP_AFFINITY",
"/proc/cpuinfo");
4271 FILE *f = fopen(
"/proc/cpuinfo",
"r");
4273 msg_id = kmp_i18n_str_CantOpenCpuinfo;
4275 file_name =
"/proc/cpuinfo";
4277 __kmp_affinity_create_cpuinfo_map(&address2os, &line, &msg_id, f);
4287 #if KMP_GROUP_AFFINITY
4289 if ((depth < 0) && (__kmp_num_proc_groups > 1)) {
4290 if (__kmp_affinity_verbose) {
4291 KMP_INFORM(AffWindowsProcGroupMap,
"KMP_AFFINITY");
4294 depth = __kmp_affinity_create_proc_group_map(&address2os, &msg_id);
4295 KMP_ASSERT(depth != 0);
4301 if (__kmp_affinity_verbose && (msg_id != kmp_i18n_null)) {
4302 if (file_name == NULL) {
4303 KMP_INFORM(UsingFlatOS, __kmp_i18n_catgets(msg_id));
4304 }
else if (line == 0) {
4305 KMP_INFORM(UsingFlatOSFile, file_name, __kmp_i18n_catgets(msg_id));
4307 KMP_INFORM(UsingFlatOSFileLine, file_name, line,
4308 __kmp_i18n_catgets(msg_id));
4314 depth = __kmp_affinity_create_flat_map(&address2os, &msg_id);
4318 KMP_ASSERT(depth > 0);
4319 KMP_ASSERT(address2os != NULL);
4324 else if (__kmp_affinity_top_method == affinity_top_method_hwloc) {
4325 KMP_ASSERT(__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC);
4326 if (__kmp_affinity_verbose) {
4327 KMP_INFORM(AffUsingHwloc,
"KMP_AFFINITY");
4329 depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
4334 #endif // KMP_USE_HWLOC
4340 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
4342 else if (__kmp_affinity_top_method == affinity_top_method_x2apicid) {
4343 if (__kmp_affinity_verbose) {
4344 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY", KMP_I18N_STR(Decodingx2APIC));
4347 depth = __kmp_affinity_create_x2apicid_map(&address2os, &msg_id);
4352 KMP_ASSERT(msg_id != kmp_i18n_null);
4353 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4355 }
else if (__kmp_affinity_top_method == affinity_top_method_apicid) {
4356 if (__kmp_affinity_verbose) {
4357 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY", KMP_I18N_STR(DecodingLegacyAPIC));
4360 depth = __kmp_affinity_create_apicid_map(&address2os, &msg_id);
4365 KMP_ASSERT(msg_id != kmp_i18n_null);
4366 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4372 else if (__kmp_affinity_top_method == affinity_top_method_cpuinfo) {
4373 const char *filename;
4374 if (__kmp_cpuinfo_file != NULL) {
4375 filename = __kmp_cpuinfo_file;
4377 filename =
"/proc/cpuinfo";
4380 if (__kmp_affinity_verbose) {
4381 KMP_INFORM(AffParseFilename,
"KMP_AFFINITY", filename);
4384 FILE *f = fopen(filename,
"r");
4387 if (__kmp_cpuinfo_file != NULL) {
4388 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4389 KMP_HNT(NameComesFrom_CPUINFO_FILE), __kmp_msg_null);
4391 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4396 depth = __kmp_affinity_create_cpuinfo_map(&address2os, &line, &msg_id, f);
4399 KMP_ASSERT(msg_id != kmp_i18n_null);
4401 KMP_FATAL(FileLineMsgExiting, filename, line,
4402 __kmp_i18n_catgets(msg_id));
4404 KMP_FATAL(FileMsgExiting, filename, __kmp_i18n_catgets(msg_id));
4407 if (__kmp_affinity_type == affinity_none) {
4408 KMP_ASSERT(depth == 0);
4413 #if KMP_GROUP_AFFINITY
4415 else if (__kmp_affinity_top_method == affinity_top_method_group) {
4416 if (__kmp_affinity_verbose) {
4417 KMP_INFORM(AffWindowsProcGroupMap,
"KMP_AFFINITY");
4420 depth = __kmp_affinity_create_proc_group_map(&address2os, &msg_id);
4421 KMP_ASSERT(depth != 0);
4423 KMP_ASSERT(msg_id != kmp_i18n_null);
4424 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4430 else if (__kmp_affinity_top_method == affinity_top_method_flat) {
4431 if (__kmp_affinity_verbose) {
4432 KMP_INFORM(AffUsingFlatOS,
"KMP_AFFINITY");
4435 depth = __kmp_affinity_create_flat_map(&address2os, &msg_id);
4440 KMP_ASSERT(depth > 0);
4441 KMP_ASSERT(address2os != NULL);
4444 #if KMP_USE_HIER_SCHED
4445 __kmp_dispatch_set_hierarchy_values();
4448 if (address2os == NULL) {
4449 if (KMP_AFFINITY_CAPABLE() &&
4450 (__kmp_affinity_verbose ||
4451 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none)))) {
4452 KMP_WARNING(ErrorInitializeAffinity);
4454 __kmp_affinity_type = affinity_none;
4455 __kmp_create_affinity_none_places();
4456 KMP_AFFINITY_DISABLE();
4460 if (__kmp_affinity_gran == affinity_gran_tile
4462 && __kmp_tile_depth == 0
4466 KMP_WARNING(AffTilesNoTiles,
"KMP_AFFINITY");
4469 __kmp_apply_thread_places(&address2os, depth);
4474 kmp_affin_mask_t *osId2Mask =
4475 __kmp_create_masks(&maxIndex, &numUnique, address2os, __kmp_avail_proc);
4476 if (__kmp_affinity_gran_levels == 0) {
4477 KMP_DEBUG_ASSERT((
int)numUnique == __kmp_avail_proc);
4483 __kmp_affinity_assign_child_nums(address2os, __kmp_avail_proc);
4485 switch (__kmp_affinity_type) {
4487 case affinity_explicit:
4488 KMP_DEBUG_ASSERT(__kmp_affinity_proclist != NULL);
4490 if (__kmp_nested_proc_bind.bind_types[0] == proc_bind_intel)
4493 __kmp_affinity_process_proclist(
4494 &__kmp_affinity_masks, &__kmp_affinity_num_masks,
4495 __kmp_affinity_proclist, osId2Mask, maxIndex);
4499 __kmp_affinity_process_placelist(
4500 &__kmp_affinity_masks, &__kmp_affinity_num_masks,
4501 __kmp_affinity_proclist, osId2Mask, maxIndex);
4504 if (__kmp_affinity_num_masks == 0) {
4505 if (__kmp_affinity_verbose ||
4506 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
4507 KMP_WARNING(AffNoValidProcID);
4509 __kmp_affinity_type = affinity_none;
4510 __kmp_create_affinity_none_places();
4520 case affinity_logical:
4521 __kmp_affinity_compact = 0;
4522 if (__kmp_affinity_offset) {
4523 __kmp_affinity_offset =
4524 __kmp_nThreadsPerCore * __kmp_affinity_offset % __kmp_avail_proc;
4528 case affinity_physical:
4529 if (__kmp_nThreadsPerCore > 1) {
4530 __kmp_affinity_compact = 1;
4531 if (__kmp_affinity_compact >= depth) {
4532 __kmp_affinity_compact = 0;
4535 __kmp_affinity_compact = 0;
4537 if (__kmp_affinity_offset) {
4538 __kmp_affinity_offset =
4539 __kmp_nThreadsPerCore * __kmp_affinity_offset % __kmp_avail_proc;
4543 case affinity_scatter:
4544 if (__kmp_affinity_compact >= depth) {
4545 __kmp_affinity_compact = 0;
4547 __kmp_affinity_compact = depth - 1 - __kmp_affinity_compact;
4551 case affinity_compact:
4552 if (__kmp_affinity_compact >= depth) {
4553 __kmp_affinity_compact = depth - 1;
4557 case affinity_balanced:
4559 if (__kmp_affinity_verbose || __kmp_affinity_warnings) {
4560 KMP_WARNING(AffBalancedNotAvail,
"KMP_AFFINITY");
4562 __kmp_affinity_type = affinity_none;
4563 __kmp_create_affinity_none_places();
4565 }
else if (!__kmp_affinity_uniform_topology()) {
4567 __kmp_aff_depth = depth;
4569 int core_level = __kmp_affinity_find_core_level(
4570 address2os, __kmp_avail_proc, depth - 1);
4571 int ncores = __kmp_affinity_compute_ncores(address2os, __kmp_avail_proc,
4572 depth - 1, core_level);
4573 int maxprocpercore = __kmp_affinity_max_proc_per_core(
4574 address2os, __kmp_avail_proc, depth - 1, core_level);
4576 int nproc = ncores * maxprocpercore;
4577 if ((nproc < 2) || (nproc < __kmp_avail_proc)) {
4578 if (__kmp_affinity_verbose || __kmp_affinity_warnings) {
4579 KMP_WARNING(AffBalancedNotAvail,
"KMP_AFFINITY");
4581 __kmp_affinity_type = affinity_none;
4585 procarr = (
int *)__kmp_allocate(
sizeof(
int) * nproc);
4586 for (
int i = 0; i < nproc; i++) {
4592 for (
int i = 0; i < __kmp_avail_proc; i++) {
4593 int proc = address2os[i].second;
4595 __kmp_affinity_find_core(address2os, i, depth - 1, core_level);
4597 if (core == lastcore) {
4604 procarr[core * maxprocpercore + inlastcore] = proc;
4607 if (__kmp_affinity_compact >= depth) {
4608 __kmp_affinity_compact = depth - 1;
4613 if (__kmp_affinity_dups) {
4614 __kmp_affinity_num_masks = __kmp_avail_proc;
4616 __kmp_affinity_num_masks = numUnique;
4620 if ((__kmp_nested_proc_bind.bind_types[0] != proc_bind_intel) &&
4621 (__kmp_affinity_num_places > 0) &&
4622 ((
unsigned)__kmp_affinity_num_places < __kmp_affinity_num_masks)) {
4623 __kmp_affinity_num_masks = __kmp_affinity_num_places;
4627 KMP_CPU_ALLOC_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4631 qsort(address2os, __kmp_avail_proc,
sizeof(*address2os),
4632 __kmp_affinity_cmp_Address_child_num);
4636 for (i = 0, j = 0; i < __kmp_avail_proc; i++) {
4637 if ((!__kmp_affinity_dups) && (!address2os[i].first.leader)) {
4640 unsigned osId = address2os[i].second;
4641 kmp_affin_mask_t *src = KMP_CPU_INDEX(osId2Mask, osId);
4642 kmp_affin_mask_t *dest = KMP_CPU_INDEX(__kmp_affinity_masks, j);
4643 KMP_ASSERT(KMP_CPU_ISSET(osId, src));
4644 KMP_CPU_COPY(dest, src);
4645 if (++j >= __kmp_affinity_num_masks) {
4649 KMP_DEBUG_ASSERT(j == __kmp_affinity_num_masks);
4654 KMP_ASSERT2(0,
"Unexpected affinity setting");
4657 KMP_CPU_FREE_ARRAY(osId2Mask, maxIndex + 1);
4658 machine_hierarchy.init(address2os, __kmp_avail_proc);
4660 #undef KMP_EXIT_AFF_NONE
4662 void __kmp_affinity_initialize(
void) {
4671 int disabled = (__kmp_affinity_type == affinity_disabled);
4672 if (!KMP_AFFINITY_CAPABLE()) {
4673 KMP_ASSERT(disabled);
4676 __kmp_affinity_type = affinity_none;
4678 __kmp_aux_affinity_initialize();
4680 __kmp_affinity_type = affinity_disabled;
4684 void __kmp_affinity_uninitialize(
void) {
4685 if (__kmp_affinity_masks != NULL) {
4686 KMP_CPU_FREE_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4687 __kmp_affinity_masks = NULL;
4689 if (__kmp_affin_fullMask != NULL) {
4690 KMP_CPU_FREE(__kmp_affin_fullMask);
4691 __kmp_affin_fullMask = NULL;
4693 __kmp_affinity_num_masks = 0;
4694 __kmp_affinity_type = affinity_default;
4696 __kmp_affinity_num_places = 0;
4698 if (__kmp_affinity_proclist != NULL) {
4699 __kmp_free(__kmp_affinity_proclist);
4700 __kmp_affinity_proclist = NULL;
4702 if (address2os != NULL) {
4703 __kmp_free(address2os);
4706 if (procarr != NULL) {
4707 __kmp_free(procarr);
4711 if (__kmp_hwloc_topology != NULL) {
4712 hwloc_topology_destroy(__kmp_hwloc_topology);
4713 __kmp_hwloc_topology = NULL;
4716 KMPAffinity::destroy_api();
4719 void __kmp_affinity_set_init_mask(
int gtid,
int isa_root) {
4720 if (!KMP_AFFINITY_CAPABLE()) {
4724 kmp_info_t *th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[gtid]);
4725 if (th->th.th_affin_mask == NULL) {
4726 KMP_CPU_ALLOC(th->th.th_affin_mask);
4728 KMP_CPU_ZERO(th->th.th_affin_mask);
4735 kmp_affin_mask_t *mask;
4739 if (KMP_AFFINITY_NON_PROC_BIND)
4742 if ((__kmp_affinity_type == affinity_none) ||
4743 (__kmp_affinity_type == affinity_balanced)) {
4744 #if KMP_GROUP_AFFINITY
4745 if (__kmp_num_proc_groups > 1) {
4749 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4751 mask = __kmp_affin_fullMask;
4753 KMP_DEBUG_ASSERT(__kmp_affinity_num_masks > 0);
4754 i = (gtid + __kmp_affinity_offset) % __kmp_affinity_num_masks;
4755 mask = KMP_CPU_INDEX(__kmp_affinity_masks, i);
4761 (__kmp_nested_proc_bind.bind_types[0] == proc_bind_false)) {
4762 #if KMP_GROUP_AFFINITY
4763 if (__kmp_num_proc_groups > 1) {
4767 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4769 mask = __kmp_affin_fullMask;
4773 KMP_DEBUG_ASSERT(__kmp_affinity_num_masks > 0);
4774 i = (gtid + __kmp_affinity_offset) % __kmp_affinity_num_masks;
4775 mask = KMP_CPU_INDEX(__kmp_affinity_masks, i);
4781 th->th.th_current_place = i;
4783 th->th.th_new_place = i;
4784 th->th.th_first_place = 0;
4785 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4786 }
else if (KMP_AFFINITY_NON_PROC_BIND) {
4789 th->th.th_first_place = 0;
4790 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4793 if (i == KMP_PLACE_ALL) {
4794 KA_TRACE(100, (
"__kmp_affinity_set_init_mask: binding T#%d to all places\n",
4797 KA_TRACE(100, (
"__kmp_affinity_set_init_mask: binding T#%d to place %d\n",
4804 (
"__kmp_affinity_set_init_mask: binding T#%d to __kmp_affin_fullMask\n",
4807 KA_TRACE(100, (
"__kmp_affinity_set_init_mask: binding T#%d to mask %d\n",
4812 KMP_CPU_COPY(th->th.th_affin_mask, mask);
4814 if (__kmp_affinity_verbose
4816 && (__kmp_affinity_type == affinity_none ||
4817 (i != KMP_PLACE_ALL && __kmp_affinity_type != affinity_balanced))) {
4818 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4819 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4820 th->th.th_affin_mask);
4821 KMP_INFORM(BoundToOSProcSet,
"KMP_AFFINITY", (kmp_int32)getpid(),
4822 __kmp_gettid(), gtid, buf);
4829 if (__kmp_affinity_type == affinity_none) {
4830 __kmp_set_system_affinity(th->th.th_affin_mask, FALSE);
4833 __kmp_set_system_affinity(th->th.th_affin_mask, TRUE);
4838 void __kmp_affinity_set_place(
int gtid) {
4839 if (!KMP_AFFINITY_CAPABLE()) {
4843 kmp_info_t *th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[gtid]);
4845 KA_TRACE(100, (
"__kmp_affinity_set_place: binding T#%d to place %d (current "
4847 gtid, th->th.th_new_place, th->th.th_current_place));
4850 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4851 KMP_ASSERT(th->th.th_new_place >= 0);
4852 KMP_ASSERT((
unsigned)th->th.th_new_place <= __kmp_affinity_num_masks);
4853 if (th->th.th_first_place <= th->th.th_last_place) {
4854 KMP_ASSERT((th->th.th_new_place >= th->th.th_first_place) &&
4855 (th->th.th_new_place <= th->th.th_last_place));
4857 KMP_ASSERT((th->th.th_new_place <= th->th.th_first_place) ||
4858 (th->th.th_new_place >= th->th.th_last_place));
4863 kmp_affin_mask_t *mask =
4864 KMP_CPU_INDEX(__kmp_affinity_masks, th->th.th_new_place);
4865 KMP_CPU_COPY(th->th.th_affin_mask, mask);
4866 th->th.th_current_place = th->th.th_new_place;
4868 if (__kmp_affinity_verbose) {
4869 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4870 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4871 th->th.th_affin_mask);
4872 KMP_INFORM(BoundToOSProcSet,
"OMP_PROC_BIND", (kmp_int32)getpid(),
4873 __kmp_gettid(), gtid, buf);
4875 __kmp_set_system_affinity(th->th.th_affin_mask, TRUE);
4880 int __kmp_aux_set_affinity(
void **mask) {
4885 if (!KMP_AFFINITY_CAPABLE()) {
4889 gtid = __kmp_entry_gtid();
4890 KA_TRACE(1000, (
""); {
4891 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4892 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4893 (kmp_affin_mask_t *)(*mask));
4895 "kmp_set_affinity: setting affinity mask for thread %d = %s\n", gtid,
4899 if (__kmp_env_consistency_check) {
4900 if ((mask == NULL) || (*mask == NULL)) {
4901 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4906 KMP_CPU_SET_ITERATE(proc, ((kmp_affin_mask_t *)(*mask))) {
4907 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
4908 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4910 if (!KMP_CPU_ISSET(proc, (kmp_affin_mask_t *)(*mask))) {
4915 if (num_procs == 0) {
4916 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4919 #if KMP_GROUP_AFFINITY
4920 if (__kmp_get_proc_group((kmp_affin_mask_t *)(*mask)) < 0) {
4921 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4927 th = __kmp_threads[gtid];
4928 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4929 retval = __kmp_set_system_affinity((kmp_affin_mask_t *)(*mask), FALSE);
4931 KMP_CPU_COPY(th->th.th_affin_mask, (kmp_affin_mask_t *)(*mask));
4935 th->th.th_current_place = KMP_PLACE_UNDEFINED;
4936 th->th.th_new_place = KMP_PLACE_UNDEFINED;
4937 th->th.th_first_place = 0;
4938 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4941 th->th.th_current_task->td_icvs.proc_bind = proc_bind_false;
4947 int __kmp_aux_get_affinity(
void **mask) {
4952 if (!KMP_AFFINITY_CAPABLE()) {
4956 gtid = __kmp_entry_gtid();
4957 th = __kmp_threads[gtid];
4958 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4960 KA_TRACE(1000, (
""); {
4961 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4962 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4963 th->th.th_affin_mask);
4964 __kmp_printf(
"kmp_get_affinity: stored affinity mask for thread %d = %s\n",
4968 if (__kmp_env_consistency_check) {
4969 if ((mask == NULL) || (*mask == NULL)) {
4970 KMP_FATAL(AffinityInvalidMask,
"kmp_get_affinity");
4976 retval = __kmp_get_system_affinity((kmp_affin_mask_t *)(*mask), FALSE);
4977 KA_TRACE(1000, (
""); {
4978 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4979 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4980 (kmp_affin_mask_t *)(*mask));
4981 __kmp_printf(
"kmp_get_affinity: system affinity mask for thread %d = %s\n",
4988 KMP_CPU_COPY((kmp_affin_mask_t *)(*mask), th->th.th_affin_mask);
4994 int __kmp_aux_get_affinity_max_proc() {
4995 if (!KMP_AFFINITY_CAPABLE()) {
4998 #if KMP_GROUP_AFFINITY
4999 if (__kmp_num_proc_groups > 1) {
5000 return (
int)(__kmp_num_proc_groups *
sizeof(DWORD_PTR) * CHAR_BIT);
5006 int __kmp_aux_set_affinity_mask_proc(
int proc,
void **mask) {
5007 if (!KMP_AFFINITY_CAPABLE()) {
5011 KA_TRACE(1000, (
""); {
5012 int gtid = __kmp_entry_gtid();
5013 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5014 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
5015 (kmp_affin_mask_t *)(*mask));
5016 __kmp_debug_printf(
"kmp_set_affinity_mask_proc: setting proc %d in "
5017 "affinity mask for thread %d = %s\n",
5021 if (__kmp_env_consistency_check) {
5022 if ((mask == NULL) || (*mask == NULL)) {
5023 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity_mask_proc");
5027 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
5030 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
5034 KMP_CPU_SET(proc, (kmp_affin_mask_t *)(*mask));
5038 int __kmp_aux_unset_affinity_mask_proc(
int proc,
void **mask) {
5039 if (!KMP_AFFINITY_CAPABLE()) {
5043 KA_TRACE(1000, (
""); {
5044 int gtid = __kmp_entry_gtid();
5045 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5046 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
5047 (kmp_affin_mask_t *)(*mask));
5048 __kmp_debug_printf(
"kmp_unset_affinity_mask_proc: unsetting proc %d in "
5049 "affinity mask for thread %d = %s\n",
5053 if (__kmp_env_consistency_check) {
5054 if ((mask == NULL) || (*mask == NULL)) {
5055 KMP_FATAL(AffinityInvalidMask,
"kmp_unset_affinity_mask_proc");
5059 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
5062 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
5066 KMP_CPU_CLR(proc, (kmp_affin_mask_t *)(*mask));
5070 int __kmp_aux_get_affinity_mask_proc(
int proc,
void **mask) {
5071 if (!KMP_AFFINITY_CAPABLE()) {
5075 KA_TRACE(1000, (
""); {
5076 int gtid = __kmp_entry_gtid();
5077 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5078 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
5079 (kmp_affin_mask_t *)(*mask));
5080 __kmp_debug_printf(
"kmp_get_affinity_mask_proc: getting proc %d in "
5081 "affinity mask for thread %d = %s\n",
5085 if (__kmp_env_consistency_check) {
5086 if ((mask == NULL) || (*mask == NULL)) {
5087 KMP_FATAL(AffinityInvalidMask,
"kmp_get_affinity_mask_proc");
5091 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
5094 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
5098 return KMP_CPU_ISSET(proc, (kmp_affin_mask_t *)(*mask));
5102 void __kmp_balanced_affinity(kmp_info_t *th,
int nthreads) {
5103 KMP_DEBUG_ASSERT(th);
5104 bool fine_gran =
true;
5105 int tid = th->th.th_info.ds.ds_tid;
5107 switch (__kmp_affinity_gran) {
5108 case affinity_gran_fine:
5109 case affinity_gran_thread:
5111 case affinity_gran_core:
5112 if (__kmp_nThreadsPerCore > 1) {
5116 case affinity_gran_package:
5117 if (nCoresPerPkg > 1) {
5125 if (__kmp_affinity_uniform_topology()) {
5129 int __kmp_nth_per_core = __kmp_avail_proc / __kmp_ncores;
5131 int ncores = __kmp_ncores;
5132 if ((nPackages > 1) && (__kmp_nth_per_core <= 1)) {
5133 __kmp_nth_per_core = __kmp_avail_proc / nPackages;
5137 int chunk = nthreads / ncores;
5139 int big_cores = nthreads % ncores;
5141 int big_nth = (chunk + 1) * big_cores;
5142 if (tid < big_nth) {
5143 coreID = tid / (chunk + 1);
5144 threadID = (tid % (chunk + 1)) % __kmp_nth_per_core;
5146 coreID = (tid - big_cores) / chunk;
5147 threadID = ((tid - big_cores) % chunk) % __kmp_nth_per_core;
5150 KMP_DEBUG_ASSERT2(KMP_AFFINITY_CAPABLE(),
5151 "Illegal set affinity operation when not capable");
5153 kmp_affin_mask_t *mask = th->th.th_affin_mask;
5157 int osID = address2os[coreID * __kmp_nth_per_core + threadID].second;
5158 KMP_CPU_SET(osID, mask);
5160 for (
int i = 0; i < __kmp_nth_per_core; i++) {
5162 osID = address2os[coreID * __kmp_nth_per_core + i].second;
5163 KMP_CPU_SET(osID, mask);
5166 if (__kmp_affinity_verbose) {
5167 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5168 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, mask);
5169 KMP_INFORM(BoundToOSProcSet,
"KMP_AFFINITY", (kmp_int32)getpid(),
5170 __kmp_gettid(), tid, buf);
5172 __kmp_set_system_affinity(mask, TRUE);
5175 kmp_affin_mask_t *mask = th->th.th_affin_mask;
5178 int core_level = __kmp_affinity_find_core_level(
5179 address2os, __kmp_avail_proc, __kmp_aff_depth - 1);
5180 int ncores = __kmp_affinity_compute_ncores(address2os, __kmp_avail_proc,
5181 __kmp_aff_depth - 1, core_level);
5182 int nth_per_core = __kmp_affinity_max_proc_per_core(
5183 address2os, __kmp_avail_proc, __kmp_aff_depth - 1, core_level);
5187 if (nthreads == __kmp_avail_proc) {
5189 int osID = address2os[tid].second;
5190 KMP_CPU_SET(osID, mask);
5192 int core = __kmp_affinity_find_core(address2os, tid,
5193 __kmp_aff_depth - 1, core_level);
5194 for (
int i = 0; i < __kmp_avail_proc; i++) {
5195 int osID = address2os[i].second;
5196 if (__kmp_affinity_find_core(address2os, i, __kmp_aff_depth - 1,
5197 core_level) == core) {
5198 KMP_CPU_SET(osID, mask);
5202 }
else if (nthreads <= ncores) {
5205 for (
int i = 0; i < ncores; i++) {
5208 for (
int j = 0; j < nth_per_core; j++) {
5209 if (procarr[i * nth_per_core + j] != -1) {
5216 for (
int j = 0; j < nth_per_core; j++) {
5217 int osID = procarr[i * nth_per_core + j];
5219 KMP_CPU_SET(osID, mask);
5235 int *nproc_at_core = (
int *)KMP_ALLOCA(
sizeof(
int) * ncores);
5237 int *ncores_with_x_procs =
5238 (
int *)KMP_ALLOCA(
sizeof(
int) * (nth_per_core + 1));
5240 int *ncores_with_x_to_max_procs =
5241 (
int *)KMP_ALLOCA(
sizeof(
int) * (nth_per_core + 1));
5243 for (
int i = 0; i <= nth_per_core; i++) {
5244 ncores_with_x_procs[i] = 0;
5245 ncores_with_x_to_max_procs[i] = 0;
5248 for (
int i = 0; i < ncores; i++) {
5250 for (
int j = 0; j < nth_per_core; j++) {
5251 if (procarr[i * nth_per_core + j] != -1) {
5255 nproc_at_core[i] = cnt;
5256 ncores_with_x_procs[cnt]++;
5259 for (
int i = 0; i <= nth_per_core; i++) {
5260 for (
int j = i; j <= nth_per_core; j++) {
5261 ncores_with_x_to_max_procs[i] += ncores_with_x_procs[j];
5266 int nproc = nth_per_core * ncores;
5268 int *newarr = (
int *)__kmp_allocate(
sizeof(
int) * nproc);
5269 for (
int i = 0; i < nproc; i++) {
5276 for (
int j = 1; j <= nth_per_core; j++) {
5277 int cnt = ncores_with_x_to_max_procs[j];
5278 for (
int i = 0; i < ncores; i++) {
5280 if (nproc_at_core[i] == 0) {
5283 for (
int k = 0; k < nth_per_core; k++) {
5284 if (procarr[i * nth_per_core + k] != -1) {
5285 if (newarr[i * nth_per_core + k] == 0) {
5286 newarr[i * nth_per_core + k] = 1;
5292 newarr[i * nth_per_core + k]++;
5300 if (cnt == 0 || nth == 0) {
5311 for (
int i = 0; i < nproc; i++) {
5315 int osID = procarr[i];
5316 KMP_CPU_SET(osID, mask);
5318 int coreID = i / nth_per_core;
5319 for (
int ii = 0; ii < nth_per_core; ii++) {
5320 int osID = procarr[coreID * nth_per_core + ii];
5322 KMP_CPU_SET(osID, mask);
5332 if (__kmp_affinity_verbose) {
5333 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5334 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, mask);
5335 KMP_INFORM(BoundToOSProcSet,
"KMP_AFFINITY", (kmp_int32)getpid(),
5336 __kmp_gettid(), tid, buf);
5338 __kmp_set_system_affinity(mask, TRUE);
5356 kmp_set_thread_affinity_mask_initial()
5361 int gtid = __kmp_get_gtid();
5364 KA_TRACE(30, (
"kmp_set_thread_affinity_mask_initial: "
5365 "non-omp thread, returning\n"));
5368 if (!KMP_AFFINITY_CAPABLE() || !__kmp_init_middle) {
5369 KA_TRACE(30, (
"kmp_set_thread_affinity_mask_initial: "
5370 "affinity not initialized, returning\n"));
5373 KA_TRACE(30, (
"kmp_set_thread_affinity_mask_initial: "
5374 "set full mask for thread %d\n",
5376 KMP_DEBUG_ASSERT(__kmp_affin_fullMask != NULL);
5377 return __kmp_set_system_affinity(__kmp_affin_fullMask, FALSE);
5381 #endif // KMP_AFFINITY_SUPPORTED