63#define DEBUG_TYPE "openmp-ir-builder"
70 cl::desc(
"Use optimistic attributes describing "
71 "'as-if' properties of runtime calls."),
75 "openmp-ir-builder-unroll-threshold-factor",
cl::Hidden,
76 cl::desc(
"Factor for the unroll threshold to account for code "
77 "simplifications still taking place"),
88 if (!IP1.isSet() || !IP2.isSet())
90 return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
95 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
96 case OMPScheduleType::UnorderedStaticChunked:
97 case OMPScheduleType::UnorderedStatic:
98 case OMPScheduleType::UnorderedDynamicChunked:
99 case OMPScheduleType::UnorderedGuidedChunked:
100 case OMPScheduleType::UnorderedRuntime:
101 case OMPScheduleType::UnorderedAuto:
102 case OMPScheduleType::UnorderedTrapezoidal:
103 case OMPScheduleType::UnorderedGreedy:
104 case OMPScheduleType::UnorderedBalanced:
105 case OMPScheduleType::UnorderedGuidedIterativeChunked:
106 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
107 case OMPScheduleType::UnorderedSteal:
108 case OMPScheduleType::UnorderedStaticBalancedChunked:
109 case OMPScheduleType::UnorderedGuidedSimd:
110 case OMPScheduleType::UnorderedRuntimeSimd:
111 case OMPScheduleType::OrderedStaticChunked:
112 case OMPScheduleType::OrderedStatic:
113 case OMPScheduleType::OrderedDynamicChunked:
114 case OMPScheduleType::OrderedGuidedChunked:
115 case OMPScheduleType::OrderedRuntime:
116 case OMPScheduleType::OrderedAuto:
117 case OMPScheduleType::OrderdTrapezoidal:
118 case OMPScheduleType::NomergeUnorderedStaticChunked:
119 case OMPScheduleType::NomergeUnorderedStatic:
120 case OMPScheduleType::NomergeUnorderedDynamicChunked:
121 case OMPScheduleType::NomergeUnorderedGuidedChunked:
122 case OMPScheduleType::NomergeUnorderedRuntime:
123 case OMPScheduleType::NomergeUnorderedAuto:
124 case OMPScheduleType::NomergeUnorderedTrapezoidal:
125 case OMPScheduleType::NomergeUnorderedGreedy:
126 case OMPScheduleType::NomergeUnorderedBalanced:
127 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
128 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
129 case OMPScheduleType::NomergeUnorderedSteal:
130 case OMPScheduleType::NomergeOrderedStaticChunked:
131 case OMPScheduleType::NomergeOrderedStatic:
132 case OMPScheduleType::NomergeOrderedDynamicChunked:
133 case OMPScheduleType::NomergeOrderedGuidedChunked:
134 case OMPScheduleType::NomergeOrderedRuntime:
135 case OMPScheduleType::NomergeOrderedAuto:
136 case OMPScheduleType::NomergeOrderedTrapezoidal:
144 SchedType & OMPScheduleType::MonotonicityMask;
145 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
156 if (Features.
count(
"+wavefrontsize64"))
157 return omp::getAMDGPUGridValues<64>();
158 return omp::getAMDGPUGridValues<32>();
169 bool HasSimdModifier) {
171 switch (ClauseKind) {
172 case OMP_SCHEDULE_Default:
173 case OMP_SCHEDULE_Static:
174 return HasChunks ? OMPScheduleType::BaseStaticChunked
175 : OMPScheduleType::BaseStatic;
176 case OMP_SCHEDULE_Dynamic:
177 return OMPScheduleType::BaseDynamicChunked;
178 case OMP_SCHEDULE_Guided:
179 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
180 : OMPScheduleType::BaseGuidedChunked;
181 case OMP_SCHEDULE_Auto:
183 case OMP_SCHEDULE_Runtime:
184 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
185 : OMPScheduleType::BaseRuntime;
193 bool HasOrderedClause) {
194 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
195 OMPScheduleType::None &&
196 "Must not have ordering nor monotonicity flags already set");
199 ? OMPScheduleType::ModifierOrdered
200 : OMPScheduleType::ModifierUnordered;
201 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
204 if (OrderingScheduleType ==
205 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
206 return OMPScheduleType::OrderedGuidedChunked;
207 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
208 OMPScheduleType::ModifierOrdered))
209 return OMPScheduleType::OrderedRuntime;
211 return OrderingScheduleType;
217 bool HasSimdModifier,
bool HasMonotonic,
218 bool HasNonmonotonic,
bool HasOrderedClause) {
219 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
220 OMPScheduleType::None &&
221 "Must not have monotonicity flags already set");
222 assert((!HasMonotonic || !HasNonmonotonic) &&
223 "Monotonic and Nonmonotonic are contradicting each other");
226 return ScheduleType | OMPScheduleType::ModifierMonotonic;
227 }
else if (HasNonmonotonic) {
228 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
238 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
239 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
245 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
253 bool HasSimdModifier,
bool HasMonotonicModifier,
254 bool HasNonmonotonicModifier,
bool HasOrderedClause) {
260 OrderedSchedule, HasSimdModifier, HasMonotonicModifier,
261 HasNonmonotonicModifier, HasOrderedClause);
275 auto *Br = cast<BranchInst>(Term);
276 assert(!Br->isConditional() &&
277 "BB's terminator must be an unconditional branch (or degenerate)");
280 Br->setSuccessor(0,
Target);
285 NewBr->setDebugLoc(
DL);
290 assert(New->getFirstInsertionPt() == New->begin() &&
291 "Target BB must not have PHI nodes");
295 New->splice(New->begin(), Old, IP.
getPoint(), Old->
end());
323 New->replaceSuccessorsPhiUsesWith(Old, New);
368 const Twine &
Name =
"",
bool AsPtr =
true) {
376 FakeVal = FakeValAddr;
404enum OpenMPOffloadingRequiresDirFlags {
406 OMP_REQ_UNDEFINED = 0x000,
408 OMP_REQ_NONE = 0x001,
410 OMP_REQ_REVERSE_OFFLOAD = 0x002,
412 OMP_REQ_UNIFIED_ADDRESS = 0x004,
414 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
416 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
423 : RequiresFlags(OMP_REQ_UNDEFINED) {}
426 bool IsTargetDevice,
bool IsGPU,
bool OpenMPOffloadMandatory,
427 bool HasRequiresReverseOffload,
bool HasRequiresUnifiedAddress,
428 bool HasRequiresUnifiedSharedMemory,
bool HasRequiresDynamicAllocators)
429 : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
430 OpenMPOffloadMandatory(OpenMPOffloadMandatory),
431 RequiresFlags(OMP_REQ_UNDEFINED) {
432 if (HasRequiresReverseOffload)
433 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
434 if (HasRequiresUnifiedAddress)
435 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
436 if (HasRequiresUnifiedSharedMemory)
437 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
438 if (HasRequiresDynamicAllocators)
439 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
443 return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
447 return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
451 return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
455 return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
460 :
static_cast<int64_t
>(OMP_REQ_NONE);
465 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
467 RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
472 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
474 RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
479 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
481 RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
486 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
488 RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
501 constexpr const size_t MaxDim = 3;
509 Value *NumThreads3D =
512 seq<unsigned>(1, std::min(KernelArgs.
NumTeams.size(), MaxDim)))
516 seq<unsigned>(1, std::min(KernelArgs.
NumThreads.size(), MaxDim)))
540 auto FnAttrs = Attrs.getFnAttrs();
541 auto RetAttrs = Attrs.getRetAttrs();
543 for (
size_t ArgNo = 0; ArgNo < Fn.
arg_size(); ++ArgNo)
548 bool Param =
true) ->
void {
549 bool HasSignExt = AS.hasAttribute(Attribute::SExt);
550 bool HasZeroExt = AS.hasAttribute(Attribute::ZExt);
551 if (HasSignExt || HasZeroExt) {
552 assert(AS.getNumAttributes() == 1 &&
553 "Currently not handling extension attr combined with others.");
555 if (
auto AK = TargetLibraryInfo::getExtAttrForI32Param(
T, HasSignExt))
558 TargetLibraryInfo::getExtAttrForI32Return(
T, HasSignExt))
565#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
566#include "llvm/Frontend/OpenMP/OMPKinds.def"
570#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
572 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
573 addAttrSet(RetAttrs, RetAttrSet, false); \
574 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
575 addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
576 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
578#include "llvm/Frontend/OpenMP/OMPKinds.def"
592#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
594 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
596 Fn = M.getFunction(Str); \
598#include "llvm/Frontend/OpenMP/OMPKinds.def"
604#define OMP_RTL(Enum, Str, ...) \
606 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
608#include "llvm/Frontend/OpenMP/OMPKinds.def"
612 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
622 LLVMContext::MD_callback,
624 2, {-1, -1},
true)}));
637 assert(Fn &&
"Failed to create OpenMP runtime function");
644 auto *Fn = dyn_cast<llvm::Function>(RTLFn.
getCallee());
645 assert(Fn &&
"Failed to create OpenMP runtime function pointer");
660 for (
auto Inst =
Block->getReverseIterator()->begin();
661 Inst !=
Block->getReverseIterator()->end();) {
662 if (
auto *
AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Inst)) {
686 ParallelRegionBlockSet.
clear();
688 OI.collectBlocks(ParallelRegionBlockSet,
Blocks);
707 ".omp_par", ArgsInZeroAddressSpace);
711 <<
" Exit: " << OI.ExitBB->getName() <<
"\n");
713 "Expected OpenMP outlining to be possible!");
715 for (
auto *V : OI.ExcludeArgsFromAggregate)
722 if (TargetCpuAttr.isStringAttribute())
725 auto TargetFeaturesAttr = OuterFn->
getFnAttribute(
"target-features");
726 if (TargetFeaturesAttr.isStringAttribute())
727 OutlinedFn->
addFnAttr(TargetFeaturesAttr);
730 LLVM_DEBUG(
dbgs() <<
" Outlined function: " << *OutlinedFn <<
"\n");
732 "OpenMP outlined functions should not return a value!");
744 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry);
751 "Expected instructions to add in the outlined region entry");
758 if (
I.isTerminator())
761 I.moveBeforePreserving(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
764 OI.EntryBB->moveBefore(&ArtificialEntry);
771 if (OI.PostOutlineCB)
772 OI.PostOutlineCB(*OutlinedFn);
803 errs() <<
"Error of kind: " << Kind
804 <<
" when emitting offload entries and metadata during "
805 "OMPIRBuilder finalization \n";
812 std::vector<WeakTrackingVH> LLVMCompilerUsed = {
814 emitUsed(
"llvm.compiler.used", LLVMCompilerUsed);
836 unsigned Reserve2Flags) {
838 LocFlags |= OMP_IDENT_FLAG_KMPC;
846 ConstantInt::get(
Int32, Reserve2Flags),
847 ConstantInt::get(
Int32, SrcLocStrSize), SrcLocStr};
854 if (
GV.getValueType() == OpenMPIRBuilder::Ident &&
GV.hasInitializer())
855 if (
GV.getInitializer() == Initializer)
860 M, OpenMPIRBuilder::Ident,
875 SrcLocStrSize = LocStr.
size();
884 if (
GV.isConstant() &&
GV.hasInitializer() &&
885 GV.getInitializer() == Initializer)
896 unsigned Line,
unsigned Column,
902 Buffer.
append(FunctionName);
904 Buffer.
append(std::to_string(Line));
906 Buffer.
append(std::to_string(Column));
914 StringRef UnknownLoc =
";unknown;unknown;0;0;;";
925 if (
DIFile *DIF = DIL->getFile())
926 if (std::optional<StringRef> Source = DIF->getSource())
932 DIL->getColumn(), SrcLocStrSize);
944 "omp_global_thread_num");
949 bool ForceSimpleCall,
bool CheckCancelFlag) {
959 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
962 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
965 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
968 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
971 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
984 bool UseCancelBarrier =
989 UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
990 : OMPRTL___kmpc_barrier),
993 if (UseCancelBarrier && CheckCancelFlag)
1003 omp::Directive CanceledDirective) {
1015 Value *CancelKind =
nullptr;
1016 switch (CanceledDirective) {
1017#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1018 case DirectiveEnum: \
1019 CancelKind = Builder.getInt32(Value); \
1021#include "llvm/Frontend/OpenMP/OMPKinds.def"
1033 if (CanceledDirective == OMPD_parallel) {
1037 omp::Directive::OMPD_unknown,
1051 UI->eraseFromParent();
1064 auto *KernelArgsPtr =
1077 NumThreads, HostPtr, KernelArgsPtr};
1105 assert(OutlinedFnID &&
"Invalid outlined function ID!");
1109 Value *Return =
nullptr;
1129 Builder, AllocaIP, Return, RTLoc, DeviceID, Args.NumTeams.front(),
1130 Args.NumThreads.front(), OutlinedFnID, ArgsVector));
1146 emitBlock(OffloadContBlock, CurFn,
true);
1151 Value *CancelFlag, omp::Directive CanceledDirective,
1154 "Unexpected cancellation!");
1207 OutlinedFn.
addFnAttr(Attribute::NoUnwind);
1210 "Expected at least tid and bounded tid as arguments");
1211 unsigned NumCapturedVars = OutlinedFn.
arg_size() - 2;
1214 assert(CI &&
"Expected call instruction to outlined function");
1215 CI->
getParent()->setName(
"omp_parallel");
1218 Type *PtrTy = OMPIRBuilder->VoidPtr;
1222 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.
saveIP();
1226 Value *Args = ArgsAlloca;
1234 for (
unsigned Idx = 0;
Idx < NumCapturedVars;
Idx++) {
1246 Value *Parallel51CallArgs[] = {
1250 NumThreads ? NumThreads : Builder.
getInt32(-1),
1253 Builder.
CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr),
1256 Builder.
getInt64(NumCapturedVars)};
1261 Builder.
CreateCall(RTLFn, Parallel51CallArgs);
1276 I->eraseFromParent();
1298 if (
auto *
F = dyn_cast<Function>(RTLFn.
getCallee())) {
1299 if (!
F->hasMetadata(LLVMContext::MD_callback)) {
1307 F->addMetadata(LLVMContext::MD_callback,
1316 OutlinedFn.
addFnAttr(Attribute::NoUnwind);
1319 "Expected at least tid and bounded tid as arguments");
1320 unsigned NumCapturedVars = OutlinedFn.
arg_size() - 2;
1323 CI->
getParent()->setName(
"omp_parallel");
1327 Value *ForkCallArgs[] = {
1328 Ident, Builder.
getInt32(NumCapturedVars),
1329 Builder.
CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr)};
1332 RealArgs.
append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
1341 auto PtrTy = OMPIRBuilder->VoidPtr;
1342 if (IfCondition && NumCapturedVars == 0) {
1346 if (IfCondition && RealArgs.
back()->getType() != PtrTy)
1364 I->eraseFromParent();
1372 omp::ProcBindKind ProcBind,
bool IsCancellable) {
1399 if (ProcBind != OMP_PROC_BIND_default) {
1403 ConstantInt::get(
Int32,
unsigned(ProcBind),
true)};
1431 TIDAddrAlloca, PointerType ::get(
M.
getContext(), 0),
"tid.addr.ascast");
1436 "zero.addr.ascast");
1460 if (IP.getBlock()->end() == IP.getPoint()) {
1466 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
1467 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
1468 "Unexpected insertion point for finalization call!");
1504 LLVM_DEBUG(
dbgs() <<
"Before body codegen: " << *OuterFn <<
"\n");
1507 assert(BodyGenCB &&
"Expected body generation callback!");
1509 if (
Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP))
1512 LLVM_DEBUG(
dbgs() <<
"After body codegen: " << *OuterFn <<
"\n");
1518 std::move(ToBeDeleted)](
Function &OutlinedFn) {
1520 IfCondition, NumThreads, PrivTID, PrivTIDAddr,
1521 ThreadID, ToBeDeletedVec);
1526 std::move(ToBeDeleted)](
Function &OutlinedFn) {
1528 PrivTID, PrivTIDAddr, ToBeDeletedVec);
1545 PRegOutlinedExitBB->
setName(
"omp.par.outlined.exit");
1546 Blocks.push_back(PRegOutlinedExitBB);
1557 ".omp_par", ArgsInZeroAddressSpace);
1562 Extractor.
findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1568 if (
auto *
GV = dyn_cast_if_present<GlobalVariable>(
I))
1569 return GV->getValueType() == OpenMPIRBuilder::Ident;
1574 LLVM_DEBUG(
dbgs() <<
"Before privatization: " << *OuterFn <<
"\n");
1580 if (&V == TIDAddr || &V == ZeroAddr) {
1586 for (
Use &U : V.uses())
1587 if (
auto *UserI = dyn_cast<Instruction>(U.getUser()))
1588 if (ParallelRegionBlockSet.
count(UserI->getParent()))
1598 if (!V.getType()->isPointerTy()) {
1617 Value *ReplacementValue =
nullptr;
1618 CallInst *CI = dyn_cast<CallInst>(&V);
1620 ReplacementValue = PrivTID;
1623 PrivCB(InnerAllocaIP,
Builder.
saveIP(), V, *Inner, ReplacementValue);
1631 assert(ReplacementValue &&
1632 "Expected copy/create callback to set replacement value!");
1633 if (ReplacementValue == &V)
1638 UPtr->set(ReplacementValue);
1657 for (
Value *Input : Inputs) {
1659 if (
Error Err = PrivHelper(*Input))
1663 for (
Value *Output : Outputs)
1667 "OpenMP outlining should not produce live-out values!");
1669 LLVM_DEBUG(
dbgs() <<
"After privatization: " << *OuterFn <<
"\n");
1672 dbgs() <<
" PBR: " << BB->getName() <<
"\n";
1680 assert(FiniInfo.DK == OMPD_parallel &&
1681 "Unexpected finalization stack state!");
1686 if (
Error Err = FiniCB(PreFiniIP))
1692 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
1693 UI->eraseFromParent();
1759 if (Dependencies.
empty())
1779 Type *DependInfo = OMPBuilder.DependInfo;
1782 Value *DepArray =
nullptr;
1788 DepArray = Builder.
CreateAlloca(DepArrayTy,
nullptr,
".dep.arr.addr");
1790 for (
const auto &[DepIdx, Dep] :
enumerate(Dependencies)) {
1796 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
1801 DependInfo,
Base,
static_cast<unsigned int>(RTLDependInfoFields::Len));
1803 Builder.
getInt64(M.getDataLayout().getTypeStoreSize(Dep.DepValueType)),
1808 static_cast<unsigned int>(RTLDependInfoFields::Flags));
1811 static_cast<unsigned int>(Dep.DepKind)),
1853 if (
Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP))
1864 Builder, AllocaIP, ToBeDeleted, TaskAllocaIP,
"global.tid",
false));
1866 OI.
PostOutlineCB = [
this, Ident, Tied, Final, IfCondition, Dependencies,
1867 Mergeable, EventHandle, TaskAllocaBB,
1868 ToBeDeleted](
Function &OutlinedFn)
mutable {
1870 assert(OutlinedFn.getNumUses() == 1 &&
1871 "there must be a single user for the outlined function");
1872 CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
1876 bool HasShareds = StaleCI->
arg_size() > 1;
1920 assert(ArgStructAlloca &&
1921 "Unable to find the alloca instruction corresponding to arguments "
1922 "for extracted function");
1925 assert(ArgStructType &&
"Unable to find struct type corresponding to "
1926 "arguments for extracted function");
1934 TaskAllocFn, {Ident, ThreadID,
Flags,
1935 TaskSize, SharedsSize,
1943 OMPRTL___kmpc_task_allow_completion_event);
1961 Value *DepArray =
nullptr;
1962 if (Dependencies.
size()) {
1977 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
1984 static_cast<unsigned int>(RTLDependInfoFields::Len));
1991 static_cast<unsigned int>(RTLDependInfoFields::Flags));
1994 static_cast<unsigned int>(Dep.DepKind)),
2025 Instruction *ThenTI = IfTerminator, *ElseTI =
nullptr;
2031 if (Dependencies.
size()) {
2055 if (Dependencies.
size()) {
2076 Shareds, [Shareds](
Use &U) {
return U.getUser() != Shareds; });
2080 I->eraseFromParent();
2129 if (IP.getBlock()->end() != IP.getPoint())
2140 auto *CaseBB = IP.getBlock()->getSinglePredecessor();
2141 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
2142 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
2173 unsigned CaseNumber = 0;
2174 for (
auto SectionCB : SectionCBs) {
2192 Value *LB = ConstantInt::get(I32Ty, 0);
2193 Value *UB = ConstantInt::get(I32Ty, SectionCBs.
size());
2194 Value *ST = ConstantInt::get(I32Ty, 1);
2196 Loc, LoopBodyGenCB, LB, UB, ST,
true,
false, AllocaIP,
"section_loop");
2201 applyStaticWorkshareLoop(Loc.
DL, *
LoopInfo, AllocaIP, !IsNowait);
2208 assert(FiniInfo.DK == OMPD_sections &&
2209 "Unexpected finalization stack state!");
2216 AfterIP = {FiniBB, FiniBB->
begin()};
2230 if (IP.getBlock()->end() != IP.getPoint())
2249 Directive OMPD = Directive::OMPD_sections;
2252 return EmitOMPInlinedRegion(OMPD,
nullptr,
nullptr, BodyGenCB, FiniCBWrapper,
2264 std::vector<WeakTrackingVH> &
List) {
2271 for (
unsigned I = 0, E =
List.size();
I != E; ++
I)
2275 if (UsedArray.
empty())
2282 GV->setSection(
"llvm.metadata");
2285Value *OpenMPIRBuilder::getGPUThreadID() {
2288 OMPRTL___kmpc_get_hardware_thread_id_in_block),
2292Value *OpenMPIRBuilder::getGPUWarpSize() {
2297Value *OpenMPIRBuilder::getNVPTXWarpID() {
2302Value *OpenMPIRBuilder::getNVPTXLaneID() {
2304 assert(LaneIDBits < 32 &&
"Invalid LaneIDBits size in NVPTX device.");
2305 unsigned LaneIDMask = ~0
u >> (32u - LaneIDBits);
2310Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP,
Value *
From,
2315 assert(FromSize > 0 &&
"From size must be greater than zero");
2316 assert(ToSize > 0 &&
"To size must be greater than zero");
2317 if (FromType == ToType)
2319 if (FromSize == ToSize)
2334Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
2339 assert(
Size <= 8 &&
"Unsupported bitwidth in shuffle instruction");
2343 Value *ElemCast = castValueToType(AllocaIP, Element, CastTy);
2347 Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
2348 : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
2349 Value *WarpSizeCast =
2351 Value *ShuffleCall =
2353 return castValueToType(AllocaIP, ShuffleCall, CastTy);
2356void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP,
Value *SrcAddr,
2372 Value *ElemPtr = DstAddr;
2374 for (
unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
2386 if ((
Size / IntSize) > 1) {
2410 Value *Res = createRuntimeShuffleFunction(
2419 Value *LocalElemPtr =
2426 Value *Res = createRuntimeShuffleFunction(
2440void OpenMPIRBuilder::emitReductionListCopy(
2441 InsertPointTy AllocaIP, CopyAction Action,
Type *ReductionArrayTy,
2443 CopyOptionsTy CopyOptions) {
2446 Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
2450 for (
auto En :
enumerate(ReductionInfos)) {
2451 const ReductionInfo &RI = En.value();
2452 Value *SrcElementAddr =
nullptr;
2453 Value *DestElementAddr =
nullptr;
2454 Value *DestElementPtrAddr =
nullptr;
2456 bool ShuffleInElement =
false;
2459 bool UpdateDestListPtr =
false;
2463 ReductionArrayTy, SrcBase,
2464 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
2470 ReductionArrayTy, DestBase,
2471 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
2477 ".omp.reduction.element");
2480 DestElementAddr = DestAlloca;
2483 DestElementAddr->
getName() +
".ascast");
2485 ShuffleInElement =
true;
2486 UpdateDestListPtr =
true;
2498 if (ShuffleInElement) {
2499 shuffleAndStore(AllocaIP, SrcElementAddr, DestElementAddr, RI.ElementType,
2500 RemoteLaneOffset, ReductionArrayTy);
2502 switch (RI.EvaluationKind) {
2511 RI.ElementType, SrcElementAddr, 0, 0,
".realp");
2513 RI.ElementType->getStructElementType(0), SrcRealPtr,
".real");
2515 RI.ElementType, SrcElementAddr, 0, 1,
".imagp");
2517 RI.ElementType->getStructElementType(1), SrcImgPtr,
".imag");
2520 RI.ElementType, DestElementAddr, 0, 0,
".realp");
2522 RI.ElementType, DestElementAddr, 0, 1,
".imagp");
2544 if (UpdateDestListPtr) {
2547 DestElementAddr->
getName() +
".ascast");
2563 "_omp_reduction_inter_warp_copy_func", &
M);
2586 "__openmp_nvptx_data_transfer_temporary_storage";
2590 if (!TransferMedium) {
2599 Value *GPUThreadID = getGPUThreadID();
2601 Value *LaneID = getNVPTXLaneID();
2603 Value *WarpID = getNVPTXWarpID();
2612 Arg0Type,
nullptr, ReduceListArg->
getName() +
".addr");
2616 ReduceListAlloca, Arg0Type, ReduceListAlloca->
getName() +
".ascast");
2619 NumWarpsAlloca->
getName() +
".ascast");
2630 for (
auto En :
enumerate(ReductionInfos)) {
2635 const ReductionInfo &RI = En.value();
2637 for (
unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
2640 unsigned NumIters = RealTySize / TySize;
2643 Value *Cnt =
nullptr;
2644 Value *CntAddr =
nullptr;
2654 CntAddr->
getName() +
".ascast");
2674 omp::Directive::OMPD_unknown,
2678 return BarrierIP1.takeError();
2689 auto *RedListArrayTy =
2695 {ConstantInt::get(IndexTy, 0),
2696 ConstantInt::get(IndexTy, En.index())});
2722 omp::Directive::OMPD_unknown,
2726 return BarrierIP2.takeError();
2733 Value *NumWarpsVal =
2736 Value *IsActiveThread =
2747 Value *TargetElemPtrPtr =
2749 {ConstantInt::get(IndexTy, 0),
2750 ConstantInt::get(IndexTy, En.index())});
2751 Value *TargetElemPtrVal =
2753 Value *TargetElemPtr = TargetElemPtrVal;
2759 Value *SrcMediumValue =
2778 RealTySize %= TySize;
2788Function *OpenMPIRBuilder::emitShuffleAndReduceFunction(
2794 {Builder.getPtrTy(), Builder.getInt16Ty(),
2795 Builder.getInt16Ty(), Builder.getInt16Ty()},
2799 "_omp_reduction_shuffle_and_reduce_func", &
M);
2820 Type *ReduceListArgType = ReduceListArg->
getType();
2824 ReduceListArgType,
nullptr, ReduceListArg->
getName() +
".addr");
2826 LaneIDArg->
getName() +
".addr");
2828 LaneIDArgType,
nullptr, RemoteLaneOffsetArg->
getName() +
".addr");
2830 AlgoVerArg->
getName() +
".addr");
2837 RedListArrayTy,
nullptr,
".omp.reduction.remote_reduce_list");
2840 ReduceListAlloca, ReduceListArgType,
2841 ReduceListAlloca->
getName() +
".ascast");
2843 LaneIdAlloca, LaneIDArgPtrType, LaneIdAlloca->
getName() +
".ascast");
2845 RemoteLaneOffsetAlloca, LaneIDArgPtrType,
2846 RemoteLaneOffsetAlloca->
getName() +
".ascast");
2848 AlgoVerAlloca, LaneIDArgPtrType, AlgoVerAlloca->
getName() +
".ascast");
2851 RemoteReductionListAlloca->
getName() +
".ascast");
2860 Value *RemoteLaneOffset =
2869 emitReductionListCopy(
2871 ReduceList, RemoteListAddrCast, {RemoteLaneOffset,
nullptr,
nullptr});
2902 Value *RemoteOffsetComp =
2919 ->addFnAttr(Attribute::NoUnwind);
2940 ReductionInfos, RemoteListAddrCast, ReduceList);
2953Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(
2960 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
2964 "_omp_reduction_list_to_global_copy_func", &
M);
2981 BufferArg->
getName() +
".addr");
2988 BufferArgAlloca->
getName() +
".ascast");
2993 ReduceListArgAlloca->
getName() +
".ascast");
2999 Value *LocalReduceList =
3001 Value *BufferArgVal =
3006 for (
auto En :
enumerate(ReductionInfos)) {
3007 const ReductionInfo &RI = En.value();
3008 auto *RedListArrayTy =
3012 RedListArrayTy, LocalReduceList,
3013 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3021 ReductionsBufferTy, BufferVD, 0, En.index());
3023 switch (RI.EvaluationKind) {
3031 RI.ElementType, ElemPtr, 0, 0,
".realp");
3033 RI.ElementType->getStructElementType(0), SrcRealPtr,
".real");
3035 RI.ElementType, ElemPtr, 0, 1,
".imagp");
3037 RI.ElementType->getStructElementType(1), SrcImgPtr,
".imag");
3040 RI.ElementType, GlobVal, 0, 0,
".realp");
3042 RI.ElementType, GlobVal, 0, 1,
".imagp");
3063Function *OpenMPIRBuilder::emitListToGlobalReduceFunction(
3070 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3074 "_omp_reduction_list_to_global_reduce_func", &
M);
3091 BufferArg->
getName() +
".addr");
3096 auto *RedListArrayTy =
3101 Value *LocalReduceList =
3106 BufferArgAlloca->
getName() +
".ascast");
3111 ReduceListArgAlloca->
getName() +
".ascast");
3114 LocalReduceList->
getName() +
".ascast");
3124 for (
auto En :
enumerate(ReductionInfos)) {
3126 RedListArrayTy, LocalReduceListAddrCast,
3127 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3132 ReductionsBufferTy, BufferVD, 0, En.index());
3140 ->addFnAttr(Attribute::NoUnwind);
3146Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(
3153 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3157 "_omp_reduction_global_to_list_copy_func", &
M);
3174 BufferArg->
getName() +
".addr");
3181 BufferArgAlloca->
getName() +
".ascast");
3186 ReduceListArgAlloca->
getName() +
".ascast");
3191 Value *LocalReduceList =
3197 for (
auto En :
enumerate(ReductionInfos)) {
3199 auto *RedListArrayTy =
3203 RedListArrayTy, LocalReduceList,
3204 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3211 ReductionsBufferTy, BufferVD, 0, En.index());
3254Function *OpenMPIRBuilder::emitGlobalToListReduceFunction(
3261 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3265 "_omp_reduction_global_to_list_reduce_func", &
M);
3282 BufferArg->
getName() +
".addr");
3292 Value *LocalReduceList =
3297 BufferArgAlloca->
getName() +
".ascast");
3302 ReduceListArgAlloca->
getName() +
".ascast");
3305 LocalReduceList->
getName() +
".ascast");
3315 for (
auto En :
enumerate(ReductionInfos)) {
3317 RedListArrayTy, ReductionList,
3318 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3323 ReductionsBufferTy, BufferVD, 0, En.index());
3331 ->addFnAttr(Attribute::NoUnwind);
3337std::string OpenMPIRBuilder::getReductionFuncName(
StringRef Name)
const {
3338 std::string Suffix =
3340 return (
Name + Suffix).str();
3345 ReductionGenCBKind ReductionGenCBKind,
AttributeList FuncAttrs) {
3347 {Builder.getPtrTy(), Builder.getPtrTy()},
3349 std::string
Name = getReductionFuncName(ReducerName);
3361 Value *LHSArrayPtr =
nullptr;
3362 Value *RHSArrayPtr =
nullptr;
3373 LHSAlloca, Arg0Type, LHSAlloca->
getName() +
".ascast");
3375 RHSAlloca, Arg1Type, RHSAlloca->
getName() +
".ascast");
3385 for (
auto En :
enumerate(ReductionInfos)) {
3386 const ReductionInfo &RI = En.value();
3388 RedArrayTy, RHSArrayPtr,
3389 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3392 RHSI8Ptr, RI.PrivateVariable->getType(),
3393 RHSI8Ptr->
getName() +
".ascast");
3396 RedArrayTy, LHSArrayPtr,
3397 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3400 LHSI8Ptr, RI.Variable->getType(), LHSI8Ptr->
getName() +
".ascast");
3412 return AfterIP.takeError();
3414 return ReductionFunc;
3420 for (
auto En :
enumerate(ReductionInfos)) {
3421 unsigned Index = En.index();
3422 const ReductionInfo &RI = En.value();
3423 Value *LHSFixupPtr, *RHSFixupPtr;
3425 Builder.
saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
3430 LHSPtrs[Index], [ReductionFunc](
const Use &U) {
3431 return cast<Instruction>(
U.getUser())->getParent()->getParent() ==
3435 RHSPtrs[Index], [ReductionFunc](
const Use &U) {
3436 return cast<Instruction>(
U.getUser())->getParent()->getParent() ==
3442 return ReductionFunc;
3450 assert(RI.Variable &&
"expected non-null variable");
3451 assert(RI.PrivateVariable &&
"expected non-null private variable");
3452 assert((RI.ReductionGen || RI.ReductionGenClang) &&
3453 "expected non-null reduction generator callback");
3456 RI.Variable->getType() == RI.PrivateVariable->getType() &&
3457 "expected variables and their private equivalents to have the same "
3460 assert(RI.Variable->getType()->isPointerTy() &&
3461 "expected variables to be pointers");
3468 bool IsNoWait,
bool IsTeamsReduction,
bool HasDistribute,
3470 unsigned ReductionBufNum,
Value *SrcLocInfo) {
3484 if (ReductionInfos.
size() == 0)
3499 if (!ReductionResult)
3501 Function *ReductionFunc = *ReductionResult;
3505 if (GridValue.has_value())
3523 Value *ReductionListAlloca =
3526 ReductionListAlloca, PtrTy, ReductionListAlloca->
getName() +
".ascast");
3530 for (
auto En :
enumerate(ReductionInfos)) {
3533 RedArrayTy, ReductionList,
3534 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3541 emitShuffleAndReduceFunction(ReductionInfos, ReductionFunc, FuncAttrs);
3543 emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs);
3551 unsigned MaxDataSize = 0;
3553 for (
auto En :
enumerate(ReductionInfos)) {
3555 if (
Size > MaxDataSize)
3557 ReductionTypeArgs.
emplace_back(En.value().ElementType);
3559 Value *ReductionDataSize =
3561 if (!IsTeamsReduction) {
3562 Value *SarFuncCast =
3566 Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
3569 RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
3574 Ctx, ReductionTypeArgs,
"struct._globalized_locals_ty");
3576 RuntimeFunction::OMPRTL___kmpc_reduction_get_fixed_buffer);
3577 Function *LtGCFunc = emitListToGlobalCopyFunction(
3578 ReductionInfos, ReductionsBufferTy, FuncAttrs);
3579 Function *LtGRFunc = emitListToGlobalReduceFunction(
3580 ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
3581 Function *GtLCFunc = emitGlobalToListCopyFunction(
3582 ReductionInfos, ReductionsBufferTy, FuncAttrs);
3583 Function *GtLRFunc = emitGlobalToListReduceFunction(
3584 ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
3588 RedFixedBuferFn, {},
"_openmp_teams_reductions_buffer_$_$ptr");
3590 Value *Args3[] = {SrcLocInfo,
3591 KernelTeamsReductionPtr,
3603 RuntimeFunction::OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2);
3620 for (
auto En :
enumerate(ReductionInfos)) {
3627 Value *LHSPtr, *RHSPtr;
3629 &LHSPtr, &RHSPtr, CurFunc));
3634 return cast<Instruction>(U.getUser())->getParent()->getParent() ==
3638 return cast<Instruction>(U.getUser())->getParent()->getParent() ==
3642 assert(
false &&
"Unhandled ReductionGenCBKind");
3658 ".omp.reduction.func", &M);
3669 assert(RI.Variable &&
"expected non-null variable");
3670 assert(RI.PrivateVariable &&
"expected non-null private variable");
3671 assert(RI.ReductionGen &&
"expected non-null reduction generator callback");
3672 assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&
3673 "expected variables and their private equivalents to have the same "
3675 assert(RI.Variable->getType()->isPointerTy() &&
3676 "expected variables to be pointers");
3689 unsigned NumReductions = ReductionInfos.
size();
3696 for (
auto En :
enumerate(ReductionInfos)) {
3697 unsigned Index = En.index();
3700 RedArrayTy, RedArray, 0, Index,
"red.array.elem." +
Twine(Index));
3715 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
3720 unsigned RedArrayByteSize =
DL.getTypeStoreSize(RedArrayTy);
3723 Value *Lock = getOMPCriticalRegionLock(
".reduction");
3725 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
3726 : RuntimeFunction::OMPRTL___kmpc_reduce);
3729 {Ident, ThreadId, NumVariables, RedArraySize, RedArray,
3730 ReductionFunc, Lock},
3749 for (
auto En :
enumerate(ReductionInfos)) {
3755 if (!IsByRef[En.index()]) {
3757 "red.value." +
Twine(En.index()));
3759 Value *PrivateRedValue =
3761 "red.private.value." +
Twine(En.index()));
3772 if (!IsByRef[En.index()])
3776 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
3777 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
3785 if (CanGenerateAtomic &&
llvm::none_of(IsByRef, [](
bool P) {
return P; })) {
3809 for (
auto En :
enumerate(ReductionInfos)) {
3812 RedArrayTy, LHSArrayPtr, 0, En.index());
3817 RedArrayTy, RHSArrayPtr, 0, En.index());
3831 if (!IsByRef[En.index()])
3847 Directive OMPD = Directive::OMPD_master;
3852 Value *Args[] = {Ident, ThreadId};
3860 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3871 Directive OMPD = Directive::OMPD_masked;
3877 Value *ArgsEnd[] = {Ident, ThreadId};
3885 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3920 IndVarPHI->
addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
3933 "omp_" +
Name +
".next",
true);
3944 CL->Header = Header;
3963 NextBB, NextBB,
Name);
3988 Value *Start,
Value *Stop,
Value *Step,
bool IsSigned,
bool InclusiveStop,
3998 auto *IndVarTy = cast<IntegerType>(Start->getType());
3999 assert(IndVarTy == Stop->
getType() &&
"Stop type mismatch");
4000 assert(IndVarTy == Step->
getType() &&
"Step type mismatch");
4006 ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
4034 Value *CountIfLooping;
4035 if (InclusiveStop) {
4045 "omp_" +
Name +
".tripcount");
4066 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
4069 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
4075 InsertPointTy AllocaIP,
4076 bool NeedsBarrier) {
4077 assert(CLI->
isValid() &&
"Requires a valid canonical loop");
4079 "Require dedicated allocate IP");
4091 Type *IVTy =
IV->getType();
4111 Constant *One = ConstantInt::get(IVTy, 1);
4119 Constant *SchedulingType = ConstantInt::get(
4120 I32Type,
static_cast<int>(OMPScheduleType::UnorderedStatic));
4125 {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
4126 PUpperBound, PStride, One,
Zero});
4131 CLI->setTripCount(TripCount);
4153 omp::Directive::OMPD_for,
false,
4156 return BarrierIP.takeError();
4166OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
DebugLoc DL,
4168 InsertPointTy AllocaIP,
4171 assert(CLI->
isValid() &&
"Requires a valid canonical loop");
4172 assert(ChunkSize &&
"Chunk size is required");
4177 Type *IVTy =
IV->getType();
4179 "Max supported tripcount bitwidth is 64 bits");
4181 :
Type::getInt64Ty(Ctx);
4184 Constant *One = ConstantInt::get(InternalIVTy, 1);
4196 Value *PLowerBound =
4198 Value *PUpperBound =
4207 Value *CastedChunkSize =
4209 Value *CastedTripCount =
4212 Constant *SchedulingType = ConstantInt::get(
4213 I32Type,
static_cast<int>(OMPScheduleType::UnorderedStaticChunked));
4227 SchedulingType, PLastIter,
4228 PLowerBound, PUpperBound,
4233 Value *FirstChunkStart =
4235 Value *FirstChunkStop =
4240 Value *NextChunkStride =
4245 Value *DispatchCounter;
4249 DispatchCounter = Counter;
4252 FirstChunkStart, CastedTripCount, NextChunkStride,
4284 Value *IsLastChunk =
4286 Value *CountUntilOrigTripCount =
4289 IsLastChunk, CountUntilOrigTripCount, ChunkRange,
"omp_chunk.tripcount");
4290 Value *BackcastedChunkTC =
4292 CLI->setTripCount(BackcastedChunkTC);
4297 Value *BackcastedDispatchCounter =
4314 return AfterIP.takeError();
4335 case WorksharingLoopType::ForStaticLoop:
4338 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
4341 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
4343 case WorksharingLoopType::DistributeStaticLoop:
4346 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
4349 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
4351 case WorksharingLoopType::DistributeForStaticLoop:
4354 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
4357 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
4360 if (Bitwidth != 32 && Bitwidth != 64) {
4382 if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
4383 RealArgs.
push_back(ConstantInt::get(TripCountTy, 0));
4388 M, omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
4389 Builder.
restoreIP({InsertBlock, std::prev(InsertBlock->
end())});
4394 RealArgs.
push_back(ConstantInt::get(TripCountTy, 0));
4395 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
4396 RealArgs.
push_back(ConstantInt::get(TripCountTy, 0));
4433 CleanUpInfo.
collectBlocks(RegionBlockSet, BlocksToBeRemoved);
4441 "Expected unique undroppable user of outlined function");
4442 CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(OutlinedFnUser);
4443 assert(OutlinedFnCallInstruction &&
"Expected outlined function call");
4445 "Expected outlined function call to be located in loop preheader");
4447 if (OutlinedFnCallInstruction->
arg_size() > 1)
4454 LoopBodyArg, ParallelTaskPtr, TripCount,
4457 for (
auto &ToBeDeletedItem : ToBeDeleted)
4458 ToBeDeletedItem->eraseFromParent();
4464 InsertPointTy AllocaIP,
4477 OI.OuterAllocaBB = AllocaIP.getBlock();
4482 "omp.prelatch",
true);
4502 OI.collectBlocks(ParallelRegionBlockSet,
Blocks);
4504 ParallelRegionBlockSet.
end());
4524 Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
4533 if (ParallelRegionBlockSet.
count(Inst->getParent())) {
4534 Inst->replaceUsesOfWith(CLI->
getIndVar(), NewLoopCntLoad);
4540 OI.ExcludeArgsFromAggregate.push_back(NewLoopCntLoad);
4547 OI.PostOutlineCB = [=, ToBeDeletedVec =
4548 std::move(ToBeDeleted)](
Function &OutlinedFn) {
4550 ToBeDeletedVec, LoopType);
4558 bool NeedsBarrier, omp::ScheduleKind SchedKind,
Value *ChunkSize,
4559 bool HasSimdModifier,
bool HasMonotonicModifier,
4560 bool HasNonmonotonicModifier,
bool HasOrderedClause,
4563 return applyWorkshareLoopTarget(
DL, CLI, AllocaIP, LoopType);
4565 SchedKind, ChunkSize, HasSimdModifier, HasMonotonicModifier,
4566 HasNonmonotonicModifier, HasOrderedClause);
4568 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
4569 OMPScheduleType::ModifierOrdered;
4570 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
4571 case OMPScheduleType::BaseStatic:
4572 assert(!ChunkSize &&
"No chunk size with static-chunked schedule");
4574 return applyDynamicWorkshareLoop(
DL, CLI, AllocaIP, EffectiveScheduleType,
4575 NeedsBarrier, ChunkSize);
4577 return applyStaticWorkshareLoop(
DL, CLI, AllocaIP, NeedsBarrier);
4579 case OMPScheduleType::BaseStaticChunked:
4581 return applyDynamicWorkshareLoop(
DL, CLI, AllocaIP, EffectiveScheduleType,
4582 NeedsBarrier, ChunkSize);
4584 return applyStaticChunkedWorkshareLoop(
DL, CLI, AllocaIP, NeedsBarrier,
4587 case OMPScheduleType::BaseRuntime:
4588 case OMPScheduleType::BaseAuto:
4589 case OMPScheduleType::BaseGreedy:
4590 case OMPScheduleType::BaseBalanced:
4591 case OMPScheduleType::BaseSteal:
4592 case OMPScheduleType::BaseGuidedSimd:
4593 case OMPScheduleType::BaseRuntimeSimd:
4595 "schedule type does not support user-defined chunk sizes");
4597 case OMPScheduleType::BaseDynamicChunked:
4598 case OMPScheduleType::BaseGuidedChunked:
4599 case OMPScheduleType::BaseGuidedIterativeChunked:
4600 case OMPScheduleType::BaseGuidedAnalyticalChunked:
4601 case OMPScheduleType::BaseStaticBalancedChunked:
4602 return applyDynamicWorkshareLoop(
DL, CLI, AllocaIP, EffectiveScheduleType,
4603 NeedsBarrier, ChunkSize);
4619 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
4622 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
4635 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
4638 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
4650 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
4653 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
4659 InsertPointTy AllocaIP,
4661 bool NeedsBarrier,
Value *Chunk) {
4662 assert(CLI->
isValid() &&
"Requires a valid canonical loop");
4664 "Require dedicated allocate IP");
4666 "Require valid schedule type");
4668 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
4669 OMPScheduleType::ModifierOrdered;
4680 Type *IVTy =
IV->getType();
4698 Constant *One = ConstantInt::get(IVTy, 1);
4719 ConstantInt::get(I32Type,
static_cast<int>(SchedType));
4723 {SrcLoc, ThreadNum, SchedulingType, One,
4724 UpperBound, One, Chunk});
4734 PLowerBound, PUpperBound, PStride});
4735 Constant *Zero32 = ConstantInt::get(I32Type, 0);
4744 auto *PI = cast<PHINode>(Phi);
4745 PI->setIncomingBlock(0, OuterCond);
4746 PI->setIncomingValue(0, LowerBound);
4750 auto *Br = cast<BranchInst>(Term);
4751 Br->setSuccessor(0, OuterCond);
4759 auto *CI = cast<CmpInst>(Comp);
4760 CI->setOperand(1, UpperBound);
4763 auto *BI = cast<BranchInst>(Branch);
4764 assert(BI->getSuccessor(1) == Exit);
4765 BI->setSuccessor(1, OuterCond);
4779 omp::Directive::OMPD_for,
false,
4782 return BarrierIP.takeError();
4801 auto HasRemainingUses = [&BBsToErase](
BasicBlock *BB) {
4802 for (
Use &U : BB->uses()) {
4803 auto *UseInst = dyn_cast<Instruction>(U.getUser());
4806 if (BBsToErase.count(UseInst->getParent()))
4813 while (BBsToErase.remove_if(HasRemainingUses)) {
4824 assert(
Loops.size() >= 1 &&
"At least one loop required");
4825 size_t NumLoops =
Loops.size();
4829 return Loops.front();
4841 Loop->collectControlBlocks(OldControlBBs);
4845 if (ComputeIP.
isSet())
4852 Value *CollapsedTripCount =
nullptr;
4855 "All loops to collapse must be valid canonical loops");
4856 Value *OrigTripCount = L->getTripCount();
4857 if (!CollapsedTripCount) {
4858 CollapsedTripCount = OrigTripCount;
4870 OrigPreheader->
getNextNode(), OrigAfter,
"collapsed");
4878 Value *Leftover = Result->getIndVar();
4880 NewIndVars.
resize(NumLoops);
4881 for (
int i = NumLoops - 1; i >= 1; --i) {
4882 Value *OrigTripCount =
Loops[i]->getTripCount();
4885 NewIndVars[i] = NewIndVar;
4890 NewIndVars[0] = Leftover;
4899 BasicBlock *ContinueBlock = Result->getBody();
4901 auto ContinueWith = [&ContinueBlock, &ContinuePred,
DL](
BasicBlock *Dest,
4908 ContinueBlock =
nullptr;
4909 ContinuePred = NextSrc;
4916 for (
size_t i = 0; i < NumLoops - 1; ++i)
4917 ContinueWith(
Loops[i]->getBody(),
Loops[i + 1]->getHeader());
4923 for (
size_t i = NumLoops - 1; i > 0; --i)
4924 ContinueWith(
Loops[i]->getAfter(),
Loops[i - 1]->getLatch());
4927 ContinueWith(Result->getLatch(),
nullptr);
4934 for (
size_t i = 0; i < NumLoops; ++i)
4935 Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
4949std::vector<CanonicalLoopInfo *>
4953 "Must pass as many tile sizes as there are loops");
4954 int NumLoops =
Loops.size();
4955 assert(NumLoops >= 1 &&
"At least one loop to tile required");
4967 Loop->collectControlBlocks(OldControlBBs);
4975 assert(L->isValid() &&
"All input loops must be valid canonical loops");
4976 OrigTripCounts.
push_back(L->getTripCount());
4987 for (
int i = 0; i < NumLoops - 1; ++i) {
5000 for (
int i = 0; i < NumLoops; ++i) {
5002 Value *OrigTripCount = OrigTripCounts[i];
5015 Value *FloorTripOverflow =
5021 "omp_floor" +
Twine(i) +
".tripcount",
true);
5029 std::vector<CanonicalLoopInfo *> Result;
5030 Result.reserve(NumLoops * 2);
5043 auto EmbeddNewLoop =
5044 [
this,
DL,
F, InnerEnter, &Enter, &
Continue, &OutroInsertBefore](
5047 DL, TripCount,
F, InnerEnter, OutroInsertBefore,
Name);
5052 Enter = EmbeddedLoop->
getBody();
5054 OutroInsertBefore = EmbeddedLoop->
getLatch();
5055 return EmbeddedLoop;
5059 const Twine &NameBase) {
5062 EmbeddNewLoop(
P.value(), NameBase +
Twine(
P.index()));
5063 Result.push_back(EmbeddedLoop);
5067 EmbeddNewLoops(FloorCount,
"floor");
5073 for (
int i = 0; i < NumLoops; ++i) {
5077 Value *FloorIsEpilogue =
5079 Value *TileTripCount =
5086 EmbeddNewLoops(TileCounts,
"tile");
5091 for (std::pair<BasicBlock *, BasicBlock *>
P : InbetweenCode) {
5100 BodyEnter =
nullptr;
5101 BodyEntered = ExitBB;
5114 for (
int i = 0; i < NumLoops; ++i) {
5117 Value *OrigIndVar = OrigIndVars[i];
5145 if (Properties.
empty())
5168 assert(
Loop->isValid() &&
"Expecting a valid CanonicalLoopInfo");
5172 assert(Latch &&
"A valid CanonicalLoopInfo must have a unique latch");
5180 if (
I.mayReadOrWriteMemory()) {
5184 I.setMetadata(LLVMContext::MD_access_group, AccessGroup);
5206 const Twine &NamePrefix) {
5212 SplitBefore = dyn_cast<Instruction>(IfCond);
5258 VMap[
Block] = NewBB;
5268 if (TargetTriple.
isX86()) {
5269 if (Features.
lookup(
"avx512f"))
5271 else if (Features.
lookup(
"avx"))
5275 if (TargetTriple.
isPPC())
5277 if (TargetTriple.
isWasm())
5284 Value *IfCond, OrderKind Order,
5303 if (AlignedVars.
size()) {
5305 for (
auto &AlignedItem : AlignedVars) {
5306 Value *AlignedPtr = AlignedItem.first;
5307 Value *Alignment = AlignedItem.second;
5308 Instruction *loadInst = dyn_cast<Instruction>(AlignedPtr);
5311 AlignedPtr, Alignment);
5318 createIfVersion(CanonicalLoop, IfCond, VMap,
"simd");
5322 "Cannot find value which corresponds to original loop latch");
5323 assert(isa<BasicBlock>(MappedLatch) &&
5324 "Cannot cast mapped latch block value to BasicBlock");
5325 BasicBlock *NewLatchBlock = dyn_cast<BasicBlock>(MappedLatch);
5354 if ((Safelen ==
nullptr) || (Order == OrderKind::OMP_ORDER_concurrent)) {
5362 Ctx, {
MDString::get(Ctx,
"llvm.loop.parallel_accesses"), AccessGroup}));
5370 Ctx, {
MDString::get(Ctx,
"llvm.loop.vectorize.enable"), BoolConst}));
5372 if (Simdlen || Safelen) {
5376 ConstantInt *VectorizeWidth = Simdlen ==
nullptr ? Safelen : Simdlen;
5402static std::unique_ptr<TargetMachine>
5406 StringRef CPU =
F->getFnAttribute(
"target-cpu").getValueAsString();
5407 StringRef Features =
F->getFnAttribute(
"target-features").getValueAsString();
5408 const std::string &
Triple = M->getTargetTriple();
5418 std::nullopt, OptLevel));
5442 [&](
const Function &
F) {
return TM->getTargetTransformInfo(
F); });
5457 assert(L &&
"Expecting CanonicalLoopInfo to be recognized as a loop");
5462 nullptr, ORE,
static_cast<int>(OptLevel),
5483 <<
" Threshold=" << UP.
Threshold <<
"\n"
5486 <<
" PartialOptSizeThreshold="
5505 if (
auto *Load = dyn_cast<LoadInst>(&
I)) {
5506 Ptr = Load->getPointerOperand();
5507 }
else if (
auto *Store = dyn_cast<StoreInst>(&
I)) {
5508 Ptr = Store->getPointerOperand();
5512 Ptr =
Ptr->stripPointerCasts();
5514 if (
auto *Alloca = dyn_cast<AllocaInst>(
Ptr)) {
5515 if (Alloca->getParent() == &
F->getEntryBlock())
5535 int MaxTripCount = 0;
5536 bool MaxOrZero =
false;
5537 unsigned TripMultiple = 0;
5539 bool UseUpperBound =
false;
5541 MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP,
5543 unsigned Factor = UP.
Count;
5544 LLVM_DEBUG(
dbgs() <<
"Suggesting unroll factor of " << Factor <<
"\n");
5555 assert(Factor >= 0 &&
"Unroll factor must not be negative");
5571 Ctx, {
MDString::get(Ctx,
"llvm.loop.unroll.count"), FactorConst}));
5584 *UnrolledCLI =
Loop;
5589 "unrolling only makes sense with a factor of 2 or larger");
5591 Type *IndVarTy =
Loop->getIndVarType();
5598 std::vector<CanonicalLoopInfo *>
LoopNest =
5613 Ctx, {
MDString::get(Ctx,
"llvm.loop.unroll.count"), FactorConst})});
5616 (*UnrolledCLI)->assertOK();
5634 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
5653 if (!CPVars.
empty()) {
5658 Directive OMPD = Directive::OMPD_single;
5663 Value *Args[] = {Ident, ThreadId};
5672 if (
Error Err = FiniCB(IP))
5693 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper,
5700 for (
size_t I = 0, E = CPVars.
size();
I < E; ++
I)
5703 ConstantInt::get(
Int64, 0), CPVars[
I],
5706 }
else if (!IsNowait) {
5709 omp::Directive::OMPD_unknown,
false,
5724 Directive OMPD = Directive::OMPD_critical;
5729 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
5730 Value *Args[] = {Ident, ThreadId, LockVar};
5747 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5755 const Twine &
Name,
bool IsDependSource) {
5758 [](
Value *SV) {
return SV->
getType()->isIntegerTy(64); }) &&
5759 "OpenMP runtime requires depend vec with i64 type");
5772 for (
unsigned I = 0;
I < NumLoops; ++
I) {
5786 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
5804 Directive OMPD = Directive::OMPD_ordered;
5813 Value *Args[] = {Ident, ThreadId};
5823 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5829 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB,
bool Conditional,
5830 bool HasFinalize,
bool IsCancellable) {
5839 if (!isa_and_nonnull<BranchInst>(SplitPos))
5846 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
5857 "Unexpected control flow graph state!!");
5859 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
5861 return AfterIP.takeError();
5863 "Unexpected Control Flow State!");
5869 "Unexpected Insertion point location!");
5872 auto InsertBB = merged ? ExitPredBB : ExitBB;
5873 if (!isa_and_nonnull<BranchInst>(SplitPos))
5883 if (!Conditional || !EntryCall)
5903 UI->eraseFromParent();
5911 omp::Directive OMPD, InsertPointTy FinIP,
Instruction *ExitCall,
5919 "Unexpected finalization stack state!");
5922 assert(Fi.DK == OMPD &&
"Unexpected Directive for Finalization call!");
5924 if (
Error Err = Fi.FiniCB(FinIP))
5973 if (isa_and_nonnull<BranchInst>(OMP_Entry->
getTerminator())) {
5975 "copyin.not.master.end");
6030 Value *DependenceAddress,
bool HaveNowaitClause) {
6038 if (Device ==
nullptr)
6040 Constant *InteropTypeVal = ConstantInt::get(
Int32, (
int)InteropType);
6041 if (NumDependences ==
nullptr) {
6042 NumDependences = ConstantInt::get(
Int32, 0);
6046 Value *HaveNowaitClauseVal = ConstantInt::get(
Int32, HaveNowaitClause);
6048 Ident, ThreadId, InteropVar, InteropTypeVal,
6049 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
6058 Value *NumDependences,
Value *DependenceAddress,
bool HaveNowaitClause) {
6066 if (Device ==
nullptr)
6068 if (NumDependences ==
nullptr) {
6069 NumDependences = ConstantInt::get(
Int32, 0);
6073 Value *HaveNowaitClauseVal = ConstantInt::get(
Int32, HaveNowaitClause);
6075 Ident, ThreadId, InteropVar, Device,
6076 NumDependences, DependenceAddress, HaveNowaitClauseVal};
6085 Value *NumDependences,
6086 Value *DependenceAddress,
6087 bool HaveNowaitClause) {
6094 if (Device ==
nullptr)
6096 if (NumDependences ==
nullptr) {
6097 NumDependences = ConstantInt::get(
Int32, 0);
6101 Value *HaveNowaitClauseVal = ConstantInt::get(
Int32, HaveNowaitClause);
6103 Ident, ThreadId, InteropVar, Device,
6104 NumDependences, DependenceAddress, HaveNowaitClauseVal};
6133 int32_t MinThreadsVal, int32_t MaxThreadsVal,
6134 int32_t MinTeamsVal, int32_t MaxTeamsVal) {
6152 const std::string DebugPrefix =
"_debug__";
6153 if (KernelName.
ends_with(DebugPrefix)) {
6154 KernelName = KernelName.
drop_back(DebugPrefix.length());
6161 if (MinTeamsVal > 1 || MaxTeamsVal > 0)
6165 if (MaxThreadsVal < 0)
6166 MaxThreadsVal = std::max(
6169 if (MaxThreadsVal > 0)
6180 omp::RuntimeFunction::OMPRTL___kmpc_target_init);
6183 Twine DynamicEnvironmentName = KernelName +
"_dynamic_environment";
6184 Constant *DynamicEnvironmentInitializer =
6188 DynamicEnvironmentInitializer, DynamicEnvironmentName,
6190 DL.getDefaultGlobalsAddressSpace());
6194 DynamicEnvironmentGV->
getType() == DynamicEnvironmentPtr
6195 ? DynamicEnvironmentGV
6197 DynamicEnvironmentPtr);
6200 ConfigurationEnvironment, {
6201 UseGenericStateMachineVal,
6202 MayUseNestedParallelismVal,
6209 ReductionBufferLength,
6212 KernelEnvironment, {
6213 ConfigurationEnvironmentInitializer,
6217 std::string KernelEnvironmentName =
6218 (KernelName +
"_kernel_environment").str();
6221 KernelEnvironmentInitializer, KernelEnvironmentName,
6223 DL.getDefaultGlobalsAddressSpace());
6227 KernelEnvironmentGV->
getType() == KernelEnvironmentPtr
6228 ? KernelEnvironmentGV
6230 KernelEnvironmentPtr);
6231 Value *KernelLaunchEnvironment = DebugKernelWrapper->
getArg(0);
6259 UI->eraseFromParent();
6267 int32_t TeamsReductionDataSize,
6268 int32_t TeamsReductionBufferLength) {
6273 omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
6277 if (!TeamsReductionBufferLength || !TeamsReductionDataSize)
6283 const std::string DebugPrefix =
"_debug__";
6285 KernelName = KernelName.
drop_back(DebugPrefix.length());
6286 auto *KernelEnvironmentGV =
6288 assert(KernelEnvironmentGV &&
"Expected kernel environment global\n");
6289 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
6291 KernelEnvironmentInitializer,
6292 ConstantInt::get(
Int32, TeamsReductionDataSize), {0, 7});
6294 NewInitializer, ConstantInt::get(
Int32, TeamsReductionBufferLength),
6296 KernelEnvironmentGV->setInitializer(NewInitializer);
6301 NamedMDNode *MD = M.getOrInsertNamedMetadata(
"nvvm.annotations");
6305 auto *KernelOp = dyn_cast<ConstantAsMetadata>(
Op->getOperand(0));
6306 if (!KernelOp || KernelOp->getValue() != &
Kernel)
6308 auto *Prop = dyn_cast<MDString>(
Op->getOperand(1));
6309 if (!Prop || Prop->getString() !=
Name)
6321 auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->
getOperand(2));
6322 int32_t OldLimit = cast<ConstantInt>(OldVal->getValue())->getZExtValue();
6325 OldVal->getValue()->getType(),
6326 Min ? std::min(OldLimit,
Value) : std::max(OldLimit,
Value))));
6335 NamedMDNode *MD = M.getOrInsertNamedMetadata(
"nvvm.annotations");
6340std::pair<int32_t, int32_t>
6342 int32_t ThreadLimit =
6347 if (!Attr.isValid() || !Attr.isStringAttribute())
6348 return {0, ThreadLimit};
6351 if (!llvm::to_integer(UBStr, UB, 10))
6352 return {0, ThreadLimit};
6353 UB = ThreadLimit ? std::min(ThreadLimit, UB) : UB;
6354 if (!llvm::to_integer(LBStr, LB, 10))
6360 auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->getOperand(2));
6361 int32_t UB = cast<ConstantInt>(OldVal->getValue())->getZExtValue();
6362 return {0, ThreadLimit ? std::min(ThreadLimit, UB) : UB};
6364 return {0, ThreadLimit};
6374 llvm::utostr(LB) +
"," + llvm::utostr(UB));
6381std::pair<int32_t, int32_t>
6388 int32_t LB, int32_t UB) {
6393 Kernel.
addFnAttr(
"amdgpu-max-num-workgroups", llvm::utostr(LB) +
",1,1");
6398void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
6413 assert(OutlinedFn &&
"The outlined function must exist if embedded");
6422Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(
Function *OutlinedFn,
6428 "Named kernel already exists?");
6446 OutlinedFn = *CBResult;
6448 OutlinedFn =
nullptr;
6454 if (!IsOffloadEntry)
6457 std::string EntryFnIDName =
6459 ? std::string(EntryFnName)
6463 EntryFnName, EntryFnIDName);
6471 setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
6472 auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
6473 auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
6475 EntryInfo, EntryAddr, OutlinedFnID,
6477 return OutlinedFnID;
6506 bool IsStandAlone = !BodyGenCB;
6532 SrcLocInfo, DeviceID,
6539 assert(MapperFunc &&
"MapperFunc missing for standalone target data");
6543 if (
Info.HasNoWait) {
6553 if (
Info.HasNoWait) {
6557 emitBlock(OffloadContBlock, CurFn,
true);
6563 bool RequiresOuterTargetTask =
Info.HasNoWait;
6564 if (!RequiresOuterTargetTask) {
6565 Error Err = TaskBodyCB(
nullptr,
nullptr,
6567 assert(!Err &&
"TaskBodyCB expected to succeed");
6571 {},
Info.HasNoWait);
6572 assert(AfterIP &&
"TaskBodyCB expected to succeed");
6576 omp::OMPRTL___tgt_target_data_begin_mapper);
6580 for (
auto DeviceMap :
Info.DevicePtrInfoMap) {
6581 if (isa<AllocaInst>(DeviceMap.second.second)) {
6629 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
6651 return emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);
6667 return emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
6671 return emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);
6683 bool IsGPUDistribute) {
6684 assert((IVSize == 32 || IVSize == 64) &&
6685 "IV size is not compatible with the omp runtime");
6687 if (IsGPUDistribute)
6689 ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
6690 : omp::OMPRTL___kmpc_distribute_static_init_4u)
6691 : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
6692 : omp::OMPRTL___kmpc_distribute_static_init_8u);
6694 Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
6695 : omp::OMPRTL___kmpc_for_static_init_4u)
6696 : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
6697 : omp::OMPRTL___kmpc_for_static_init_8u);
6704 assert((IVSize == 32 || IVSize == 64) &&
6705 "IV size is not compatible with the omp runtime");
6707 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
6708 : omp::OMPRTL___kmpc_dispatch_init_4u)
6709 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
6710 : omp::OMPRTL___kmpc_dispatch_init_8u);
6717 assert((IVSize == 32 || IVSize == 64) &&
6718 "IV size is not compatible with the omp runtime");
6720 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
6721 : omp::OMPRTL___kmpc_dispatch_next_4u)
6722 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
6723 : omp::OMPRTL___kmpc_dispatch_next_8u);
6730 assert((IVSize == 32 || IVSize == 64) &&
6731 "IV size is not compatible with the omp runtime");
6733 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
6734 : omp::OMPRTL___kmpc_dispatch_fini_4u)
6735 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
6736 : omp::OMPRTL___kmpc_dispatch_fini_8u);
6759 for (
auto &Arg : Inputs)
6760 ParameterTypes.
push_back(Arg->getType()->isPointerTy()
6764 for (
auto &Arg : Inputs)
6765 ParameterTypes.
push_back(Arg->getType());
6769 auto M = BB->getModule();
6788 DB.createSubroutineType(DB.getOrCreateTypeArray({}));
6790 DISubprogram::SPFlagOptimized |
6791 DISubprogram::SPFlagLocalToUnit;
6794 CU, FuncName, FuncName, SP->getFile(),
DL.getLine(), Ty,
6795 DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
6798 Func->setSubprogram(OutlinedSP);
6803 OutlinedSP,
DL.getInlinedAt()));
6826 splitBB(Builder,
true,
"outlined.body");
6841 auto AllocaIP = Builder.
saveIP();
6846 const auto &ArgRange =
6848 ?
make_range(Func->arg_begin() + 1, Func->arg_end())
6869 if (
auto *Const = dyn_cast<Constant>(Input))
6874 if (
auto *Instr = dyn_cast<Instruction>(
User))
6875 if (Instr->getFunction() == Func)
6876 Instr->replaceUsesOfWith(Input, InputCopy);
6882 for (
auto InArg :
zip(Inputs, ArgRange)) {
6883 Value *Input = std::get<0>(InArg);
6884 Argument &Arg = std::get<1>(InArg);
6885 Value *InputCopy =
nullptr;
6888 ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.
saveIP());
6909 if (llvm::isa<llvm::GlobalValue>(std::get<0>(InArg)) ||
6910 llvm::isa<llvm::GlobalObject>(std::get<0>(InArg)) ||
6911 llvm::isa<llvm::GlobalVariable>(std::get<0>(InArg))) {
6912 DeferredReplacement.
push_back(std::make_pair(Input, InputCopy));
6916 ReplaceValue(Input, InputCopy, Func);
6920 for (
auto Deferred : DeferredReplacement)
6921 ReplaceValue(std::get<0>(Deferred), std::get<1>(Deferred), Func);
6961 Type *TaskPtrTy = OMPBuilder.TaskPtr;
6962 Type *TaskTy = OMPBuilder.Task;
6967 ".omp_target_task_proxy_func",
6969 ProxyFn->getArg(0)->setName(
"thread.id");
6970 ProxyFn->getArg(1)->setName(
"task");
6976 bool HasShareds = StaleCI->
arg_size() > 1;
6982 "StaleCI with shareds should have exactly two arguments.");
6984 auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->
getArgOperand(1));
6985 assert(ArgStructAlloca &&
6986 "Unable to find the alloca instruction corresponding to arguments "
6987 "for extracted function");
6988 auto *ArgStructType = cast<StructType>(ArgStructAlloca->getAllocatedType());
6991 Builder.
CreateAlloca(ArgStructType,
nullptr,
"structArg");
6992 Value *TaskT = ProxyFn->getArg(1);
6993 Value *ThreadId = ProxyFn->getArg(0);
6994 Value *SharedsSize =
6995 Builder.
getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
7002 NewArgStructAlloca, NewArgStructAlloca->
getAlign(), LoadShared,
7005 Builder.
CreateCall(KernelLaunchFunction, {ThreadId, NewArgStructAlloca});
7019 [&OMPBuilder, &Builder, &Inputs, &CBFunc,
7020 &ArgAccessorFuncCB](
StringRef EntryFnName) {
7022 CBFunc, ArgAccessorFuncCB);
7026 EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
7134 TargetTaskAllocaBB->
begin());
7138 OI.
EntryBB = TargetTaskAllocaBB;
7144 Builder, AllocaIP, ToBeDeleted, TargetTaskAllocaIP,
"global.tid",
false));
7148 if (
Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))
7152 OI.
PostOutlineCB = [
this, ToBeDeleted, Dependencies, HasNoWait,
7153 DeviceID](
Function &OutlinedFn)
mutable {
7155 "there must be a single user for the outlined function");
7158 bool HasShareds = StaleCI->
arg_size() > 1;
7162 LLVM_DEBUG(
dbgs() <<
"Proxy task entry function created: " << *ProxyFn
7181 OMPRTL___kmpc_omp_target_task_alloc);
7199 auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->
getArgOperand(1));
7200 assert(ArgStructAlloca &&
7201 "Unable to find the alloca instruction corresponding to arguments "
7202 "for extracted function");
7203 auto *ArgStructType =
7204 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
7205 assert(ArgStructType &&
"Unable to find struct type corresponding to "
7206 "arguments for extracted function");
7227 TaskSize, SharedsSize,
7275 }
else if (DepArray) {
7294 I->eraseFromParent();
7312 DeviceAddrCB, CustomMapperCB);
7323 bool HasNoWait =
false) {
7334 bool HasDependencies = Dependencies.size() > 0;
7335 bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
7352 EmitTargetCallFallbackCB, KArgs,
7353 DeviceID, RTLoc, TargetTaskAllocaIP);
7356 return EmitTargetCallFallbackCB(OMPBuilder.
Builder.
saveIP());
7368 if (!OutlinedFnID) {
7370 if (RequiresOuterTargetTask) {
7376 Dependencies, HasNoWait);
7378 return EmitTargetCallFallbackCB(Builder.
saveIP());
7401 for (
auto V : NumTeams)
7403 for (
auto V : NumThreads)
7406 unsigned NumTargetItems =
Info.NumberOfPtrs;
7419 NumTargetItems, RTArgs, NumIterations, NumTeamsC, NumThreadsC,
7420 DynCGGroupMem, HasNoWait);
7425 if (RequiresOuterTargetTask)
7426 return OMPBuilder.
emitTargetTask(TaskBodyCB, DeviceID, RTLoc, AllocaIP,
7427 Dependencies, HasNoWait);
7430 EmitTargetCallFallbackCB, KArgs,
7431 DeviceID, RTLoc, AllocaIP);
7461 *
this,
Builder, IsOffloadEntry, EntryInfo, OutlinedFn, OutlinedFnID,
7462 Args, CBFunc, ArgAccessorFuncCB))
7470 NumThreads, Args, GenMapInfoCB, Dependencies, HasNowait);
7484 return OS.str().str();
7498 assert(Elem.second->getValueType() == Ty &&
7499 "OMP internal variable has different type than requested");
7515 GV->setAlignment(std::max(TypeAlign, PtrAlign));
7522Value *OpenMPIRBuilder::getOMPCriticalRegionLock(
StringRef CriticalName) {
7523 std::string Prefix =
Twine(
"gomp_critical_user_", CriticalName).
str();
7524 std::string
Name = getNameWithSeparators({Prefix,
"var"},
".",
".");
7535 return SizePtrToInt;
7540 std::string VarName) {
7548 return MaptypesArrayGlobal;
7553 unsigned NumOperands,
7562 ArrI8PtrTy,
nullptr,
".offload_baseptrs");
7566 ArrI64Ty,
nullptr,
".offload_sizes");
7577 int64_t DeviceID,
unsigned NumOperands) {
7583 Value *ArgsBaseGEP =
7585 {Builder.getInt32(0), Builder.getInt32(0)});
7588 {Builder.getInt32(0), Builder.getInt32(0)});
7589 Value *ArgSizesGEP =
7591 {Builder.getInt32(0), Builder.getInt32(0)});
7597 ArgSizesGEP, MaptypesArg, MapnamesArg, NullPtr});
7604 assert((!ForEndCall ||
Info.separateBeginEndCalls()) &&
7605 "expected region end call to runtime only when end call is separate");
7607 auto VoidPtrTy = UnqualPtrTy;
7608 auto VoidPtrPtrTy = UnqualPtrTy;
7610 auto Int64PtrTy = UnqualPtrTy;
7612 if (!
Info.NumberOfPtrs) {
7624 Info.RTArgs.BasePointersArray,
7635 ForEndCall &&
Info.RTArgs.MapTypesArrayEnd ?
Info.RTArgs.MapTypesArrayEnd
7636 :
Info.RTArgs.MapTypesArray,
7642 if (!
Info.EmitDebug)
7651 if (!
Info.HasMapper)
7676 "struct.descriptor_dim");
7678 enum { OffsetFD = 0, CountFD, StrideFD };
7682 for (
unsigned I = 0, L = 0, E = NonContigInfo.
Dims.
size();
I < E; ++
I) {
7685 if (NonContigInfo.
Dims[
I] == 1)
7692 for (
unsigned II = 0, EE = NonContigInfo.
Dims[
I];
II < EE; ++
II) {
7693 unsigned RevIdx = EE -
II - 1;
7696 {Builder.getInt64(0), Builder.getInt64(II)});
7700 NonContigInfo.
Offsets[L][RevIdx], OffsetLVal,
7705 NonContigInfo.
Counts[L][RevIdx], CountLVal,
7710 NonContigInfo.
Strides[L][RevIdx], StrideLVal,
7719 Info.RTArgs.PointersArray, 0,
I);
7726void OpenMPIRBuilder::emitUDMapperArrayInitOrDel(
7730 StringRef Prefix = IsInit ?
".init" :
".del";
7740 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7741 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
7751 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7752 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ)));
7777 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7778 OpenMPOffloadMappingFlags::OMP_MAP_TO |
7779 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
7783 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7784 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
7788 Value *OffloadingArgs[] = {MapperHandle,
Base, Begin,
7789 ArraySize, MapTypeArg, MapName};
7816 MapperFn->
addFnAttr(Attribute::NoInline);
7817 MapperFn->
addFnAttr(Attribute::NoUnwind);
7848 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, BaseIn, BeginIn,
Size,
7849 MapType, MapName, ElementSize, HeadBB,
7875 Value *OffloadingArgs[] = {MapperHandle};
7879 Value *ShiftedPreviousSize =
7883 for (
unsigned I = 0;
I <
Info.BasePointers.size(); ++
I) {
7886 Value *CurBeginArg =
7895 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7897 Value *MemberMapType =
7915 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7916 OpenMPOffloadMappingFlags::OMP_MAP_TO |
7917 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
7933 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7934 OpenMPOffloadMappingFlags::OMP_MAP_TO |
7935 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
7941 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7942 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
7949 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7950 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
7956 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7957 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
7964 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7965 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
7976 Value *OffloadingArgs[] = {MapperHandle, CurBaseArg, CurBeginArg,
7977 CurSizeArg, CurMapType, CurNameArg};
7979 if (CustomMapperCB && CustomMapperCB(
I, &ChildMapperFn)) {
7994 "omp.arraymap.next");
8003 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, BaseIn, BeginIn,
Size,
8004 MapType, MapName, ElementSize, DoneBB,
8022 Info.clearArrayInfo();
8025 if (
Info.NumberOfPtrs == 0)
8035 PointerArrayType,
nullptr,
".offload_baseptrs");
8038 PointerArrayType,
nullptr,
".offload_ptrs");
8040 PointerArrayType,
nullptr,
".offload_mappers");
8041 Info.RTArgs.MappersArray = MappersArray;
8048 ConstantInt::get(Int64Ty, 0));
8050 for (
unsigned I = 0, E = CombinedInfo.
Sizes.
size();
I < E; ++
I) {
8051 if (
auto *CI = dyn_cast<Constant>(CombinedInfo.
Sizes[
I])) {
8052 if (!isa<ConstantExpr>(CI) && !isa<GlobalValue>(CI)) {
8053 if (IsNonContiguous &&
8054 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
8056 OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG))
8064 RuntimeSizes.
set(
I);
8067 if (RuntimeSizes.
all()) {
8070 SizeArrayType,
nullptr,
".offload_sizes");
8076 auto *SizesArrayGbl =
8081 if (!RuntimeSizes.
any()) {
8082 Info.RTArgs.SizesArray = SizesArrayGbl;
8088 SizeArrayType,
nullptr,
".offload_sizes");
8093 SizesArrayGbl, OffloadSizeAlign,
8098 Info.RTArgs.SizesArray = Buffer;
8106 for (
auto mapFlag : CombinedInfo.
Types)
8108 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
8112 Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
8117 auto *MapNamesArrayGbl =
8119 Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
8120 Info.EmitDebug =
true;
8122 Info.RTArgs.MapNamesArray =
8124 Info.EmitDebug =
false;
8129 if (
Info.separateBeginEndCalls()) {
8130 bool EndMapTypesDiffer =
false;
8132 if (
Type &
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
8133 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
8134 Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
8135 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
8136 EndMapTypesDiffer =
true;
8139 if (EndMapTypesDiffer) {
8141 Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
8146 for (
unsigned I = 0;
I <
Info.NumberOfPtrs; ++
I) {
8154 if (
Info.requiresDevicePointerInfo()) {
8161 DeviceAddrCB(
I,
Info.DevicePtrInfoMap[BPVal].second);
8163 Info.DevicePtrInfoMap[BPVal] = {BP, BP};
8165 DeviceAddrCB(
I, BP);
8177 if (RuntimeSizes.
test(
I)) {
8191 if (
Value *CustomMFunc = CustomMapperCB(
I))
8195 {Builder.getIntN(IndexSize, 0), Builder.getIntN(IndexSize, I)});
8201 Info.NumberOfPtrs == 0)
8246 if (
auto *CI = dyn_cast<ConstantInt>(
Cond)) {
8247 auto CondConstant = CI->getSExtValue();
8279bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
8283 "Unexpected Atomic Ordering.");
8347 assert(
X.Var->getType()->isPointerTy() &&
8348 "OMP Atomic expects a pointer to target memory");
8349 Type *XElemTy =
X.ElemTy;
8352 "OMP atomic read expected a scalar type");
8354 Value *XRead =
nullptr;
8360 XRead = cast<Value>(XLD);
8373 XRead = AtomicLoadRes.first;
8388 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Read);
8400 assert(
X.Var->getType()->isPointerTy() &&
8401 "OMP Atomic expects a pointer to target memory");
8402 Type *XElemTy =
X.ElemTy;
8405 "OMP atomic write expected a scalar type");
8420 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Write);
8433 Type *XTy =
X.Var->getType();
8435 "OMP Atomic expects a pointer to target memory");
8436 Type *XElemTy =
X.ElemTy;
8439 "OMP atomic update expected a scalar type");
8442 "OpenMP atomic does not support LT or GT operations");
8446 emitAtomicUpdate(AllocaIP,
X.Var,
X.ElemTy, Expr, AO, RMWOp, UpdateOp,
8447 X.IsVolatile, IsXBinopExpr);
8449 return AtomicResult.takeError();
8450 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
8455Value *OpenMPIRBuilder::emitRMWOpAsInstruction(
Value *Src1,
Value *Src2,
8492 AtomicUpdateCallbackTy &UpdateOp,
bool VolatileX,
bool IsXBinopExpr) {
8495 bool emitRMWOp =
false;
8503 emitRMWOp = XElemTy;
8506 emitRMWOp = (IsXBinopExpr && XElemTy);
8513 std::pair<Value *, Value *> Res;
8520 Res.second = Res.first;
8522 Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
8535 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
8542 X->getName() +
".atomic.cont");
8546 NewAtomicAddr->
setName(
X->getName() +
"x.new.val");
8549 PHI->addIncoming(AtomicLoadRes.first, CurBB);
8554 Value *Upd = *CBResult;
8558 auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
8559 AtomicLoadRes.second, NewAtomicAddr, AO, Failure);
8564 Res.first = OldExprVal;
8591 X->getName() +
".atomic.cont");
8595 NewAtomicAddr->
setName(
X->getName() +
"x.new.val");
8598 PHI->addIncoming(OldVal, CurBB);
8604 X->getName() +
".atomic.fltCast");
8607 X->getName() +
".atomic.ptrCast");
8614 Value *Upd = *CBResult;
8621 Result->setVolatile(VolatileX);
8627 Res.first = OldExprVal;
8647 bool UpdateExpr,
bool IsPostfixUpdate,
bool IsXBinopExpr) {
8652 Type *XTy =
X.Var->getType();
8654 "OMP Atomic expects a pointer to target memory");
8655 Type *XElemTy =
X.ElemTy;
8658 "OMP atomic capture expected a scalar type");
8660 "OpenMP atomic does not support LT or GT operations");
8667 emitAtomicUpdate(AllocaIP,
X.Var,
X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
8668 X.IsVolatile, IsXBinopExpr);
8671 Value *CapturedVal =
8672 (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);
8675 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);
8687 IsPostfixUpdate, IsFailOnly, Failure);
8699 assert(
X.Var->getType()->isPointerTy() &&
8700 "OMP atomic expects a pointer to target memory");
8703 assert(V.Var->getType()->isPointerTy() &&
"v.var must be of pointer type");
8704 assert(V.ElemTy ==
X.ElemTy &&
"x and v must be of same type");
8709 if (
Op == OMPAtomicCompareOp::EQ) {
8728 "OldValue and V must be of same type");
8729 if (IsPostfixUpdate) {
8747 CurBBTI,
X.Var->getName() +
".atomic.exit");
8767 Value *CapturedValue =
8775 assert(R.Var->getType()->isPointerTy() &&
8776 "r.var must be of pointer type");
8777 assert(R.ElemTy->isIntegerTy() &&
"r must be of integral type");
8780 Value *ResultCast = R.IsSigned
8786 assert((
Op == OMPAtomicCompareOp::MAX ||
Op == OMPAtomicCompareOp::MIN) &&
8787 "Op should be either max or min at this point");
8788 assert(!IsFailOnly &&
"IsFailOnly is only valid when the comparison is ==");
8828 Value *CapturedValue =
nullptr;
8829 if (IsPostfixUpdate) {
8830 CapturedValue = OldValue;
8862 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Compare);
8909 bool SubClausesPresent =
8910 (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
8913 assert((NumTeamsLower ==
nullptr || NumTeamsUpper !=
nullptr) &&
8914 "if lowerbound is non-null, then upperbound must also be non-null "
8915 "for bounds on num_teams");
8917 if (NumTeamsUpper ==
nullptr)
8920 if (NumTeamsLower ==
nullptr)
8921 NumTeamsLower = NumTeamsUpper;
8925 "argument to if clause must be an integer value");
8930 ConstantInt::get(IfExpr->
getType(), 0));
8939 if (ThreadLimit ==
nullptr)
8945 {Ident, ThreadNum, NumTeamsLower, NumTeamsUpper, ThreadLimit});
8950 if (
Error Err = BodyGenCB(AllocaIP, CodeGenIP))
8962 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP,
"gid",
true));
8964 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP,
"tid",
true));
8966 auto HostPostOutlineCB = [
this, Ident,
8967 ToBeDeleted](
Function &OutlinedFn)
mutable {
8972 "there must be a single user for the outlined function");
8977 "Outlined function must have two or three arguments only");
8979 bool HasShared = OutlinedFn.
arg_size() == 3;
8987 assert(StaleCI &&
"Error while outlining - no CallInst user found for the "
8988 "outlined function.");
8995 omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
8999 I->eraseFromParent();
9014 std::string VarName) {
9023 return MapNamesArrayGlobal;
9028void OpenMPIRBuilder::initializeTypes(
Module &M) {
9031#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
9032#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
9033 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
9034 VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
9035#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
9036 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
9037 VarName##Ptr = PointerType::getUnqual(VarName);
9038#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
9039 T = StructType::getTypeByName(Ctx, StructName); \
9041 T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
9043 VarName##Ptr = PointerType::getUnqual(T);
9044#include "llvm/Frontend/OpenMP/OMPKinds.def"
9055 while (!Worklist.
empty()) {
9059 if (BlockSet.
insert(SuccBB).second)
9071 "omp_offloading_entries");
9095 Fn->
addFnAttr(
"uniform-work-group-size",
"true");
9114 auto &&GetMDInt = [
this](
unsigned V) {
9122 auto &&TargetRegionMetadataEmitter =
9123 [&
C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
9138 GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
9139 GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
9140 GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
9141 GetMDInt(E.getOrder())};
9144 OrderedEntries[E.getOrder()] = std::make_pair(&E, EntryInfo);
9153 auto &&DeviceGlobalVarMetadataEmitter =
9154 [&
C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
9164 Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
9165 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
9169 OrderedEntries[E.getOrder()] = std::make_pair(&E, varInfo);
9176 DeviceGlobalVarMetadataEmitter);
9178 for (
const auto &E : OrderedEntries) {
9179 assert(E.first &&
"All ordered entries must exist!");
9180 if (
const auto *CE =
9181 dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
9183 if (!CE->getID() || !CE->getAddress()) {
9195 }
else if (
const auto *CE =
dyn_cast<
9206 if (!CE->getAddress()) {
9211 if (CE->getVarSize() == 0)
9217 "Declaret target link address is set.");
9220 if (!CE->getAddress()) {
9232 if (
auto *
GV = dyn_cast<GlobalValue>(CE->getAddress()))
9233 if ((
GV->hasLocalLinkage() ||
GV->hasHiddenVisibility()) &&
9241 Flags, CE->getLinkage(), CE->getVarName());
9244 Flags, CE->getLinkage());
9265 unsigned FileID,
unsigned Line,
unsigned Count) {
9268 <<
llvm::format(
"_%x_", FileID) << ParentName <<
"_l" << Line;
9275 unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
9278 EntryInfo.
Line, NewCount);
9285 auto FileIDInfo = CallBack();
9288 "getTargetEntryUniqueInfo, error message: " +
9294 std::get<1>(FileIDInfo));
9300 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>
>(
9302 !(Remain & 1); Remain = Remain >> 1)
9320 if (
static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>
>(
9322 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>
>(
9329 Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
9330 Flags |= MemberOfFlag;
9336 bool IsDeclaration,
bool IsExternallyVisible,
9338 std::vector<GlobalVariable *> &GeneratedRefs,
bool OpenMPSIMD,
9339 std::vector<Triple> TargetTriple,
Type *LlvmPtrTy,
9340 std::function<
Constant *()> GlobalInitializer,
9356 if (!IsExternallyVisible)
9358 OS <<
"_decl_tgt_ref_ptr";
9367 auto *
GV = cast<GlobalVariable>(
Ptr);
9371 if (GlobalInitializer)
9372 GV->setInitializer(GlobalInitializer());
9378 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
9379 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
9380 GlobalInitializer, VariableLinkage, LlvmPtrTy, cast<Constant>(
Ptr));
9383 return cast<Constant>(
Ptr);
9392 bool IsDeclaration,
bool IsExternallyVisible,
9394 std::vector<GlobalVariable *> &GeneratedRefs,
bool OpenMPSIMD,
9395 std::vector<Triple> TargetTriple,
9396 std::function<
Constant *()> GlobalInitializer,
9413 VarName = MangledName;
9421 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->
getLinkage();
9437 auto *GvAddrRef = cast<GlobalVariable>(AddrRef);
9438 GvAddrRef->setConstant(
true);
9440 GvAddrRef->setInitializer(
Addr);
9441 GeneratedRefs.push_back(GvAddrRef);
9451 VarName = (
Addr) ?
Addr->getName() :
"";
9455 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
9456 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
9457 LlvmPtrTy, GlobalInitializer, VariableLinkage);
9458 VarName = (
Addr) ?
Addr->getName() :
"";
9479 auto &&GetMDInt = [MN](
unsigned Idx) {
9480 auto *V = cast<ConstantAsMetadata>(MN->getOperand(
Idx));
9481 return cast<ConstantInt>(V->getValue())->getZExtValue();
9484 auto &&GetMDString = [MN](
unsigned Idx) {
9485 auto *V = cast<MDString>(MN->getOperand(
Idx));
9486 return V->getString();
9489 switch (GetMDInt(0)) {
9517 if (HostFilePath.
empty())
9521 if (std::error_code Err = Buf.getError()) {
9523 "OpenMPIRBuilder: " +
9531 if (std::error_code Err =
M.getError()) {
9533 (
"error parsing host file inside of OpenMPIRBuilder: " + Err.message())
9545 return OffloadEntriesTargetRegion.empty() &&
9546 OffloadEntriesDeviceGlobalVar.empty();
9549unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
9551 auto It = OffloadEntriesTargetRegionCount.find(
9552 getTargetRegionEntryCountKey(EntryInfo));
9553 if (It == OffloadEntriesTargetRegionCount.end())
9558void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
9560 OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
9561 EntryInfo.
Count + 1;
9567 OffloadEntriesTargetRegion[EntryInfo] =
9569 OMPTargetRegionEntryTargetRegion);
9570 ++OffloadingEntriesNum;
9576 assert(EntryInfo.
Count == 0 &&
"expected default EntryInfo");
9579 EntryInfo.
Count = getTargetRegionEntryInfoCount(EntryInfo);
9585 if (!hasTargetRegionEntryInfo(EntryInfo)) {
9588 auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
9589 Entry.setAddress(
Addr);
9591 Entry.setFlags(
Flags);
9594 hasTargetRegionEntryInfo(EntryInfo,
true))
9596 assert(!hasTargetRegionEntryInfo(EntryInfo) &&
9597 "Target region entry already registered!");
9599 OffloadEntriesTargetRegion[EntryInfo] = Entry;
9600 ++OffloadingEntriesNum;
9602 incrementTargetRegionEntryInfoCount(EntryInfo);
9609 EntryInfo.
Count = getTargetRegionEntryInfoCount(EntryInfo);
9611 auto It = OffloadEntriesTargetRegion.find(EntryInfo);
9612 if (It == OffloadEntriesTargetRegion.end()) {
9616 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
9624 for (
const auto &It : OffloadEntriesTargetRegion) {
9625 Action(It.first, It.second);
9631 OffloadEntriesDeviceGlobalVar.try_emplace(
Name, Order,
Flags);
9632 ++OffloadingEntriesNum;
9640 if (!hasDeviceGlobalVarEntryInfo(VarName))
9642 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
9643 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
9644 if (Entry.getVarSize() == 0) {
9645 Entry.setVarSize(VarSize);
9646 Entry.setLinkage(Linkage);
9650 Entry.setVarSize(VarSize);
9651 Entry.setLinkage(Linkage);
9652 Entry.setAddress(
Addr);
9654 if (hasDeviceGlobalVarEntryInfo(VarName)) {
9655 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
9656 assert(Entry.isValid() && Entry.getFlags() ==
Flags &&
9657 "Entry not initialized!");
9658 if (Entry.getVarSize() == 0) {
9659 Entry.setVarSize(VarSize);
9660 Entry.setLinkage(Linkage);
9665 OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
9669 OffloadEntriesDeviceGlobalVar.try_emplace(
9670 VarName, OffloadingEntriesNum,
Addr, VarSize,
Flags, Linkage,
"");
9671 ++OffloadingEntriesNum;
9678 for (
const auto &E : OffloadEntriesDeviceGlobalVar)
9679 Action(E.getKey(), E.getValue());
9686void CanonicalLoopInfo::collectControlBlocks(
9693 BBs.
append({getPreheader(), Header,
Cond, Latch, Exit, getAfter()});
9705void CanonicalLoopInfo::setTripCount(
Value *TripCount) {
9709 assert(isa<CmpInst>(CmpI) &&
"First inst must compare IV with TripCount");
9717void CanonicalLoopInfo::mapIndVar(
9727 for (
Use &U : OldIV->
uses()) {
9728 auto *
User = dyn_cast<Instruction>(U.getUser());
9731 if (
User->getParent() == getCond())
9733 if (
User->getParent() == getLatch())
9739 Value *NewIV = Updater(OldIV);
9742 for (
Use *U : ReplacableUses)
9763 "Preheader must terminate with unconditional branch");
9765 "Preheader must jump to header");
9768 assert(isa<BranchInst>(Header->getTerminator()) &&
9769 "Header must terminate with unconditional branch");
9770 assert(Header->getSingleSuccessor() ==
Cond &&
9771 "Header must jump to exiting block");
9774 assert(
Cond->getSinglePredecessor() == Header &&
9775 "Exiting block only reachable from header");
9777 assert(isa<BranchInst>(
Cond->getTerminator()) &&
9778 "Exiting block must terminate with conditional branch");
9780 "Exiting block must have two successors");
9781 assert(cast<BranchInst>(
Cond->getTerminator())->getSuccessor(0) == Body &&
9782 "Exiting block's first successor jump to the body");
9783 assert(cast<BranchInst>(
Cond->getTerminator())->getSuccessor(1) == Exit &&
9784 "Exiting block's second successor must exit the loop");
9788 "Body only reachable from exiting block");
9793 "Latch must terminate with unconditional branch");
9801 assert(isa<BranchInst>(Exit->getTerminator()) &&
9802 "Exit block must terminate with unconditional branch");
9804 "Exit block must jump to after block");
9808 "After block only reachable from exit block");
9812 assert(IndVar &&
"Canonical induction variable not found?");
9814 "Induction variable must be an integer");
9816 "Induction variable must be a PHI in the loop header");
9817 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
9819 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->
isZero());
9820 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
9822 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
9824 assert(cast<BinaryOperator>(NextIndVar)->
getOpcode() == BinaryOperator::Add);
9825 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
9826 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
9829 Value *TripCount = getTripCount();
9830 assert(TripCount &&
"Loop trip count not found?");
9832 "Trip count and induction variable must have the same type");
9834 auto *CmpI = cast<CmpInst>(&
Cond->front());
9836 "Exit condition must be a signed less-than comparison");
9838 "Exit condition must compare the induction variable");
9840 "Exit condition must compare with the trip count");
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
#define LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE() pulls the operator overloads used by LLVM_MARK_AS_BITMASK_EN...
#define LLVM_MARK_AS_BITMASK_ENUM(LargestValue)
LLVM_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you can perform bitwise operatio...
BlockVerifier::State From
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
DenseMap< Block *, BlockRelaxAux > Blocks
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This header defines various interfaces for pass management in LLVM.
iv Induction Variable Users
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static cl::opt< unsigned > TileSize("fuse-matrix-tile-size", cl::init(4), cl::Hidden, cl::desc("Tile size for matrix instruction fusion using square-shaped tiles."))
uint64_t IntrinsicInst * II
#define OMP_KERNEL_ARG_VERSION
Provides definitions for Target specific Grid Values.
static OMPScheduleType getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks, bool HasSimdModifier)
Determine which scheduling algorithm to use, determined from schedule clause arguments.
static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL)
Make Source branch to Target.
Value * createFakeIntVal(IRBuilderBase &Builder, OpenMPIRBuilder::InsertPointTy OuterAllocaIP, llvm::SmallVectorImpl< Instruction * > &ToBeDeleted, OpenMPIRBuilder::InsertPointTy InnerAllocaIP, const Twine &Name="", bool AsPtr=true)
static FunctionCallee getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
Returns an LLVM function to call for finalizing the dynamic loop using depending on type.
static Function * emitTargetTaskProxyFunction(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI)
Create an entry point for a target task with the following.
static void updateNVPTXMetadata(Function &Kernel, StringRef Name, int32_t Value, bool Min)
static OMPScheduleType getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType, bool HasOrderedClause)
Adds ordering modifier flags to schedule type.
static OMPScheduleType getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType, bool HasSimdModifier, bool HasMonotonic, bool HasNonmonotonic, bool HasOrderedClause)
Adds monotonicity modifier flags to schedule type.
static void addSimdMetadata(BasicBlock *Block, MDNode *AccessGroup, LoopInfo &LI)
Attach llvm.access.group metadata to the memref instructions of Block.
static OMPScheduleType computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks, bool HasSimdModifier, bool HasMonotonicModifier, bool HasNonmonotonicModifier, bool HasOrderedClause)
Determine the schedule type using schedule and ordering clause arguments.
static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType)
static Function * getFreshReductionFunc(Module &M)
static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder, Function *Function)
static MDNode * getNVPTXMDNode(Function &Kernel, StringRef Name)
static FunctionCallee getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
Returns an LLVM function to call for updating the next loop using OpenMP dynamic scheduling depending...
static bool isConflictIP(IRBuilder<>::InsertPoint IP1, IRBuilder<>::InsertPoint IP2)
Return whether IP1 and IP2 are ambiguous, i.e.
static void checkReductionInfos(ArrayRef< OpenMPIRBuilder::ReductionInfo > ReductionInfos, bool IsGPU)
static FunctionCallee getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
Returns an LLVM function to call for initializing loop bounds using OpenMP dynamic scheduling dependi...
static cl::opt< double > UnrollThresholdFactor("openmp-ir-builder-unroll-threshold-factor", cl::Hidden, cl::desc("Factor for the unroll threshold to account for code " "simplifications still taking place"), cl::init(1.5))
static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI)
Heuristically determine the best-performant unroll factor for CLI.
static Value * emitTaskDependencies(OpenMPIRBuilder &OMPBuilder, const SmallVectorImpl< OpenMPIRBuilder::DependData > &Dependencies)
static void workshareLoopTargetCallback(OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident, Function &OutlinedFn, Type *ParallelTaskPtr, const SmallVector< Instruction *, 4 > &ToBeDeleted, WorksharingLoopType LoopType)
static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I)
static void redirectAllPredecessorsTo(BasicBlock *OldTarget, BasicBlock *NewTarget, DebugLoc DL)
Redirect all edges that branch to OldTarget to NewTarget.
static std::unique_ptr< TargetMachine > createTargetMachine(Function *F, CodeGenOptLevel OptLevel)
Create the TargetMachine object to query the backend for optimization preferences.
static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
static void addBasicBlockMetadata(BasicBlock *BB, ArrayRef< Metadata * > Properties)
Attach metadata Properties to the basic block described by BB.
static cl::opt< bool > OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden, cl::desc("Use optimistic attributes describing " "'as-if' properties of runtime calls."), cl::init(false))
static FunctionCallee getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType)
static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType, BasicBlock *InsertBlock, Value *Ident, Value *LoopBodyArg, Type *ParallelTaskPtr, Value *TripCount, Function &LoopBodyFn)
static const omp::GV & getGridValue(const Triple &T, Function *Kernel)
static void addLoopMetadata(CanonicalLoopInfo *Loop, ArrayRef< Metadata * > Properties)
Attach loop metadata Properties to the loop described by Loop.
static void removeUnusedBlocksFromParent(ArrayRef< BasicBlock * > BBs)
Determine which blocks in BBs are reachable from outside and remove the ones that are not reachable f...
static void emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn, Constant *OutlinedFnID, ArrayRef< int32_t > NumTeams, ArrayRef< int32_t > NumThreads, SmallVectorImpl< Value * > &Args, OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB, SmallVector< llvm::OpenMPIRBuilder::DependData > Dependencies={}, bool HasNoWait=false)
static void targetParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn, BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition, Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr, Value *ThreadID, const SmallVector< Instruction *, 4 > &ToBeDeleted)
static Error emitTargetOutlinedFunction(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry, TargetRegionEntryInfo &EntryInfo, Function *&OutlinedFn, Constant *&OutlinedFnID, SmallVectorImpl< Value * > &Inputs, OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc, OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB)
static Expected< Function * > createOutlinedFunction(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName, SmallVectorImpl< Value * > &Inputs, OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc, OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB)
static void hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn, Value *Ident, Value *IfCondition, Instruction *PrivTID, AllocaInst *PrivTIDAddr, const SmallVector< Instruction *, 4 > &ToBeDeleted)
FunctionAnalysisManager FAM
This file defines the Pass Instrumentation classes that provide instrumentation points into the pass ...
const SmallVectorImpl< MachineOperand > & Cond
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the SmallBitVector class.
This file defines the SmallSet class.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static cl::opt< unsigned > MaxThreads("xcore-max-threads", cl::Optional, cl::desc("Maximum number of threads (for emulation thread-local storage)"), cl::Hidden, cl::value_desc("number"), cl::init(8))
static const uint32_t IV[8]
Class for arbitrary precision integers.
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
PointerType * getType() const
Overload to return most specific pointer type.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
unsigned getAddressSpace() const
Return the address space for the allocation.
std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
void setAlignment(Align Align)
const Value * getArraySize() const
Get the number of elements allocated.
A container for analyses that lazily runs them and caches their results.
bool registerPass(PassBuilderT &&PassBuilder)
Register an analysis pass with the manager.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
Class to represent array types.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
A function analysis which provides an AssumptionCache.
AssumptionCache run(Function &F, FunctionAnalysisManager &)
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)
Returns the strongest permitted ordering on failure, given the desired ordering on success.
std::pair< LoadInst *, AllocaInst * > EmitAtomicLoadLibcall(AtomicOrdering AO)
an instruction that atomically reads a memory location, combines it with another value,...
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
AttributeSet getFnAttrs() const
The function attributes are returned.
AttributeList addFnAttributes(LLVMContext &C, const AttrBuilder &B) const
Add function attribute to the list.
AttributeSet addAttributes(LLVMContext &C, AttributeSet AS) const
Add attributes to the attribute set.
AttributeSet addAttribute(LLVMContext &C, Attribute::AttrKind Kind) const
Add an argument attribute.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
StringRef getValueAsString() const
Return the attribute's value as a string.
LLVM Basic Block Representation.
void replaceSuccessorsPhiUsesWith(BasicBlock *Old, BasicBlock *New)
Update all phi nodes in this basic block's successors to refer to basic block New instead of basic bl...
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
reverse_iterator rbegin()
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const BasicBlock * getUniqueSuccessor() const
Return the successor of this block if it has a unique successor.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::reverse_iterator reverse_iterator
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
const Function * getParent() const
Return the enclosing method, or null if none.
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
void splice(BasicBlock::iterator ToIt, BasicBlock *FromBB)
Transfer all instructions from FromBB to this basic block at ToIt.
const Instruction & back() const
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
Class to represented the control flow structure of an OpenMP canonical loop.
Value * getTripCount() const
Returns the llvm::Value containing the number of loop iterations.
BasicBlock * getHeader() const
The header is the entry for each iteration.
void assertOK() const
Consistency self-check.
Type * getIndVarType() const
Return the type of the induction variable (and the trip count).
BasicBlock * getBody() const
The body block is the single entry for a loop iteration and not controlled by CanonicalLoopInfo.
bool isValid() const
Returns whether this object currently represents the IR of a loop.
OpenMPIRBuilder::InsertPointTy getAfterIP() const
Return the insertion point for user code after the loop.
OpenMPIRBuilder::InsertPointTy getBodyIP() const
Return the insertion point for user code in the body.
BasicBlock * getAfter() const
The after block is intended for clean-up code such as lifetime end markers.
Function * getFunction() const
void invalidate()
Invalidate this loop.
BasicBlock * getLatch() const
Reaching the latch indicates the end of the loop body code.
OpenMPIRBuilder::InsertPointTy getPreheaderIP() const
Return the insertion point for user code before the loop.
BasicBlock * getCond() const
The condition block computes whether there is another loop iteration.
BasicBlock * getExit() const
Reaching the exit indicates no more iterations are being executed.
BasicBlock * getPreheader() const
The preheader ensures that there is only a single edge entering the loop.
Instruction * getIndVar() const
Returns the instruction representing the current logical induction variable.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
static Constant * getAddrSpaceCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static ConstantInt * getFalse(LLVMContext &Context)
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
DISPFlags
Debug info subprogram flags.
Type array for a subprogram.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getDefaultGlobalsAddressSpace() const
Align getABIIntegerTypeAlignment(unsigned BitWidth) const
Returns the minimum ABI-required alignment for an integer type of the specified bitwidth.
unsigned getAllocaAddrSpace() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size in bytes, rounded up to a whole number of bytes.
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Analysis pass which computes a DominatorTree.
DominatorTree run(Function &F, FunctionAnalysisManager &)
Run the analysis pass over a function and produce a dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Class to represent function types.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
void removeFromParent()
removeFromParent - This method unlinks 'this' from the containing module, but does not delete it.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
AttributeList getAttributes() const
Return the attribute list for this Function.
const Function & getFunction() const
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Function::iterator insert(Function::iterator Position, BasicBlock *BB)
Insert BB in the basic block list at Position.
Type * getReturnType() const
Returns the type of the ret val.
void setCallingConv(CallingConv::ID CC)
Argument * getArg(unsigned i) const
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void addMetadata(unsigned KindID, MDNode &MD)
Add a metadata attachment.
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
Module * getParent()
Get the module that this global value is contained inside of...
void setDSOLocal(bool Local)
PointerType * getType() const
Global values are always pointers.
@ HiddenVisibility
The GV is hidden.
@ ProtectedVisibility
The GV is protected.
void setVisibility(VisibilityTypes V)
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ CommonLinkage
Tentative definitions.
@ InternalLinkage
Rename collisions when linking (static functions).
@ WeakODRLinkage
Same, but only replaced by something equivalent.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AppendingLinkage
Special purpose, only applies to global arrays.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Type * getValueType() const
InsertPoint - A saved insertion point.
BasicBlock * getBlock() const
bool isSet() const
Returns true if this insert point is set.
BasicBlock::iterator getPoint() const
Common base class shared among various IRBuilders.
Value * CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, const Twine &Name="")
Return the i64 difference between two pointer values, dividing out the size of the pointed-to objects...
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID=SyncScope::System)
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
UnreachableInst * CreateUnreachable()
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, unsigned Alignment, Value *OffsetValue=nullptr)
Create an assume intrinsic call that represents an alignment assumption on the provided pointer.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
IntegerType * getIndexTy(const DataLayout &DL, unsigned AddrSpace)
Fetch the type of an integer that should be used to index GEP operations within AddressSpace.
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name="")
BasicBlock * GetInsertBlock() const
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Value * CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt16Ty()
Fetch the type representing a 16-bit integer.
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
InsertPoint saveIP() const
Returns the current insert point.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
SwitchInst * CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases=10, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a switch instruction with the specified value, default dest, and with a hint for the number of...
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Value * CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, const Twine &Name="")
Value * CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
BranchInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
void ClearInsertionPoint()
Clear the insertion point: created instructions will not be inserted into a block.
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt16(uint16_t C)
Get a constant 16-bit value.
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Type * getVoidTy()
Fetch the type representing void.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateSExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a SExt or Trunc from the integer value V to DestTy.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
GlobalVariable * CreateGlobalString(StringRef Str, const Twine &Name="", unsigned AddressSpace=0, Module *M=nullptr, bool AddNull=true)
Make a new global variable with initializer type i8*.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveBeforePreserving(Instruction *MovePos)
Perform a moveBefore operation, while signalling that the caller intends to preserve the original ord...
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static bool classof(const Value *V)
Methods for support type inquiry through isa, cast, and dyn_cast:
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
Align getAlign() const
Return the alignment of the access that is being performed.
Analysis pass that exposes the LoopInfo for a function.
LoopInfo run(Function &F, FunctionAnalysisManager &AM)
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
This class represents a loop nest and can be used to query its properties.
Represents a single loop in the control flow graph.
MDNode * createCallbackEncoding(unsigned CalleeArgNo, ArrayRef< int > Arguments, bool VarArgsArePassed)
Return metadata describing a callback (see llvm::AbstractCallSite).
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
static MDTuple * getDistinct(LLVMContext &Context, ArrayRef< Metadata * > MDs)
const MDOperand & getOperand(unsigned I) const
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
This class implements a map that also provides access to all stored values in a deterministic order.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
A Module instance is used to store all the information related to an LLVM module.
NamedMDNode * getNamedMetadata(StringRef Name) const
Return the first NamedMDNode in the module with the specified name.
LLVMContext & getContext() const
Get the global data context.
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
StringRef getName() const
Get a short "name" for the module.
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
iterator_range< global_iterator > globals()
const FunctionListType & getFunctionList() const
Get the Module's list of functions (constant).
GlobalVariable * getGlobalVariable(StringRef Name) const
Look up the specified global variable in the module symbol table.
GlobalValue * getNamedValue(StringRef Name) const
Return the global value in the module with the specified name, of arbitrary type.
NamedMDNode * getOrInsertNamedMetadata(StringRef Name)
Return the named MDNode in the module with the specified name.
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
iterator_range< op_iterator > operands()
void addOperand(MDNode *M)
Device global variable entries info.
Target region entries info.
Base class of the entries info.
@ OffloadingEntryInfoTargetRegion
Entry is a target region.
@ OffloadingEntryInfoDeviceGlobalVar
Entry is a declare target variable.
OMPTargetDeviceClauseKind
Kind of device clause for declare target variables and functions NOTE: Currently not used as a part o...
@ OMPTargetDeviceClauseAny
The target is marked for all devices.
void registerDeviceGlobalVarEntryInfo(StringRef VarName, Constant *Addr, int64_t VarSize, OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage)
Register device global variable entry.
void initializeDeviceGlobalVarEntryInfo(StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order)
Initialize device global variable entry.
void actOnDeviceGlobalVarEntriesInfo(const OffloadDeviceGlobalVarEntryInfoActTy &Action)
OMPTargetRegionEntryKind
Kind of the target registry entry.
@ OMPTargetRegionEntryTargetRegion
Mark the entry as target region.
void getTargetRegionEntryFnName(SmallVectorImpl< char > &Name, const TargetRegionEntryInfo &EntryInfo)
bool hasTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId=false) const
Return true if a target region entry with the provided information exists.
void registerTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID, OMPTargetRegionEntryKind Flags)
Register target region entry.
void actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action)
unsigned size() const
Return number of entries defined so far.
void initializeTargetRegionEntryInfo(const TargetRegionEntryInfo &EntryInfo, unsigned Order)
Initialize target region entry.
OMPTargetGlobalVarEntryKind
Kind of the global variable entry..
@ OMPTargetGlobalVarEntryEnter
Mark the entry as a declare target enter.
@ OMPTargetGlobalRegisterRequires
Mark the entry as a register requires global.
@ OMPTargetGlobalVarEntryIndirect
Mark the entry as a declare target indirect global.
@ OMPTargetGlobalVarEntryLink
Mark the entry as a to declare target link.
@ OMPTargetGlobalVarEntryTo
Mark the entry as a to declare target.
bool hasDeviceGlobalVarEntryInfo(StringRef VarName) const
Checks if the variable with the given name has been registered already.
bool empty() const
Return true if a there are no entries defined.
std::optional< bool > IsTargetDevice
Flag to define whether to generate code for the role of the OpenMP host (if set to false) or device (...
void setGridValue(omp::GV G)
StringRef separator() const
int64_t getRequiresFlags() const
Returns requires directive clauses as flags compatible with those expected by libomptarget.
StringRef firstSeparator() const
std::optional< bool > EmitLLVMUsedMetaInfo
Flag for specifying if LLVMUsed information should be emitted.
omp::GV getGridValue() const
void setHasRequiresReverseOffload(bool Value)
bool hasRequiresUnifiedSharedMemory() const
void setHasRequiresUnifiedSharedMemory(bool Value)
bool hasRequiresDynamicAllocators() const
bool openMPOffloadMandatory() const
void setHasRequiresUnifiedAddress(bool Value)
bool isTargetDevice() const
void setHasRequiresDynamicAllocators(bool Value)
void setEmitLLVMUsed(bool Value=true)
bool hasRequiresReverseOffload() const
bool hasRequiresFlags() const
bool hasRequiresUnifiedAddress() const
Struct that keeps the information that should be kept throughout a 'target data' region.
An interface to create LLVM-IR for OpenMP directives.
InsertPointOrErrorTy createOrderedThreadsSimd(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool IsThreads)
Generator for '#omp ordered [threads | simd]'.
Constant * getOrCreateIdent(Constant *SrcLocStr, uint32_t SrcLocStrSize, omp::IdentFlag Flags=omp::IdentFlag(0), unsigned Reserve2Flags=0)
Return an ident_t* encoding the source location SrcLocStr and Flags.
FunctionCallee getOrCreateRuntimeFunction(Module &M, omp::RuntimeFunction FnID)
Return the function declaration for the runtime function with FnID.
InsertPointOrErrorTy createCancel(const LocationDescription &Loc, Value *IfCondition, omp::Directive CanceledDirective)
Generator for '#omp cancel'.
InsertPointTy createTargetInit(const LocationDescription &Loc, bool IsSPMD, int32_t MinThreadsVal=0, int32_t MaxThreadsVal=0, int32_t MinTeamsVal=0, int32_t MaxTeamsVal=0)
The omp target interface.
ReductionGenCBKind
Enum class for the RedctionGen CallBack type to be used.
CanonicalLoopInfo * collapseLoops(DebugLoc DL, ArrayRef< CanonicalLoopInfo * > Loops, InsertPointTy ComputeIP)
Collapse a loop nest into a single loop.
void createTaskyield(const LocationDescription &Loc)
Generator for '#omp taskyield'.
std::function< Error(InsertPointTy CodeGenIP)> FinalizeCallbackTy
Callback type for variable finalization (think destructors).
void emitBranch(BasicBlock *Target)
InsertPointTy createAtomicWrite(const LocationDescription &Loc, AtomicOpValue &X, Value *Expr, AtomicOrdering AO)
Emit atomic write for : X = Expr — Only Scalar data types.
static void writeThreadBoundsForKernel(const Triple &T, Function &Kernel, int32_t LB, int32_t UB)
static TargetRegionEntryInfo getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack, StringRef ParentName="")
Creates a unique info for a target entry when provided a filename and line number from.
void emitTaskwaitImpl(const LocationDescription &Loc)
Generate a taskwait runtime call.
Constant * registerTargetRegionFunction(TargetRegionEntryInfo &EntryInfo, Function *OutlinedFunction, StringRef EntryFnName, StringRef EntryFnIDName)
Registers the given function and sets up the attribtues of the function Returns the FunctionID.
void initialize()
Initialize the internal state, this will put structures types and potentially other helpers into the ...
void createTargetDeinit(const LocationDescription &Loc, int32_t TeamsReductionDataSize=0, int32_t TeamsReductionBufferLength=1024)
Create a runtime call for kmpc_target_deinit.
InsertPointOrErrorTy createTaskgroup(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB)
Generator for the taskgroup construct.
void loadOffloadInfoMetadata(Module &M)
Loads all the offload entries information from the host IR metadata.
InsertPointOrErrorTy emitTargetTask(TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc, OpenMPIRBuilder::InsertPointTy AllocaIP, const SmallVector< llvm::OpenMPIRBuilder::DependData > &Dependencies, bool HasNoWait)
Generate a target-task for the target construct.
void unrollLoopFull(DebugLoc DL, CanonicalLoopInfo *Loop)
Fully unroll a loop.
void emitFlush(const LocationDescription &Loc)
Generate a flush runtime call.
static std::pair< int32_t, int32_t > readThreadBoundsForKernel(const Triple &T, Function &Kernel)
}
OpenMPIRBuilderConfig Config
The OpenMPIRBuilder Configuration.
CallInst * createOMPInteropDestroy(const LocationDescription &Loc, Value *InteropVar, Value *Device, Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause)
Create a runtime call for __tgt_interop_destroy.
InsertPointTy createAtomicRead(const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V, AtomicOrdering AO)
Emit atomic Read for : V = X — Only Scalar data types.
Error emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen, BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP={})
Emits code for OpenMP 'if' clause using specified BodyGenCallbackTy Here is the logic: if (Cond) { Th...
std::function< void(EmitMetadataErrorKind, TargetRegionEntryInfo)> EmitMetadataErrorReportFunctionTy
Callback function type.
InsertPointOrErrorTy createSingle(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef< llvm::Value * > CPVars={}, ArrayRef< llvm::Function * > CPFuncs={})
Generator for '#omp single'.
InsertPointOrErrorTy createTeams(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower=nullptr, Value *NumTeamsUpper=nullptr, Value *ThreadLimit=nullptr, Value *IfExpr=nullptr)
Generator for #omp teams
std::forward_list< CanonicalLoopInfo > LoopInfos
Collection of owned canonical loop objects that eventually need to be free'd.
void createTaskwait(const LocationDescription &Loc)
Generator for '#omp taskwait'.
CanonicalLoopInfo * createLoopSkeleton(DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore, BasicBlock *PostInsertBefore, const Twine &Name={})
Create the control flow structure of a canonical OpenMP loop.
std::string createPlatformSpecificName(ArrayRef< StringRef > Parts) const
Get the create a name using the platform specific separators.
FunctionCallee createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned.
static void getKernelArgsVector(TargetKernelArgs &KernelArgs, IRBuilderBase &Builder, SmallVector< Value * > &ArgsVector)
Create the kernel args vector used by emitTargetKernel.
void unrollLoopHeuristic(DebugLoc DL, CanonicalLoopInfo *Loop)
Fully or partially unroll a loop.
InsertPointOrErrorTy createParallel(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable)
Generator for '#omp parallel'.
omp::OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position)
Get OMP_MAP_MEMBER_OF flag with extra bits reserved based on the position given.
void addAttributes(omp::RuntimeFunction FnID, Function &Fn)
Add attributes known for FnID to Fn.
Module & M
The underlying LLVM-IR module.
StringMap< Constant * > SrcLocStrMap
Map to remember source location strings.
void createMapperAllocas(const LocationDescription &Loc, InsertPointTy AllocaIP, unsigned NumOperands, struct MapperAllocas &MapperAllocas)
Create the allocas instruction used in call to mapper functions.
Constant * getOrCreateSrcLocStr(StringRef LocStr, uint32_t &SrcLocStrSize)
Return the (LLVM-IR) string describing the source location LocStr.
void addOutlineInfo(OutlineInfo &&OI)
Add a new region that will be outlined later.
Error emitTargetRegionFunction(TargetRegionEntryInfo &EntryInfo, FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry, Function *&OutlinedFn, Constant *&OutlinedFnID)
Create a unique name for the entry function using the source location information of the current targ...
FunctionCallee createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned.
InsertPointOrErrorTy createTarget(const LocationDescription &Loc, bool IsOffloadEntry, OpenMPIRBuilder::InsertPointTy AllocaIP, OpenMPIRBuilder::InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo, ArrayRef< int32_t > NumTeams, ArrayRef< int32_t > NumThreads, SmallVectorImpl< Value * > &Inputs, GenMapInfoCallbackTy GenMapInfoCB, TargetBodyGenCallbackTy BodyGenCB, TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB, SmallVector< DependData > Dependencies={}, bool HasNowait=false)
Generator for '#omp target'.
void unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop, int32_t Factor, CanonicalLoopInfo **UnrolledCLI)
Partially unroll a loop.
void emitTaskyieldImpl(const LocationDescription &Loc)
Generate a taskyield runtime call.
void emitMapperCall(const LocationDescription &Loc, Function *MapperFunc, Value *SrcLocInfo, Value *MaptypesArg, Value *MapnamesArg, struct MapperAllocas &MapperAllocas, int64_t DeviceID, unsigned NumOperands)
Create the call for the target mapper function.
InsertPointTy createAtomicCompare(const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V, AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO, omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly)
Emit atomic compare for constructs: — Only scalar data types cond-expr-stmt: x = x ordop expr ?...
InsertPointOrErrorTy createAtomicCapture(const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, AtomicOpValue &V, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr)
Emit atomic update for constructs: — Only Scalar data types V = X; X = X BinOp Expr ,...
InsertPointTy createOrderedDepend(const LocationDescription &Loc, InsertPointTy AllocaIP, unsigned NumLoops, ArrayRef< llvm::Value * > StoreValues, const Twine &Name, bool IsDependSource)
Generator for '#omp ordered depend (source | sink)'.
InsertPointTy createCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, llvm::IntegerType *IntPtrTy, bool BranchtoEnd=true)
Generate conditional branch and relevant BasicBlocks through which private threads copy the 'copyin' ...
void emitOffloadingArrays(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo, TargetDataInfo &Info, bool IsNonContiguous=false, function_ref< void(unsigned int, Value *)> DeviceAddrCB=nullptr, function_ref< Value *(unsigned int)> CustomMapperCB=nullptr)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
SmallVector< FinalizationInfo, 8 > FinalizationStack
The finalization stack made up of finalize callbacks currently in-flight, wrapped into FinalizationIn...
std::vector< CanonicalLoopInfo * > tileLoops(DebugLoc DL, ArrayRef< CanonicalLoopInfo * > Loops, ArrayRef< Value * > TileSizes)
Tile a loop nest.
CallInst * createOMPInteropInit(const LocationDescription &Loc, Value *InteropVar, omp::OMPInteropType InteropType, Value *Device, Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause)
Create a runtime call for __tgt_interop_init.
void finalize(Function *Fn=nullptr)
Finalize the underlying module, e.g., by outlining regions.
SmallVector< OutlineInfo, 16 > OutlineInfos
Collection of regions that need to be outlined during finalization.
Function * getOrCreateRuntimeFunctionPtr(omp::RuntimeFunction FnID)
const Triple T
The target triple of the underlying module.
DenseMap< std::pair< Constant *, uint64_t >, Constant * > IdentMap
Map to remember existing ident_t*.
CallInst * createOMPFree(const LocationDescription &Loc, Value *Addr, Value *Allocator, std::string Name="")
Create a runtime call for kmpc_free.
FunctionCallee createForStaticInitFunction(unsigned IVSize, bool IVSigned, bool IsGPUDistribute)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned.
CallInst * createOMPAlloc(const LocationDescription &Loc, Value *Size, Value *Allocator, std::string Name="")
Create a runtime call for kmpc_Alloc.
void emitNonContiguousDescriptor(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo, TargetDataInfo &Info)
Emit an array of struct descriptors to be assigned to the offload args.
InsertPointOrErrorTy createSection(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB)
Generator for '#omp section'.
void emitBlock(BasicBlock *BB, Function *CurFn, bool IsFinished=false)
Value * getOrCreateThreadID(Value *Ident)
Return the current thread ID.
void emitOffloadingArraysAndArgs(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info, TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo, bool IsNonContiguous=false, bool ForEndCall=false, function_ref< void(unsigned int, Value *)> DeviceAddrCB=nullptr, function_ref< Value *(unsigned int)> CustomMapperCB=nullptr)
Allocates memory for and populates the arrays required for offloading (offload_{baseptrs|ptrs|mappers...
InsertPointOrErrorTy createMaster(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB)
Generator for '#omp master'.
Error emitCancelationCheckImpl(Value *CancelFlag, omp::Directive CanceledDirective, FinalizeCallbackTy ExitCB={})
Generate control flow and cleanup for cancellation.
InsertPointOrErrorTy emitKernelLaunch(const LocationDescription &Loc, Value *OutlinedFnID, EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args, Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP)
Generate a target region entry call and host fallback call.
StringMap< GlobalVariable *, BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
GlobalVariable * getOrCreateInternalVariable(Type *Ty, const StringRef &Name, unsigned AddressSpace=0)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
InsertPointOrErrorTy createReductionsGPU(const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, ArrayRef< ReductionInfo > ReductionInfos, bool IsNoWait=false, bool IsTeamsReduction=false, bool HasDistribute=false, ReductionGenCBKind ReductionGenCBKind=ReductionGenCBKind::MLIR, std::optional< omp::GV > GridValue={}, unsigned ReductionBufNum=1024, Value *SrcLocInfo=nullptr)
Design of OpenMP reductions on the GPU.
FunctionCallee createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned.
Function * emitUserDefinedMapper(function_ref< MapInfosTy &(InsertPointTy CodeGenIP, llvm::Value *PtrPHI, llvm::Value *BeginArg)> PrivAndGenMapInfoCB, llvm::Type *ElemTy, StringRef FuncName, function_ref< bool(unsigned int, Function **)> CustomMapperCB=nullptr)
Emit the user-defined mapper function.
CallInst * createOMPInteropUse(const LocationDescription &Loc, Value *InteropVar, Value *Device, Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause)
Create a runtime call for __tgt_interop_use.
IRBuilder<>::InsertPoint InsertPointTy
Type used throughout for insertion points.
InsertPointOrErrorTy createReductions(const LocationDescription &Loc, InsertPointTy AllocaIP, ArrayRef< ReductionInfo > ReductionInfos, ArrayRef< bool > IsByRef, bool IsNoWait=false)
Generator for '#omp reduction'.
GlobalVariable * createOffloadMapnames(SmallVectorImpl< llvm::Constant * > &Names, std::string VarName)
Create the global variable holding the offload names information.
InsertPointOrErrorTy createTask(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, bool Tied=true, Value *Final=nullptr, Value *IfCondition=nullptr, SmallVector< DependData > Dependencies={}, bool Mergeable=false, Value *EventHandle=nullptr)
Generator for #omp task
std::function< Expected< Function * >(StringRef FunctionName)> FunctionGenCallback
Functions used to generate a function with the given name.
static void writeTeamsForKernel(const Triple &T, Function &Kernel, int32_t LB, int32_t UB)
InsertPointOrErrorTy createBarrier(const LocationDescription &Loc, omp::Directive Kind, bool ForceSimpleCall=false, bool CheckCancelFlag=true)
Emitter methods for OpenMP directives.
void setCorrectMemberOfFlag(omp::OpenMPOffloadMappingFlags &Flags, omp::OpenMPOffloadMappingFlags MemberOfFlag)
Given an initial flag set, this function modifies it to contain the passed in MemberOfFlag generated ...
Constant * getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize)
Return the (LLVM-IR) string describing the default source location.
InsertPointOrErrorTy createCritical(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst)
Generator for '#omp critical'.
void createOffloadEntry(Constant *ID, Constant *Addr, uint64_t Size, int32_t Flags, GlobalValue::LinkageTypes, StringRef Name="")
Creates offloading entry for the provided entry ID ID, address Addr, size Size, and flags Flags.
static unsigned getOpenMPDefaultSimdAlign(const Triple &TargetTriple, const StringMap< bool > &Features)
Get the default alignment value for given target.
unsigned getFlagMemberOffset()
Get the offset of the OMP_MAP_MEMBER_OF field.
void createOffloadEntriesAndInfoMetadata(EmitMetadataErrorReportFunctionTy &ErrorReportFunction)
void applySimd(CanonicalLoopInfo *Loop, MapVector< Value *, Value * > AlignedVars, Value *IfCond, omp::OrderKind Order, ConstantInt *Simdlen, ConstantInt *Safelen)
Add metadata to simd-ize a loop.
bool isLastFinalizationInfoCancellable(omp::Directive DK)
Return true if the last entry in the finalization stack is of kind DK and cancellable.
InsertPointTy emitTargetKernel(const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return, Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads, Value *HostPtr, ArrayRef< Value * > KernelArgs)
Generate a target region entry call.
GlobalVariable * createOffloadMaptypes(SmallVectorImpl< uint64_t > &Mappings, std::string VarName)
Create the global variable holding the offload mappings information.
CallInst * createCachedThreadPrivate(const LocationDescription &Loc, llvm::Value *Pointer, llvm::ConstantInt *Size, const llvm::Twine &Name=Twine(""))
Create a runtime call for kmpc_threadprivate_cached.
IRBuilder Builder
The LLVM-IR Builder used to create IR.
GlobalValue * createGlobalFlag(unsigned Value, StringRef Name)
Create a hidden global flag Name in the module with initial value Value.
InsertPointOrErrorTy applyWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, bool NeedsBarrier, llvm::omp::ScheduleKind SchedKind=llvm::omp::OMP_SCHEDULE_Default, Value *ChunkSize=nullptr, bool HasSimdModifier=false, bool HasMonotonicModifier=false, bool HasNonmonotonicModifier=false, bool HasOrderedClause=false, omp::WorksharingLoopType LoopType=omp::WorksharingLoopType::ForStaticLoop)
Modifies the canonical loop to be a workshare loop.
void emitOffloadingArraysArgument(IRBuilderBase &Builder, OpenMPIRBuilder::TargetDataRTArgs &RTArgs, OpenMPIRBuilder::TargetDataInfo &Info, bool ForEndCall=false)
Emit the arguments to be passed to the runtime library based on the arrays of base pointers,...
InsertPointOrErrorTy createMasked(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, Value *Filter)
Generator for '#omp masked'.
Expected< CanonicalLoopInfo * > createCanonicalLoop(const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB, Value *TripCount, const Twine &Name="loop")
Generator for the control flow structure of an OpenMP canonical loop.
Value * getSizeInBytes(Value *BasePtr)
Computes the size of type in bytes.
FunctionCallee createDispatchDeinitFunction()
Returns __kmpc_dispatch_deinit runtime function.
void registerTargetGlobalVariable(OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause, OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause, bool IsDeclaration, bool IsExternallyVisible, TargetRegionEntryInfo EntryInfo, StringRef MangledName, std::vector< GlobalVariable * > &GeneratedRefs, bool OpenMPSIMD, std::vector< Triple > TargetTriple, std::function< Constant *()> GlobalInitializer, std::function< GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy, Constant *Addr)
Registers a target variable for device or host.
InsertPointOrErrorTy createTargetData(const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond, TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB, omp::RuntimeFunction *MapperFunc=nullptr, function_ref< InsertPointOrErrorTy(InsertPointTy CodeGenIP, BodyGenTy BodyGenType)> BodyGenCB=nullptr, function_ref< void(unsigned int, Value *)> DeviceAddrCB=nullptr, function_ref< Value *(unsigned int)> CustomMapperCB=nullptr, Value *SrcLocInfo=nullptr)
Generator for '#omp target data'.
BodyGenTy
Type of BodyGen to use for region codegen.
InsertPointOrErrorTy createAtomicUpdate(const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr)
Emit atomic update for constructs: X = X BinOp Expr ,or X = Expr BinOp X For complex Operations: X = ...
SmallVector< llvm::Function *, 16 > ConstantAllocaRaiseCandidates
A collection of candidate target functions that's constant allocas will attempt to be raised on a cal...
OffloadEntriesInfoManager OffloadInfoManager
Info manager to keep track of target regions.
static std::pair< int32_t, int32_t > readTeamBoundsForKernel(const Triple &T, Function &Kernel)
Read/write a bounds on teams for Kernel.
std::function< std::tuple< std::string, uint64_t >()> FileIdentifierInfoCallbackTy
const std::string ompOffloadInfoName
OMP Offload Info Metadata name string.
Expected< InsertPointTy > InsertPointOrErrorTy
Type used to represent an insertion point or an error value.
InsertPointTy createCopyPrivate(const LocationDescription &Loc, llvm::Value *BufSize, llvm::Value *CpyBuf, llvm::Value *CpyFn, llvm::Value *DidIt)
Generator for __kmpc_copyprivate.
InsertPointOrErrorTy createSections(const LocationDescription &Loc, InsertPointTy AllocaIP, ArrayRef< StorableBodyGenCallbackTy > SectionCBs, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait)
Generator for '#omp sections'.
bool updateToLocation(const LocationDescription &Loc)
Update the internal location to Loc.
void createFlush(const LocationDescription &Loc)
Generator for '#omp flush'.
Constant * getAddrOfDeclareTargetVar(OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause, OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause, bool IsDeclaration, bool IsExternallyVisible, TargetRegionEntryInfo EntryInfo, StringRef MangledName, std::vector< GlobalVariable * > &GeneratedRefs, bool OpenMPSIMD, std::vector< Triple > TargetTriple, Type *LlvmPtrTy, std::function< Constant *()> GlobalInitializer, std::function< GlobalValue::LinkageTypes()> VariableLinkage)
Retrieve (or create if non-existent) the address of a declare target variable, used in conjunction wi...
EmitMetadataErrorKind
The kind of errors that can occur when emitting the offload entries and metadata.
@ EMIT_MD_DECLARE_TARGET_ERROR
@ EMIT_MD_GLOBAL_VAR_LINK_ERROR
@ EMIT_MD_TARGET_REGION_ERROR
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Pseudo-analysis pass that exposes the PassInstrumentation to pass managers.
Class to represent pointers.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Analysis pass that exposes the ScalarEvolution for a function.
ScalarEvolution run(Function &F, FunctionAnalysisManager &AM)
The main scalar evolution driver.
A vector that has set insertion semantics.
bool remove_if(UnaryPredicate P)
Remove items from the set vector based on a predicate function.
bool empty() const
Determine if the SetVector is empty or not.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool test(unsigned Idx) const
bool all() const
Returns true if all bits are set.
bool any() const
Returns true if any bit is set.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
void setAlignment(Align Align)
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this store instruction.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
ValueTy lookup(StringRef Key) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr size_t size() const
size - Get the string size.
size_t count(char C) const
Return the number of occurrences of C in the string.
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
StringRef drop_back(size_t N=1) const
Return a StringRef equal to 'this' but with the last N elements dropped.
Class to represent struct types.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
Analysis pass providing the TargetTransformInfo.
Result run(const Function &F, FunctionAnalysisManager &)
Analysis pass providing the TargetLibraryInfo.
Target - Wrapper for Target specific information.
TargetMachine * createTargetMachine(StringRef TT, StringRef CPU, StringRef Features, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM=std::nullopt, CodeGenOptLevel OL=CodeGenOptLevel::Default, bool JIT=false) const
createTargetMachine - Create a target specific machine implementation for the specified Triple.
Triple - Helper class for working with autoconf configuration names.
bool isPPC() const
Tests whether the target is PowerPC (32- or 64-bit LE or BE).
bool isX86() const
Tests whether the target is x86 (32- or 64-bit).
bool isWasm() const
Tests whether the target is wasm (32- and 64-bit).
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
bool isPointerTy() const
True if this is an instance of PointerType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isStructTy() const
True if this is an instance of StructType.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
Produce an estimate of the unrolled cost of the specified loop.
bool canUnroll() const
Whether it is legal to unroll this loop.
uint64_t getRolledLoopSize() const
A Use represents the edge between a Value definition and its users.
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
void replaceUsesWithIf(Value *New, llvm::function_ref< bool(Use &U)> ShouldReplace)
Go through the uses list for this definition and make each use point to "V" if the callback ShouldRep...
User * getUniqueUndroppableUser()
Return true if there is exactly one unique user of this value that cannot be dropped (that user can h...
unsigned getNumUses() const
This method computes the number of uses of this Value.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
iterator insertAfter(iterator where, pointer New)
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
initializer< Ty > init(const Ty &Val)
void emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, uint64_t Size, int32_t Flags, int32_t Data, StringRef SectionName)
Create an offloading section struct used to register this global at runtime.
OpenMPOffloadMappingFlags
Values for bit flags used to specify the mapping type for offloading.
@ OMP_MAP_PTR_AND_OBJ
The element being mapped is a pointer-pointee pair; both the pointer and the pointee should be mapped...
@ OMP_MAP_MEMBER_OF
The 16 MSBs of the flags indicate whether the entry is member of some struct/class.
@ OMP_DEVICEID_UNDEF
Device ID if the device was not defined, runtime should get it from environment variables in the spec...
IdentFlag
IDs for all omp runtime library ident_t flag encodings (see their defintion in openmp/runtime/src/kmp...
RuntimeFunction
IDs for all omp runtime library (RTL) functions.
static constexpr GV NVPTXGridValues
For Nvidia GPUs.
@ OMP_TGT_EXEC_MODE_GENERIC
WorksharingLoopType
A type of worksharing loop construct.
OMPAtomicCompareOp
Atomic compare operations. Currently OpenMP only supports ==, >, and <.
NodeAddr< PhiNode * > Phi
std::error_code getUniqueID(const Twine Path, UniqueID &Result)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
BasicBlock * splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch, llvm::Twine Suffix=".split")
Like splitBB, but reuses the current block's name for the new name.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Expected< std::unique_ptr< Module > > parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context, ParserCallbacks Callbacks={})
Read the specified bitcode file, returning the module.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
auto successors(const MachineBasicBlock *BB)
testing::Matcher< const detail::ErrorHolder & > Failed()
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
ErrorOr< T > expectedToErrorOrAndEmitErrors(LLVMContext &Ctx, Expected< T > Val)
bool convertUsersOfConstantsToInstructions(ArrayRef< Constant * > Consts, Function *RestrictToFunc=nullptr, bool RemoveDeadConstants=true, bool IncludeSelf=false)
Replace constant expressions users of the given constants with instructions.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
auto reverse(ContainerTy &&C)
TargetTransformInfo::PeelingPreferences gatherPeelingPreferences(Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, std::optional< bool > UserAllowPeeling, std::optional< bool > UserAllowProfileBasedPeeling, bool UnrollingSpecficValues=false)
void spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New, bool CreateBranch)
Move the instruction after an InsertPoint to the beginning of another BasicBlock.
void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
BasicBlock * splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch, llvm::Twine Name={})
Split a BasicBlock at an InsertPoint, even if the block is degenerate (missing the terminator).
CodeGenOptLevel
Code generation optimization level.
bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI, AssumptionCache *AC, ScalarEvolution &SE, const SmallPtrSetImpl< const Value * > &EphValues, OptimizationRemarkEmitter *ORE, unsigned TripCount, unsigned MaxTripCount, bool MaxOrZero, unsigned TripMultiple, const UnrollCostEstimator &UCE, TargetTransformInfo::UnrollingPreferences &UP, TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound)
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
BasicBlock * CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix="", Function *F=nullptr, ClonedCodeInfo *CodeInfo=nullptr)
Return a copy of the specified basic block, but without embedding the block into a particular functio...
bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false, DominatorTree *DT=nullptr)
Attempts to merge a block into its predecessor, if possible.
DWARFExpression::Operation Op
void remapInstructionsInBlocks(ArrayRef< BasicBlock * > Blocks, ValueToValueMapTy &VMap)
Remaps instructions in Blocks using the mapping in VMap.
TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, llvm::OptimizationRemarkEmitter &ORE, int OptLevel, std::optional< unsigned > UserThreshold, std::optional< unsigned > UserCount, std::optional< bool > UserAllowPartial, std::optional< bool > UserRuntime, std::optional< bool > UserUpperBound, std::optional< unsigned > UserFullUnrollMaxCount)
Gather the various unrolling parameters based on the defaults, compiler flags, TTI overrides and user...
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto predecessors(const MachineBasicBlock *BB)
Constant * ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, ArrayRef< unsigned > Idxs)
ConstantFoldInsertValueInstruction - Attempt to constant fold an insertvalue instruction with the spe...
void DeleteDeadBlocks(ArrayRef< BasicBlock * > BBs, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified blocks from BB.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
a struct to pack relevant information while generating atomic Ops
A struct to pack the relevant information for an OpenMP depend clause.
Description of a LLVM-IR insertion point (IP) and a debug/source location (filename,...
MapNonContiguousArrayTy Offsets
MapNonContiguousArrayTy Counts
MapNonContiguousArrayTy Strides
This structure contains combined information generated for mappable clauses, including base pointers,...
MapDeviceInfoArrayTy DevicePointers
MapValuesArrayTy BasePointers
MapValuesArrayTy Pointers
StructNonContiguousInfo NonContigInfo
Helper that contains information about regions we need to outline during finalization.
PostOutlineCBTy PostOutlineCB
void collectBlocks(SmallPtrSetImpl< BasicBlock * > &BlockSet, SmallVectorImpl< BasicBlock * > &BlockVector)
Collect all blocks in between EntryBB and ExitBB in both the given vector and set.
SmallVector< Value *, 2 > ExcludeArgsFromAggregate
BasicBlock * OuterAllocaBB
Information about an OpenMP reduction.
EvalKind EvaluationKind
Reduction evaluation kind - scalar, complex or aggregate.
ReductionGenAtomicCBTy AtomicReductionGen
Callback for generating the atomic reduction body, may be null.
ReductionGenCBTy ReductionGen
Callback for generating the reduction body.
Value * Variable
Reduction variable of pointer type.
Value * PrivateVariable
Thread-private partial reduction variable.
ReductionGenClangCBTy ReductionGenClang
Clang callback for generating the reduction body.
Type * ElementType
Reduction element type, must match pointee type of variable.
Container for the arguments used to pass data to the runtime library.
Value * SizesArray
The array of sizes passed to the runtime library.
Value * PointersArray
The array of section pointers passed to the runtime library.
Value * MappersArray
The array of user-defined mappers passed to the runtime library.
Value * BasePointersArray
The array of base pointer passed to the runtime library.
Value * MapTypesArray
The array of map types passed to the runtime library for the beginning of the region or for the entir...
Value * MapNamesArray
The array of original declaration names of mapped pointers sent to the runtime library for debugging.
Data structure that contains the needed information to construct the kernel args vector.
Value * DynCGGroupMem
The size of the dynamic shared memory.
ArrayRef< Value * > NumThreads
The number of threads.
TargetDataRTArgs RTArgs
Arguments passed to the runtime library.
Value * NumIterations
The number of iterations.
unsigned NumTargetItems
Number of arguments passed to the runtime library.
bool HasNoWait
True if the kernel has 'no wait' clause.
ArrayRef< Value * > NumTeams
The number of teams.
Data structure to contain the information needed to uniquely identify a target entry.
static void getTargetRegionEntryFnName(SmallVectorImpl< char > &Name, StringRef ParentName, unsigned DeviceID, unsigned FileID, unsigned Line, unsigned Count)
static const Target * lookupTarget(StringRef Triple, std::string &Error)
lookupTarget - Lookup a target based on a target triple.
Defines various target-specific GPU grid values that must be consistent between host RTL (plugin),...
unsigned GV_Warp_Size
The default value of maximum number of threads in a worker warp.