52#include "llvm/IR/IntrinsicsAMDGPU.h"
53#include "llvm/IR/IntrinsicsNVPTX.h"
74#define DEBUG_TYPE "openmp-opt"
77 "Number of OpenMP runtime calls deduplicated");
79 "Number of OpenMP parallel regions deleted");
81 "Number of OpenMP runtime functions identified");
83 "Number of OpenMP runtime function uses identified");
85 "Number of OpenMP target region entry points (=kernels) identified");
87 "Number of non-OpenMP target region kernels identified");
89 "Number of OpenMP target region entry points (=kernels) executed in "
90 "SPMD-mode instead of generic-mode");
91STATISTIC(NumOpenMPTargetRegionKernelsWithoutStateMachine,
92 "Number of OpenMP target region entry points (=kernels) executed in "
93 "generic-mode without a state machines");
94STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback,
95 "Number of OpenMP target region entry points (=kernels) executed in "
96 "generic-mode with customized state machines with fallback");
97STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback,
98 "Number of OpenMP target region entry points (=kernels) executed in "
99 "generic-mode with customized state machines without fallback");
101 NumOpenMPParallelRegionsReplacedInGPUStateMachine,
102 "Number of OpenMP parallel regions replaced with ID in GPU state machines");
104 "Number of OpenMP parallel regions merged");
106 "Amount of memory pushed to shared memory");
107STATISTIC(NumBarriersEliminated,
"Number of redundant barriers eliminated");
135#define KERNEL_ENVIRONMENT_IDX(MEMBER, IDX) \
136 constexpr unsigned MEMBER##Idx = IDX;
141#undef KERNEL_ENVIRONMENT_IDX
143#define KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MEMBER, IDX) \
144 constexpr unsigned MEMBER##Idx = IDX;
154#undef KERNEL_ENVIRONMENT_CONFIGURATION_IDX
156#define KERNEL_ENVIRONMENT_GETTER(MEMBER, RETURNTYPE) \
157 RETURNTYPE *get##MEMBER##FromKernelEnvironment(ConstantStruct *KernelEnvC) { \
158 return cast<RETURNTYPE>(KernelEnvC->getAggregateElement(MEMBER##Idx)); \
164#undef KERNEL_ENVIRONMENT_GETTER
166#define KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MEMBER) \
167 ConstantInt *get##MEMBER##FromKernelEnvironment( \
168 ConstantStruct *KernelEnvC) { \
169 ConstantStruct *ConfigC = \
170 getConfigurationFromKernelEnvironment(KernelEnvC); \
171 return dyn_cast<ConstantInt>(ConfigC->getAggregateElement(MEMBER##Idx)); \
182#undef KERNEL_ENVIRONMENT_CONFIGURATION_GETTER
186 constexpr int InitKernelEnvironmentArgNo = 0;
201struct AAHeapToShared;
208 OMPInformationCache(
Module &M, AnalysisGetter &AG,
212 OpenMPPostLink(OpenMPPostLink) {
215 const Triple
T(OMPBuilder.M.getTargetTriple());
216 switch (
T.getArch()) {
220 assert(OMPBuilder.Config.IsTargetDevice &&
221 "OpenMP AMDGPU/NVPTX is only prepared to deal with device code.");
222 OMPBuilder.Config.IsGPU =
true;
225 OMPBuilder.Config.IsGPU =
false;
228 OMPBuilder.initialize();
229 initializeRuntimeFunctions(M);
230 initializeInternalControlVars();
234 struct InternalControlVarInfo {
242 StringRef EnvVarName;
248 ConstantInt *InitValue;
261 struct RuntimeFunctionInfo {
282 using UseVector = SmallVector<Use *, 16>;
285 void clearUsesMap() { UsesMap.clear(); }
288 operator bool()
const {
return Declaration; }
291 UseVector &getOrCreateUseVector(
Function *
F) {
292 std::shared_ptr<UseVector> &UV = UsesMap[
F];
294 UV = std::make_shared<UseVector>();
300 const UseVector *getUseVector(
Function &
F)
const {
301 auto I = UsesMap.find(&
F);
302 if (
I != UsesMap.end())
303 return I->second.get();
308 size_t getNumFunctionsWithUses()
const {
return UsesMap.size(); }
312 size_t getNumArgs()
const {
return ArgumentTypes.size(); }
317 void foreachUse(SmallVectorImpl<Function *> &SCC,
318 function_ref<
bool(Use &,
Function &)> CB) {
326 SmallVector<unsigned, 8> ToBeDeleted;
330 UseVector &UV = getOrCreateUseVector(
F);
340 while (!ToBeDeleted.
empty()) {
350 DenseMap<Function *, std::shared_ptr<UseVector>> UsesMap;
354 decltype(UsesMap)::iterator
begin() {
return UsesMap.begin(); }
355 decltype(UsesMap)::iterator
end() {
return UsesMap.end(); }
359 OpenMPIRBuilder OMPBuilder;
363 RuntimeFunction::OMPRTL___last>
367 DenseMap<Function *, RuntimeFunction> RuntimeFunctionIDMap;
371 InternalControlVar::ICV___last>
376 void initializeInternalControlVars() {
377#define ICV_RT_SET(_Name, RTL) \
379 auto &ICV = ICVs[_Name]; \
382#define ICV_RT_GET(Name, RTL) \
384 auto &ICV = ICVs[Name]; \
387#define ICV_DATA_ENV(Enum, _Name, _EnvVarName, Init) \
389 auto &ICV = ICVs[Enum]; \
392 ICV.InitKind = Init; \
393 ICV.EnvVarName = _EnvVarName; \
394 switch (ICV.InitKind) { \
395 case ICV_IMPLEMENTATION_DEFINED: \
396 ICV.InitValue = nullptr; \
399 ICV.InitValue = ConstantInt::get( \
400 Type::getInt32Ty(OMPBuilder.Int32->getContext()), 0); \
403 ICV.InitValue = ConstantInt::getFalse(OMPBuilder.Int1->getContext()); \
409#include "llvm/Frontend/OpenMP/OMPKinds.def"
415 static bool declMatchesRTFTypes(
Function *
F,
Type *RTFRetType,
422 if (
F->getReturnType() != RTFRetType)
424 if (
F->arg_size() != RTFArgTypes.
size())
427 auto *RTFTyIt = RTFArgTypes.
begin();
428 for (Argument &Arg :
F->args()) {
429 if (Arg.getType() != *RTFTyIt)
439 unsigned collectUses(RuntimeFunctionInfo &RFI,
bool CollectStats =
true) {
440 unsigned NumUses = 0;
441 if (!RFI.Declaration)
443 OMPBuilder.addAttributes(RFI.Kind, *RFI.Declaration);
446 NumOpenMPRuntimeFunctionsIdentified += 1;
447 NumOpenMPRuntimeFunctionUsesIdentified += RFI.Declaration->getNumUses();
451 for (Use &U : RFI.Declaration->uses()) {
453 if (!
CGSCC ||
CGSCC->empty() ||
CGSCC->contains(UserI->getFunction())) {
454 RFI.getOrCreateUseVector(UserI->getFunction()).push_back(&U);
458 RFI.getOrCreateUseVector(
nullptr).push_back(&U);
467 auto &RFI = RFIs[RTF];
469 collectUses(RFI,
false);
477 void setCallbackMetadata(
Function *
F,
unsigned ArgNo, ArrayRef<int> Indices,
479 if (!
F ||
F->hasMetadata(LLVMContext::MD_callback))
482 LLVMContext &Ctx =
F->getContext();
484 F->addMetadata(LLVMContext::MD_callback,
485 *
MDNode::get(Ctx, {MDB.createCallbackEncoding(ArgNo, Indices,
493 static Function *getAnalyzableCallback(
const CallBase &CB) {
497 MDNode *CallbackMD =
Callee->getMetadata(LLVMContext::MD_callback);
503 if (!Encoding || Encoding->getNumOperands() == 0)
514 if (!Callback ||
Callback->isDeclaration())
520 void recollectUses() {
521 for (
int Idx = 0;
Idx < RFIs.size(); ++
Idx)
526 void setCallingConvention(FunctionCallee Callee, CallInst *CI) {
541 RuntimeFunctionInfo &RFI = RFIs[Fn];
543 if (!RFI.Declaration || RFI.Declaration->isDeclaration())
551 void initializeRuntimeFunctions(
Module &M) {
554#define OMP_TYPE(VarName, ...) \
555 Type *VarName = OMPBuilder.VarName; \
558#define OMP_ARRAY_TYPE(VarName, ...) \
559 ArrayType *VarName##Ty = OMPBuilder.VarName##Ty; \
561 PointerType *VarName##PtrTy = OMPBuilder.VarName##PtrTy; \
562 (void)VarName##PtrTy;
564#define OMP_FUNCTION_TYPE(VarName, ...) \
565 FunctionType *VarName = OMPBuilder.VarName; \
567 PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr; \
570#define OMP_STRUCT_TYPE(VarName, ...) \
571 StructType *VarName = OMPBuilder.VarName; \
573 PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr; \
576#define OMP_RTL(_Enum, _Name, _IsVarArg, _ReturnType, ...) \
578 SmallVector<Type *, 8> ArgsTypes({__VA_ARGS__}); \
579 Function *F = M.getFunction(_Name); \
580 RTLFunctions.insert(F); \
581 if (declMatchesRTFTypes(F, OMPBuilder._ReturnType, ArgsTypes)) { \
582 RuntimeFunctionIDMap[F] = _Enum; \
583 auto &RFI = RFIs[_Enum]; \
586 RFI.IsVarArg = _IsVarArg; \
587 RFI.ReturnType = OMPBuilder._ReturnType; \
588 RFI.ArgumentTypes = std::move(ArgsTypes); \
589 RFI.Declaration = F; \
590 unsigned NumUses = collectUses(RFI); \
593 dbgs() << TAG << RFI.Name << (RFI.Declaration ? "" : " not") \
595 if (RFI.Declaration) \
596 dbgs() << TAG << "-> got " << NumUses << " uses in " \
597 << RFI.getNumFunctionsWithUses() \
598 << " different functions.\n"; \
603#define OMP_RTL_CB_INFO(_Enum, _Name, _ArgNo, _ArgIndices, _IsVarArg) \
604 setCallbackMetadata(M.getFunction(_Name), _ArgNo, _ArgIndices, _IsVarArg);
606#include "llvm/Frontend/OpenMP/OMPKinds.def"
612 for (StringRef Prefix : {
"__kmpc",
"_ZN4ompx",
"omp_"})
613 if (
F.hasFnAttribute(Attribute::NoInline) &&
614 F.getName().starts_with(Prefix) &&
615 !
F.hasFnAttribute(Attribute::OptimizeNone))
616 F.removeFnAttr(Attribute::NoInline);
624 DenseSet<const Function *> RTLFunctions;
627 bool OpenMPPostLink =
false;
634 SmallPtrSet<Function *, 8> SPMDizedKernels;
637template <
typename Ty,
bool InsertInval
idates = true>
639 bool contains(
const Ty &Elem)
const {
return Set.contains(Elem); }
640 bool insert(
const Ty &Elem) {
641 if (InsertInvalidates)
642 BooleanState::indicatePessimisticFixpoint();
643 return Set.insert(Elem);
646 const Ty &operator[](
int Idx)
const {
return Set[
Idx]; }
647 bool operator==(
const BooleanStateWithSetVector &
RHS)
const {
648 return BooleanState::operator==(
RHS) && Set ==
RHS.Set;
650 bool operator!=(
const BooleanStateWithSetVector &
RHS)
const {
651 return !(*
this ==
RHS);
654 bool empty()
const {
return Set.empty(); }
655 size_t size()
const {
return Set.size(); }
658 BooleanStateWithSetVector &
operator^=(
const BooleanStateWithSetVector &
RHS) {
659 BooleanState::operator^=(
RHS);
660 Set.insert_range(
RHS.Set);
669 typename decltype(Set)::iterator
begin() {
return Set.begin(); }
670 typename decltype(Set)::iterator
end() {
return Set.end(); }
671 typename decltype(Set)::const_iterator
begin()
const {
return Set.begin(); }
672 typename decltype(Set)::const_iterator
end()
const {
return Set.end(); }
675template <
typename Ty,
bool InsertInval
idates = true>
676using BooleanStateWithPtrSetVector =
677 BooleanStateWithSetVector<Ty *, InsertInvalidates>;
681 bool IsAtFixpoint =
false;
685 BooleanStateWithPtrSetVector<CallBase,
false>
686 ReachedKnownParallelRegions;
689 BooleanStateWithPtrSetVector<CallBase> ReachedUnknownParallelRegions;
694 BooleanStateWithPtrSetVector<Instruction, false> SPMDCompatibilityTracker;
698 CallBase *KernelInitCB =
nullptr;
702 ConstantStruct *KernelEnvC =
nullptr;
706 CallBase *KernelDeinitCB =
nullptr;
709 bool IsKernelEntry =
false;
712 BooleanStateWithPtrSetVector<Function, false> ReachingKernelEntries;
717 BooleanStateWithSetVector<uint8_t> ParallelLevels;
720 bool NestedParallelism =
false;
725 KernelInfoState() =
default;
726 KernelInfoState(
bool BestState) {
728 indicatePessimisticFixpoint();
731 bool isValidState()
const override {
return true; }
733 bool isAtFixpoint()
const override {
return IsAtFixpoint; }
737 ParallelLevels.indicatePessimisticFixpoint();
738 ReachingKernelEntries.indicatePessimisticFixpoint();
739 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
740 ReachedKnownParallelRegions.indicatePessimisticFixpoint();
741 ReachedUnknownParallelRegions.indicatePessimisticFixpoint();
742 NestedParallelism =
true;
743 return ChangeStatus::CHANGED;
748 ParallelLevels.indicateOptimisticFixpoint();
749 ReachingKernelEntries.indicateOptimisticFixpoint();
750 SPMDCompatibilityTracker.indicateOptimisticFixpoint();
751 ReachedKnownParallelRegions.indicateOptimisticFixpoint();
752 ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
753 return ChangeStatus::UNCHANGED;
757 KernelInfoState &getAssumed() {
return *
this; }
758 const KernelInfoState &getAssumed()
const {
return *
this; }
761 if (SPMDCompatibilityTracker !=
RHS.SPMDCompatibilityTracker)
763 if (ReachedKnownParallelRegions !=
RHS.ReachedKnownParallelRegions)
765 if (ReachedUnknownParallelRegions !=
RHS.ReachedUnknownParallelRegions)
767 if (ReachingKernelEntries !=
RHS.ReachingKernelEntries)
769 if (ParallelLevels !=
RHS.ParallelLevels)
771 if (NestedParallelism !=
RHS.NestedParallelism)
777 bool mayContainParallelRegion() {
778 return !ReachedKnownParallelRegions.empty() ||
779 !ReachedUnknownParallelRegions.empty();
783 static KernelInfoState getBestState() {
return KernelInfoState(
true); }
785 static KernelInfoState getBestState(KernelInfoState &KIS) {
786 return getBestState();
790 static KernelInfoState getWorstState() {
return KernelInfoState(
false); }
793 KernelInfoState
operator^=(
const KernelInfoState &KIS) {
795 if (KIS.KernelInitCB) {
796 if (KernelInitCB && KernelInitCB != KIS.KernelInitCB)
799 KernelInitCB = KIS.KernelInitCB;
801 if (KIS.KernelDeinitCB) {
802 if (KernelDeinitCB && KernelDeinitCB != KIS.KernelDeinitCB)
805 KernelDeinitCB = KIS.KernelDeinitCB;
807 if (KIS.KernelEnvC) {
808 if (KernelEnvC && KernelEnvC != KIS.KernelEnvC)
811 KernelEnvC = KIS.KernelEnvC;
813 SPMDCompatibilityTracker ^= KIS.SPMDCompatibilityTracker;
814 ReachedKnownParallelRegions ^= KIS.ReachedKnownParallelRegions;
815 ReachedUnknownParallelRegions ^= KIS.ReachedUnknownParallelRegions;
816 NestedParallelism |= KIS.NestedParallelism;
820 KernelInfoState
operator&=(
const KernelInfoState &KIS) {
821 return (*
this ^= KIS);
831 AllocaInst *Array =
nullptr;
833 SmallVector<Value *, 8> StoredValues;
835 SmallVector<StoreInst *, 8> LastAccesses;
837 OffloadArray() =
default;
843 bool initialize(AllocaInst &Array, Instruction &Before) {
844 if (!getValues(Array, Before))
847 this->Array = &Array;
851 static const unsigned DeviceIDArgNum = 1;
852 static const unsigned BasePtrsArgNum = 3;
853 static const unsigned PtrsArgNum = 4;
854 static const unsigned SizesArgNum = 5;
860 bool getValues(AllocaInst &Array, Instruction &Before) {
862 const DataLayout &
DL = Array.getDataLayout();
863 std::optional<TypeSize> ArraySize = Array.getAllocationSize(
DL);
864 if (!ArraySize || !ArraySize->isFixed())
868 StoredValues.assign(NumValues,
nullptr);
869 LastAccesses.assign(NumValues,
nullptr);
877 for (Instruction &
I : *BB) {
893 LastAccesses[
Idx] = S;
904 const unsigned NumValues = StoredValues.size();
905 for (
unsigned I = 0;
I < NumValues; ++
I) {
906 if (!StoredValues[
I] || !LastAccesses[
I])
917static std::optional<uint64_t>
919 unsigned CallbackOpNo) {
920 std::optional<uint64_t> EntryCount;
926 if (std::optional<uint64_t> EC =
Callback->getEntryCount())
929 EntryCount = EntryCount ? std::max(*EntryCount, *EC) : *
EC;
934static bool moduleHasSampleProfile(
const Module &M) {
935 std::unique_ptr<ProfileSummary> Summary(
942 using OptimizationRemarkGetter =
943 function_ref<OptimizationRemarkEmitter &(
Function *)>;
945 OpenMPOpt(SmallVectorImpl<Function *> &SCC, CallGraphUpdater &CGUpdater,
946 OptimizationRemarkGetter OREGetter,
947 OMPInformationCache &OMPInfoCache, Attributor &A)
948 : M(*(*SCC.
begin())->
getParent()), SCC(SCC), CGUpdater(CGUpdater),
949 OREGetter(OREGetter), OMPInfoCache(OMPInfoCache), A(A) {}
952 bool remarksEnabled() {
953 auto &Ctx = M.getContext();
958 bool run(
bool IsModulePass) {
968 Changed |= runAttributor(IsModulePass);
971 OMPInfoCache.recollectUses();
974 Changed |= rewriteDeviceCodeStateMachine();
980 Changed |= removeSPMDParallelWrappers();
982 if (remarksEnabled())
983 analysisGlobalization();
985 if (A.Opts.openmp_print_icv_values)
987 if (A.Opts.openmp_print_gpu_kernels)
990 Changed |= runAttributor(IsModulePass);
993 OMPInfoCache.recollectUses();
995 Changed |= deleteParallelRegions();
997 if (A.Opts.openmp_hide_memory_transfer_latency)
998 Changed |= hideMemTransfersLatency();
999 Changed |= deduplicateRuntimeCalls();
1000 if (A.Opts.openmp_opt_enable_merging) {
1001 if (mergeParallelRegions()) {
1002 deduplicateRuntimeCalls();
1008 if (OMPInfoCache.OpenMPPostLink)
1009 Changed |= removeRuntimeSymbols();
1016 void printICVs()
const {
1021 for (
auto ICV : ICVs) {
1022 auto ICVInfo = OMPInfoCache.ICVs[ICV];
1023 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
1024 return ORA <<
"OpenMP ICV " <<
ore::NV(
"OpenMPICV", ICVInfo.Name)
1026 << (ICVInfo.InitValue
1027 ?
toString(ICVInfo.InitValue->getValue(), 10,
true)
1028 :
"IMPLEMENTATION_DEFINED");
1037 void printKernels()
const {
1042 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
1043 return ORA <<
"OpenMP GPU kernel "
1044 <<
ore::NV(
"OpenMPGPUKernel",
F->getName()) <<
"\n";
1053 static CallInst *getCallIfRegularCall(
1054 Use &U, OMPInformationCache::RuntimeFunctionInfo *RFI =
nullptr) {
1065 static CallInst *getCallIfRegularCall(
1066 Value &
V, OMPInformationCache::RuntimeFunctionInfo *RFI =
nullptr) {
1077 bool mergeParallelRegions() {
1078 const unsigned CallbackCalleeOperand = 2;
1079 const unsigned CallbackFirstArgOperand = 3;
1083 OMPInformationCache::RuntimeFunctionInfo &RFI =
1084 OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
1086 if (!RFI.Declaration)
1090 OMPInformationCache::RuntimeFunctionInfo UnmergableCallsInfo[] = {
1091 OMPInfoCache.RFIs[OMPRTL___kmpc_push_proc_bind],
1092 OMPInfoCache.RFIs[OMPRTL___kmpc_push_num_threads],
1096 LoopInfo *LI =
nullptr;
1097 DominatorTree *DT =
nullptr;
1099 SmallDenseMap<BasicBlock *, SmallPtrSet<Instruction *, 4>> BB2PRMap;
1101 BasicBlock *StartBB =
nullptr, *EndBB =
nullptr;
1102 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1104 BasicBlock *CGStartBB = CodeGenIP.getNodeParent();
1106 assert(StartBB !=
nullptr &&
"StartBB should not be null");
1108 assert(EndBB !=
nullptr &&
"EndBB should not be null");
1109 EndBB->getTerminator()->setSuccessor(0, CGEndBB);
1113 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
Value &,
1114 Value &Inner,
Value *&ReplacementValue) -> InsertPointTy {
1115 ReplacementValue = &Inner;
1119 auto FiniCB = [&](InsertPointTy CodeGenIP) {
return Error::success(); };
1123 auto CreateSequentialRegion = [&](
Function *OuterFn,
1129 BasicBlock *ParentBB = SeqStartI->getParent();
1131 SplitBlock(ParentBB, SeqEndI->getNextNode(), DT, LI);
1135 SplitBlock(ParentBB, SeqStartI, DT, LI,
nullptr,
"seq.par.merged");
1138 "Expected a different CFG");
1142 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1144 BasicBlock *CGStartBB = CodeGenIP.getNodeParent();
1146 assert(SeqStartBB !=
nullptr &&
"SeqStartBB should not be null");
1148 assert(SeqEndBB !=
nullptr &&
"SeqEndBB should not be null");
1152 auto FiniCB = [&](InsertPointTy CodeGenIP) {
return Error::success(); };
1156 for (Instruction &
I : *SeqStartBB) {
1157 SmallPtrSet<Instruction *, 4> OutsideUsers;
1158 for (User *Usr :
I.users()) {
1166 OutsideUsers.
insert(&UsrI);
1169 if (OutsideUsers.
empty())
1174 const DataLayout &
DL = M.getDataLayout();
1175 AllocaInst *AllocaI =
new AllocaInst(
1176 I.getType(),
DL.getAllocaAddrSpace(),
nullptr,
1181 new StoreInst(&
I, AllocaI, SeqStartBB->getTerminator()->getIterator());
1185 for (Instruction *UsrI : OutsideUsers) {
1186 LoadInst *LoadI =
new LoadInst(
I.getType(), AllocaI,
1187 I.getName() +
".seq.output.load",
1193 OpenMPIRBuilder::LocationDescription Loc(ParentBB->
end(),
DL);
1195 OMPInfoCache.OMPBuilder.createMaster(Loc, BodyGenCB, FiniCB));
1196 cantFail(OMPInfoCache.OMPBuilder.createBarrier({SeqAfterIP, DL},
1212 auto Merge = [&](
const SmallVectorImpl<CallInst *> &MergableCIs,
1216 assert(MergableCIs.
size() > 1 &&
"Assumed multiple mergable CIs");
1218 auto Remark = [&](OptimizationRemark
OR) {
1219 OR <<
"Parallel region merged with parallel region"
1220 << (MergableCIs.
size() > 2 ?
"s" :
"") <<
" at ";
1223 if (CI != MergableCIs.
back())
1231 Function *OriginalFn = BB->getParent();
1233 <<
" parallel regions in " << OriginalFn->
getName()
1237 EndBB =
SplitBlock(BB, MergableCIs.
back()->getNextNode(), DT, LI);
1239 SplitBlock(EndBB, &*EndBB->getFirstInsertionPt(), DT, LI);
1243 assert(BB->getUniqueSuccessor() == StartBB &&
"Expected a different CFG");
1244 const DebugLoc DL = BB->getTerminator()->getDebugLoc();
1249 for (
auto *It = MergableCIs.
begin(), *
End = MergableCIs.
end() - 1;
1258 CreateSequentialRegion(OriginalFn, BB, ForkCI->
getNextNode(),
1262 OpenMPIRBuilder::LocationDescription Loc(BB->end(),
DL);
1263 IRBuilder<>::InsertPoint AllocaIP(
1268 cantFail(OMPInfoCache.OMPBuilder.createParallel(
1269 Loc, AllocaIP, {}, BodyGenCB, PrivCB, FiniCB,
1270 nullptr,
nullptr, OMP_PROC_BIND_default,
1275 OMPInfoCache.OMPBuilder.finalize(OriginalFn);
1278 std::optional<uint64_t> WrapperCount =
1279 getMergedWrapperEntryCount(MergableCIs, CallbackCalleeOperand);
1285 const bool SampleProfile =
1286 moduleHasSampleProfile(*OriginalFn->
getParent());
1290 SmallVector<Value *, 8>
Args;
1291 for (
auto *CI : MergableCIs) {
1293 FunctionType *FT = OMPInfoCache.OMPBuilder.ParallelTask;
1297 for (
unsigned U = CallbackFirstArgOperand,
E = CI->
arg_size(); U <
E;
1307 if (WrapperCount && SampleProfile) {
1313 for (
unsigned U = CallbackFirstArgOperand,
E = CI->
arg_size(); U <
E;
1317 U - (CallbackFirstArgOperand - CallbackCalleeOperand), A);
1320 if (CI != MergableCIs.back()) {
1323 cantFail(OMPInfoCache.OMPBuilder.createBarrier(
1324 {NewCI->getNextNode()->getIterator(), NewCI->getDebugLoc()},
1331 assert(OutlinedFn != OriginalFn &&
"Outlining failed");
1332 CGUpdater.registerOutlinedFunction(*OriginalFn, *OutlinedFn);
1333 CGUpdater.reanalyzeFunction(*OriginalFn);
1335 NumOpenMPParallelRegionsMerged += MergableCIs.size();
1343 CallInst *CI = getCallIfRegularCall(U, &RFI);
1350 RFI.foreachUse(SCC, DetectPRsCB);
1356 for (
auto &It : BB2PRMap) {
1357 auto &CIs = It.getSecond();
1372 auto IsMergable = [&](
Instruction &
I,
bool IsBeforeMergableRegion) {
1375 if (
I.isTerminator())
1382 if (IsBeforeMergableRegion) {
1384 if (!CalledFunction)
1391 for (
const auto &RFI : UnmergableCallsInfo) {
1392 if (CalledFunction == RFI.Declaration)
1411 if (CIs.count(&
I)) {
1417 if (IsMergable(
I, MergableCIs.
empty()))
1422 for (; It !=
End; ++It) {
1424 if (CIs.count(&SkipI)) {
1426 <<
" due to " <<
I <<
"\n");
1433 if (MergableCIs.
size() > 1) {
1434 MergableCIsVector.
push_back(MergableCIs);
1436 <<
" parallel regions in block " << BB->
getName()
1441 MergableCIs.
clear();
1444 if (!MergableCIsVector.
empty()) {
1447 for (
auto &MergableCIs : MergableCIsVector)
1448 Merge(MergableCIs, BB);
1449 MergableCIsVector.clear();
1456 OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_fork_call);
1457 OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_barrier);
1458 OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_master);
1459 OMPInfoCache.recollectUsesForFunction(OMPRTL___kmpc_end_master);
1466 bool deleteParallelRegions() {
1467 const unsigned CallbackCalleeOperand = 2;
1469 OMPInformationCache::RuntimeFunctionInfo &RFI =
1470 OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
1472 if (!RFI.Declaration)
1477 CallInst *CI = getCallIfRegularCall(U);
1484 if (!Fn->onlyReadsMemory())
1486 if (!Fn->hasFnAttribute(Attribute::WillReturn))
1492 auto Remark = [&](OptimizationRemark
OR) {
1493 return OR <<
"Removing parallel region with no side-effects.";
1499 ++NumOpenMPParallelRegionsDeleted;
1503 RFI.foreachUse(SCC, DeleteCallCB);
1509 bool deduplicateRuntimeCalls() {
1513 OMPRTL_omp_get_num_threads,
1514 OMPRTL_omp_in_parallel,
1515 OMPRTL_omp_get_cancellation,
1516 OMPRTL_omp_get_supported_active_levels,
1517 OMPRTL_omp_get_level,
1518 OMPRTL_omp_get_ancestor_thread_num,
1519 OMPRTL_omp_get_team_size,
1520 OMPRTL_omp_get_active_level,
1521 OMPRTL_omp_in_final,
1522 OMPRTL_omp_get_proc_bind,
1523 OMPRTL_omp_get_num_places,
1524 OMPRTL_omp_get_num_procs,
1525 OMPRTL_omp_get_place_num,
1526 OMPRTL_omp_get_partition_num_places,
1527 OMPRTL_omp_get_partition_place_nums};
1530 SmallSetVector<Value *, 16> GTIdArgs;
1531 collectGlobalThreadIdArguments(GTIdArgs);
1533 <<
" global thread ID arguments\n");
1536 for (
auto DeduplicableRuntimeCallID : DeduplicableRuntimeCallIDs)
1537 Changed |= deduplicateRuntimeCalls(
1538 *
F, OMPInfoCache.RFIs[DeduplicableRuntimeCallID]);
1542 Value *GTIdArg =
nullptr;
1543 for (Argument &Arg :
F->args())
1544 if (GTIdArgs.
count(&Arg)) {
1548 Changed |= deduplicateRuntimeCalls(
1549 *
F, OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num], GTIdArg);
1556 bool removeRuntimeSymbols() {
1561 if (GlobalVariable *GV = M.getNamedGlobal(
"__llvm_rpc_client")) {
1562 if (GV->hasNUsesOrMore(1))
1566 GV->eraseFromParent();
1578 bool hideMemTransfersLatency() {
1579 auto &RFI = OMPInfoCache.RFIs[OMPRTL___tgt_target_data_begin_mapper];
1582 auto *RTCall = getCallIfRegularCall(U, &RFI);
1586 OffloadArray OffloadArrays[3];
1587 if (!getValuesInOffloadArrays(*RTCall, OffloadArrays))
1590 LLVM_DEBUG(dumpValuesInOffloadArrays(OffloadArrays));
1593 bool WasSplit =
false;
1594 Instruction *WaitMovementPoint = canBeMovedDownwards(*RTCall);
1595 if (WaitMovementPoint)
1596 WasSplit = splitTargetDataBeginRTC(*RTCall, *WaitMovementPoint);
1601 if (OMPInfoCache.runtimeFnsAvailable(
1602 {OMPRTL___tgt_target_data_begin_mapper_issue,
1603 OMPRTL___tgt_target_data_begin_mapper_wait}))
1604 RFI.foreachUse(SCC, SplitMemTransfers);
1609 void analysisGlobalization() {
1610 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
1612 auto CheckGlobalization = [&](
Use &
U,
Function &Decl) {
1613 if (CallInst *CI = getCallIfRegularCall(U, &RFI)) {
1614 auto Remark = [&](OptimizationRemarkMissed ORM) {
1616 <<
"Found thread data sharing on the GPU. "
1617 <<
"Expect degraded performance due to data globalization.";
1625 RFI.foreachUse(SCC, CheckGlobalization);
1630 bool getValuesInOffloadArrays(CallInst &RuntimeCall,
1632 assert(OAs.
size() == 3 &&
"Need space for three offload arrays!");
1642 Value *BasePtrsArg =
1654 if (!OAs[0].
initialize(*BasePtrsArray, RuntimeCall))
1662 if (!OAs[1].
initialize(*PtrsArray, RuntimeCall))
1674 if (!OAs[2].
initialize(*SizesArray, RuntimeCall))
1685 assert(OAs.
size() == 3 &&
"There are three offload arrays to debug!");
1688 std::string ValuesStr;
1689 raw_string_ostream
Printer(ValuesStr);
1690 std::string Separator =
" --- ";
1692 for (
auto *BP : OAs[0].StoredValues) {
1696 LLVM_DEBUG(
dbgs() <<
"\t\toffload_baseptrs: " << ValuesStr <<
"\n");
1699 for (
auto *
P : OAs[1].StoredValues) {
1706 for (
auto *S : OAs[2].StoredValues) {
1710 LLVM_DEBUG(
dbgs() <<
"\t\toffload_sizes: " << ValuesStr <<
"\n");
1715 Instruction *canBeMovedDownwards(CallInst &RuntimeCall) {
1720 bool IsWorthIt =
false;
1739 return RuntimeCall.
getParent()->getTerminator();
1743 bool splitTargetDataBeginRTC(CallInst &RuntimeCall,
1744 Instruction &WaitMovementPoint) {
1748 auto &
IRBuilder = OMPInfoCache.OMPBuilder;
1751 IRBuilder.Builder.SetInsertPoint(
Entry.getFirstNonPHIOrAlloca());
1753 IRBuilder.AsyncInfo,
nullptr,
"handle");
1760 FunctionCallee IssueDecl =
IRBuilder.getOrCreateRuntimeFunction(
1761 M, OMPRTL___tgt_target_data_begin_mapper_issue);
1764 SmallVector<Value *, 16>
Args;
1765 for (
auto &Arg : RuntimeCall.
args())
1766 Args.push_back(Arg.get());
1767 Args.push_back(Handle);
1771 OMPInfoCache.setCallingConvention(IssueDecl, IssueCallsite);
1776 FunctionCallee WaitDecl =
IRBuilder.getOrCreateRuntimeFunction(
1777 M, OMPRTL___tgt_target_data_begin_mapper_wait);
1779 Value *WaitParams[2] = {
1781 OffloadArray::DeviceIDArgNum),
1785 WaitDecl, WaitParams,
"", WaitMovementPoint.
getIterator());
1786 OMPInfoCache.setCallingConvention(WaitDecl, WaitCallsite);
1791 static Value *combinedIdentStruct(
Value *CurrentIdent,
Value *NextIdent,
1792 bool GlobalOnly,
bool &SingleChoice) {
1793 if (CurrentIdent == NextIdent)
1794 return CurrentIdent;
1799 SingleChoice = !CurrentIdent;
1811 getCombinedIdentFromCallUsesIn(OMPInformationCache::RuntimeFunctionInfo &RFI,
1813 bool SingleChoice =
true;
1814 Value *Ident =
nullptr;
1816 CallInst *CI = getCallIfRegularCall(U, &RFI);
1817 if (!CI || &
F != &Caller)
1820 true, SingleChoice);
1823 RFI.foreachUse(SCC, CombineIdentStruct);
1825 if (!Ident || !SingleChoice) {
1829 if (!OMPInfoCache.OMPBuilder.getInsertionPoint().isValid())
1830 OMPInfoCache.OMPBuilder.updateToLocation(
1831 {
F.getEntryBlock().begin(),
DebugLoc()});
1834 uint32_t SrcLocStrSize;
1836 OMPInfoCache.OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1837 Ident = OMPInfoCache.OMPBuilder.getOrCreateIdent(Loc, SrcLocStrSize);
1844 bool deduplicateRuntimeCalls(
Function &
F,
1845 OMPInformationCache::RuntimeFunctionInfo &RFI,
1846 Value *ReplVal =
nullptr) {
1847 auto *UV = RFI.getUseVector(
F);
1848 if (!UV || UV->size() + (ReplVal !=
nullptr) < 2)
1852 dbgs() <<
TAG <<
"Deduplicate " << UV->size() <<
" uses of " << RFI.Name
1853 << (ReplVal ?
" with an existing value\n" :
"\n") <<
"\n");
1857 "Unexpected replacement value!");
1860 auto CanBeMoved = [
this](CallBase &CB) {
1866 for (
unsigned U = 1;
U < NumArgs; ++
U)
1874 OMPInfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
F);
1878 for (Use *U : *UV) {
1879 if (CallInst *CI = getCallIfRegularCall(*U, &RFI)) {
1884 if (!CanBeMoved(*CI))
1892 assert(IP &&
"Expected insertion point!");
1902 Value *Ident = getCombinedIdentFromCallUsesIn(RFI,
F,
1910 CallInst *CI = getCallIfRegularCall(U, &RFI);
1911 if (!CI || CI == ReplVal || &
F != &Caller)
1915 auto Remark = [&](OptimizationRemark
OR) {
1916 return OR <<
"OpenMP runtime call "
1917 <<
ore::NV(
"OpenMPOptRuntime", RFI.Name) <<
" deduplicated.";
1926 ++NumOpenMPRuntimeCallsDeduplicated;
1930 RFI.foreachUse(SCC, ReplaceAndDeleteCB);
1936 void collectGlobalThreadIdArguments(SmallSetVector<Value *, 16> >IdArgs) {
1943 auto CallArgOpIsGTId = [&](
Function &
F,
unsigned ArgNo, CallInst &RefCI) {
1944 if (!
F.hasLocalLinkage())
1946 for (Use &U :
F.uses()) {
1947 if (CallInst *CI = getCallIfRegularCall(U)) {
1949 if (CI == &RefCI || GTIdArgs.
count(ArgOp) ||
1950 getCallIfRegularCall(
1951 *ArgOp, &OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num]))
1960 auto AddUserArgs = [&](
Value >Id) {
1961 for (Use &U : GTId.uses())
1965 if (CallArgOpIsGTId(*Callee,
U.getOperandNo(), *CI))
1970 OMPInformationCache::RuntimeFunctionInfo &GlobThreadNumRFI =
1971 OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num];
1973 GlobThreadNumRFI.foreachUse(SCC, [&](Use &U,
Function &
F) {
1974 if (CallInst *CI = getCallIfRegularCall(U, &GlobThreadNumRFI))
1982 for (
unsigned U = 0;
U < GTIdArgs.
size(); ++
U)
1983 AddUserArgs(*GTIdArgs[U]);
1991 DenseMap<Function *, std::optional<Kernel>> UniqueKernelMap;
1997 Kernel getUniqueKernelFor(Instruction &
I) {
1998 return getUniqueKernelFor(*
I.getFunction());
2003 bool rewriteDeviceCodeStateMachine();
2008 bool removeSPMDParallelWrappers();
2024 template <
typename RemarkKind,
typename RemarkCallBack>
2025 void emitRemark(Instruction *
I, StringRef RemarkName,
2026 RemarkCallBack &&RemarkCB)
const {
2028 auto &ORE = OREGetter(
F);
2032 return RemarkCB(RemarkKind(
DEBUG_TYPE, RemarkName,
I))
2033 <<
" [" << RemarkName <<
"]";
2037 [&]() {
return RemarkCB(RemarkKind(
DEBUG_TYPE, RemarkName,
I)); });
2041 template <
typename RemarkKind,
typename RemarkCallBack>
2043 RemarkCallBack &&RemarkCB)
const {
2044 auto &ORE = OREGetter(
F);
2048 return RemarkCB(RemarkKind(
DEBUG_TYPE, RemarkName,
F))
2049 <<
" [" << RemarkName <<
"]";
2053 [&]() {
return RemarkCB(RemarkKind(
DEBUG_TYPE, RemarkName,
F)); });
2060 SmallVectorImpl<Function *> &SCC;
2064 CallGraphUpdater &CGUpdater;
2067 OptimizationRemarkGetter OREGetter;
2070 OMPInformationCache &OMPInfoCache;
2076 bool runAttributor(
bool IsModulePass) {
2080 registerAAs(IsModulePass);
2085 <<
" functions, result: " <<
Changed <<
".\n");
2087 if (
Changed == ChangeStatus::CHANGED)
2088 OMPInfoCache.invalidateAnalyses();
2090 return Changed == ChangeStatus::CHANGED;
2097 void registerAAs(
bool IsModulePass);
2102 static void registerAAsForFunction(Attributor &A,
const Function &
F);
2106 if (OMPInfoCache.CGSCC && !OMPInfoCache.CGSCC->empty() &&
2107 !OMPInfoCache.CGSCC->contains(&
F))
2112 std::optional<Kernel> &CachedKernel = UniqueKernelMap[&
F];
2114 return *CachedKernel;
2121 return *CachedKernel;
2124 CachedKernel =
nullptr;
2125 if (!
F.hasLocalLinkage()) {
2128 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
2129 return ORA <<
"Potentially unknown OpenMP target region caller.";
2137 auto GetUniqueKernelForUse = [&](
const Use &
U) ->
Kernel {
2140 if (
Cmp->isEquality())
2141 return getUniqueKernelFor(*Cmp);
2147 return getUniqueKernelFor(*CB);
2149 OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
2150 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
2152 if (OpenMPOpt::getCallIfRegularCall(*
U.getUser(), &KernelParallelRFI))
2153 return getUniqueKernelFor(*CB);
2161 SmallPtrSet<Kernel, 2> PotentialKernels;
2162 OMPInformationCache::foreachUse(
F, [&](
const Use &U) {
2163 PotentialKernels.
insert(GetUniqueKernelForUse(U));
2167 if (PotentialKernels.
size() == 1)
2168 K = *PotentialKernels.
begin();
2171 UniqueKernelMap[&
F] =
K;
2176bool OpenMPOpt::rewriteDeviceCodeStateMachine() {
2177 OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
2178 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
2181 if (!KernelParallelRFI)
2185 if (!
A.Opts.openmp_opt_state_machine_rewrite)
2192 bool UnknownUse =
false;
2193 bool KernelParallelUse =
false;
2194 unsigned NumDirectCalls = 0;
2197 OMPInformationCache::foreachUse(*
F, [&](Use &U) {
2205 ToBeReplacedStateMachineUses.
push_back(&U);
2211 OpenMPOpt::getCallIfRegularCall(*
U.getUser(), &KernelParallelRFI);
2212 const unsigned int WrapperFunctionArgNo = 6;
2213 if (!KernelParallelUse && CI &&
2215 KernelParallelUse =
true;
2216 ToBeReplacedStateMachineUses.
push_back(&U);
2224 if (!KernelParallelUse)
2230 if (UnknownUse || NumDirectCalls != 1 ||
2231 ToBeReplacedStateMachineUses.
size() > 2) {
2232 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
2233 return ORA <<
"Parallel region is used in "
2234 << (UnknownUse ?
"unknown" :
"unexpected")
2235 <<
" ways. Will not attempt to rewrite the state machine.";
2245 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
2246 return ORA <<
"Parallel region is not called from a unique kernel. "
2247 "Will not attempt to rewrite the state machine.";
2259 Type *Int8Ty = Type::getInt8Ty(
M.getContext());
2261 auto *
ID =
new GlobalVariable(
2265 for (Use *U : ToBeReplacedStateMachineUses)
2267 ID,
U->get()->getType()));
2269 ++NumOpenMPParallelRegionsReplacedInGPUStateMachine;
2277bool OpenMPOpt::removeSPMDParallelWrappers() {
2279 if (OMPInfoCache.SPMDizedKernels.empty())
2282 OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
2283 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
2284 if (!KernelParallelRFI || !KernelParallelRFI.Declaration)
2287 constexpr unsigned WrapperFunctionArgNo = 6;
2289 for (User *U : KernelParallelRFI.Declaration->
users()) {
2292 CI->
arg_size() <= WrapperFunctionArgNo)
2306 if (!K || !OMPInfoCache.SPMDizedKernels.contains(K))
2310 WrapperFunctionArgNo,
2319struct AAICVTracker :
public StateWrapper<BooleanState, AbstractAttribute> {
2320 using Base = StateWrapper<BooleanState, AbstractAttribute>;
2321 AAICVTracker(
const IRPosition &IRP, Attributor &
A) :
Base(IRP) {}
2324 bool isAssumedTracked()
const {
return getAssumed(); }
2327 bool isKnownTracked()
const {
return getAssumed(); }
2330 static AAICVTracker &createForPosition(
const IRPosition &IRP, Attributor &
A);
2334 const Instruction *
I,
2335 Attributor &
A)
const {
2336 return std::nullopt;
2342 virtual std::optional<Value *>
2349 StringRef
getName()
const override {
return "AAICVTracker"; }
2351 const char *getIdAddr()
const override {
return &ID; }
2354 static bool classof(
const AbstractAttribute *AA) {
2358 static const char ID;
2361struct AAICVTrackerFunction :
public AAICVTracker {
2362 AAICVTrackerFunction(
const IRPosition &IRP, Attributor &
A)
2363 : AAICVTracker(IRP,
A) {}
2366 const std::string getAsStr(Attributor *)
const override {
2367 return "ICVTrackerFunction";
2371 void trackStatistics()
const override {}
2375 return ChangeStatus::UNCHANGED;
2380 InternalControlVar::ICV___last>
2381 ICVReplacementValuesMap;
2388 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
2391 auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
2393 auto &ValuesMap = ICVReplacementValuesMap[ICV];
2395 CallInst *CI = OpenMPOpt::getCallIfRegularCall(U);
2401 if (ValuesMap.insert(std::make_pair(CI, CI->
getArgOperand(0))).second)
2402 HasChanged = ChangeStatus::CHANGED;
2408 std::optional<Value *> ReplVal = getValueForCall(
A,
I, ICV);
2409 if (ReplVal && ValuesMap.insert(std::make_pair(&
I, *ReplVal)).second)
2410 HasChanged = ChangeStatus::CHANGED;
2416 SetterRFI.foreachUse(TrackValues,
F);
2418 bool UsedAssumedInformation =
false;
2419 A.checkForAllInstructions(CallCheck, *
this, {Instruction::Call},
2420 UsedAssumedInformation,
2426 if (HasChanged == ChangeStatus::CHANGED)
2427 ValuesMap.try_emplace(Entry);
2435 std::optional<Value *> getValueForCall(Attributor &
A,
const Instruction &
I,
2439 if (!CB || CB->
hasFnAttr(
"no_openmp") ||
2442 return std::nullopt;
2444 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
2445 auto &GetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Getter];
2446 auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
2450 if (CalledFunction ==
nullptr)
2452 if (CalledFunction == GetterRFI.Declaration)
2453 return std::nullopt;
2454 if (CalledFunction == SetterRFI.Declaration) {
2455 if (ICVReplacementValuesMap[ICV].
count(&
I))
2456 return ICVReplacementValuesMap[ICV].lookup(&
I);
2465 const auto *ICVTrackingAA =
A.getAAFor<AAICVTracker>(
2468 if (ICVTrackingAA->isAssumedTracked()) {
2469 std::optional<Value *> URV =
2470 ICVTrackingAA->getUniqueReplacementValue(ICV);
2481 std::optional<Value *>
2483 return std::nullopt;
2488 const Instruction *
I,
2489 Attributor &
A)
const override {
2490 const auto &ValuesMap = ICVReplacementValuesMap[ICV];
2491 if (ValuesMap.count(
I))
2492 return ValuesMap.lookup(
I);
2495 SmallPtrSet<const Instruction *, 16> Visited;
2498 std::optional<Value *> ReplVal;
2500 while (!Worklist.
empty()) {
2502 if (!Visited.
insert(CurrInst).second)
2510 if (ValuesMap.count(CurrInst)) {
2511 std::optional<Value *> NewReplVal = ValuesMap.lookup(CurrInst);
2514 ReplVal = NewReplVal;
2520 if (ReplVal != NewReplVal)
2526 std::optional<Value *> NewReplVal = getValueForCall(
A, *CurrInst, ICV);
2532 ReplVal = NewReplVal;
2538 if (ReplVal != NewReplVal)
2543 if (CurrBB ==
I->getParent() && ReplVal)
2555struct AAICVTrackerFunctionReturned : AAICVTracker {
2556 AAICVTrackerFunctionReturned(
const IRPosition &IRP, Attributor &
A)
2557 : AAICVTracker(IRP,
A) {}
2560 const std::string getAsStr(Attributor *)
const override {
2561 return "ICVTrackerFunctionReturned";
2565 void trackStatistics()
const override {}
2569 return ChangeStatus::UNCHANGED;
2574 InternalControlVar::ICV___last>
2575 ICVReplacementValuesMap;
2578 std::optional<Value *>
2580 return ICVReplacementValuesMap[ICV];
2585 const auto *ICVTrackingAA =
A.getAAFor<AAICVTracker>(
2588 if (!ICVTrackingAA->isAssumedTracked())
2589 return indicatePessimisticFixpoint();
2592 std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2593 std::optional<Value *> UniqueICVValue;
2596 std::optional<Value *> NewReplVal =
2597 ICVTrackingAA->getReplacementValue(ICV, &
I,
A);
2600 if (UniqueICVValue && UniqueICVValue != NewReplVal)
2603 UniqueICVValue = NewReplVal;
2608 bool UsedAssumedInformation =
false;
2609 if (!
A.checkForAllInstructions(CheckReturnInst, *
this, {Instruction::Ret},
2610 UsedAssumedInformation,
2612 UniqueICVValue =
nullptr;
2614 if (UniqueICVValue == ReplVal)
2617 ReplVal = UniqueICVValue;
2618 Changed = ChangeStatus::CHANGED;
2625struct AAICVTrackerCallSite : AAICVTracker {
2626 AAICVTrackerCallSite(
const IRPosition &IRP, Attributor &
A)
2627 : AAICVTracker(IRP,
A) {}
2630 assert(getAnchorScope() &&
"Expected anchor function");
2634 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
2636 auto ICVInfo = OMPInfoCache.ICVs[ICV];
2637 auto &Getter = OMPInfoCache.RFIs[ICVInfo.Getter];
2638 if (Getter.Declaration == getAssociatedFunction()) {
2639 AssociatedICV = ICVInfo.Kind;
2645 indicatePessimisticFixpoint();
2649 if (!ReplVal || !*ReplVal)
2650 return ChangeStatus::UNCHANGED;
2653 A.deleteAfterManifest(*getCtxI());
2655 return ChangeStatus::CHANGED;
2659 const std::string getAsStr(Attributor *)
const override {
2660 return "ICVTrackerCallSite";
2664 void trackStatistics()
const override {}
2667 std::optional<Value *> ReplVal;
2670 const auto *ICVTrackingAA =
A.getAAFor<AAICVTracker>(
2674 if (!ICVTrackingAA->isAssumedTracked())
2675 return indicatePessimisticFixpoint();
2677 std::optional<Value *> NewReplVal =
2678 ICVTrackingAA->getReplacementValue(AssociatedICV, getCtxI(),
A);
2680 if (ReplVal == NewReplVal)
2681 return ChangeStatus::UNCHANGED;
2683 ReplVal = NewReplVal;
2684 return ChangeStatus::CHANGED;
2689 std::optional<Value *>
2695struct AAICVTrackerCallSiteReturned : AAICVTracker {
2696 AAICVTrackerCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
2697 : AAICVTracker(IRP,
A) {}
2700 const std::string getAsStr(Attributor *)
const override {
2701 return "ICVTrackerCallSiteReturned";
2705 void trackStatistics()
const override {}
2709 return ChangeStatus::UNCHANGED;
2714 InternalControlVar::ICV___last>
2715 ICVReplacementValuesMap;
2719 std::optional<Value *>
2721 return ICVReplacementValuesMap[ICV];
2726 const auto *ICVTrackingAA =
A.getAAFor<AAICVTracker>(
2728 DepClassTy::REQUIRED);
2731 if (!ICVTrackingAA->isAssumedTracked())
2732 return indicatePessimisticFixpoint();
2735 std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2736 std::optional<Value *> NewReplVal =
2737 ICVTrackingAA->getUniqueReplacementValue(ICV);
2739 if (ReplVal == NewReplVal)
2742 ReplVal = NewReplVal;
2743 Changed = ChangeStatus::CHANGED;
2751static bool hasFunctionEndAsUniqueSuccessor(
const BasicBlock *BB) {
2757 return hasFunctionEndAsUniqueSuccessor(
Successor);
2760struct AAExecutionDomainFunction :
public AAExecutionDomain {
2761 AAExecutionDomainFunction(
const IRPosition &IRP, Attributor &
A)
2762 : AAExecutionDomain(IRP,
A) {}
2764 ~AAExecutionDomainFunction()
override {
delete RPOT; }
2768 assert(
F &&
"Expected anchor function");
2769 RPOT =
new ReversePostOrderTraversal<Function *>(
F);
2772 const std::string getAsStr(Attributor *)
const override {
2773 unsigned TotalBlocks = 0, InitialThreadBlocks = 0, AlignedBlocks = 0;
2774 for (
auto &It : BEDMap) {
2778 InitialThreadBlocks += It.getSecond().IsExecutedByInitialThreadOnly;
2779 AlignedBlocks += It.getSecond().IsReachedFromAlignedBarrierOnly &&
2780 It.getSecond().IsReachingAlignedBarrierOnly;
2782 return "[AAExecutionDomain] " + std::to_string(InitialThreadBlocks) +
"/" +
2783 std::to_string(AlignedBlocks) +
" of " +
2784 std::to_string(TotalBlocks) +
2785 " executed by initial thread / aligned";
2788 void trackStatistics()
const override {}
2792 for (
const BasicBlock &BB : *getAnchorScope()) {
2793 if (!isExecutedByInitialThreadOnly(BB))
2795 dbgs() <<
TAG <<
" Basic block @" << getAnchorScope()->getName() <<
" "
2796 << BB.
getName() <<
" is executed by a single thread.\n";
2802 if (!
A.Opts.openmp_opt_barrier_elimination)
2805 SmallPtrSet<CallBase *, 16> DeletedBarriers;
2806 auto HandleAlignedBarrier = [&](CallBase *CB) {
2807 const ExecutionDomainTy &ED = CB ? CEDMap[{CB, PRE}] : BEDMap[
nullptr];
2808 if (!ED.IsReachedFromAlignedBarrierOnly ||
2809 ED.EncounteredNonLocalSideEffect)
2811 if (!ED.EncounteredAssumes.empty() && !
A.isModulePass())
2822 DeletedBarriers.
insert(CB);
2823 A.deleteAfterManifest(*CB);
2824 ++NumBarriersEliminated;
2825 Changed = ChangeStatus::CHANGED;
2826 }
else if (!ED.AlignedBarriers.empty()) {
2827 Changed = ChangeStatus::CHANGED;
2829 ED.AlignedBarriers.end());
2830 SmallSetVector<CallBase *, 16> Visited;
2831 while (!Worklist.
empty()) {
2833 if (!Visited.
insert(LastCB))
2837 if (!hasFunctionEndAsUniqueSuccessor(LastCB->
getParent()))
2839 if (!DeletedBarriers.
count(LastCB)) {
2840 ++NumBarriersEliminated;
2841 A.deleteAfterManifest(*LastCB);
2847 const ExecutionDomainTy &LastED = CEDMap[{LastCB, PRE}];
2848 Worklist.
append(LastED.AlignedBarriers.begin(),
2849 LastED.AlignedBarriers.end());
2855 if (!ED.EncounteredAssumes.empty() && (CB || !ED.AlignedBarriers.empty()))
2856 for (
auto *AssumeCB : ED.EncounteredAssumes)
2857 A.deleteAfterManifest(*AssumeCB);
2860 for (
auto *CB : AlignedBarriers)
2861 HandleAlignedBarrier(CB);
2865 HandleAlignedBarrier(
nullptr);
2870 bool isNoOpFence(
const FenceInst &FI)
const override {
2871 return getState().isValidState() && !NonNoOpFences.count(&FI);
2877 mergeInPredecessorBarriersAndAssumptions(Attributor &
A, ExecutionDomainTy &ED,
2878 const ExecutionDomainTy &PredED);
2883 bool mergeInPredecessor(Attributor &
A, ExecutionDomainTy &ED,
2884 const ExecutionDomainTy &PredED,
2885 bool InitialEdgeOnly =
false);
2888 bool handleCallees(Attributor &
A, ExecutionDomainTy &EntryBBED);
2894 bool isExecutedByInitialThreadOnly(
const BasicBlock &BB)
const override {
2895 if (!isValidState())
2897 assert(BB.
getParent() == getAnchorScope() &&
"Block is out of scope!");
2898 return BEDMap.lookup(&BB).IsExecutedByInitialThreadOnly;
2901 bool isExecutedInAlignedRegion(Attributor &
A,
2902 const Instruction &
I)
const override {
2903 assert(
I.getFunction() == getAnchorScope() &&
2904 "Instruction is out of scope!");
2905 if (!isValidState())
2908 bool ForwardIsOk =
true;
2917 if (CB != &
I && AlignedBarriers.contains(
const_cast<CallBase *
>(CB)))
2919 const auto &It = CEDMap.find({CB, PRE});
2920 if (It == CEDMap.end())
2922 if (!It->getSecond().IsReachingAlignedBarrierOnly)
2923 ForwardIsOk =
false;
2927 if (!CurI && !BEDMap.lookup(
I.getParent()).IsReachingAlignedBarrierOnly)
2928 ForwardIsOk =
false;
2936 if (CB != &
I && AlignedBarriers.contains(
const_cast<CallBase *
>(CB)))
2938 const auto &It = CEDMap.find({CB, POST});
2939 if (It == CEDMap.end())
2941 if (It->getSecond().IsReachedFromAlignedBarrierOnly)
2954 return BEDMap.lookup(
nullptr).IsReachedFromAlignedBarrierOnly;
2956 return BEDMap.lookup(PredBB).IsReachedFromAlignedBarrierOnly;
2966 ExecutionDomainTy getExecutionDomain(
const BasicBlock &BB)
const override {
2968 "No request should be made against an invalid state!");
2969 return BEDMap.lookup(&BB);
2971 std::pair<ExecutionDomainTy, ExecutionDomainTy>
2972 getExecutionDomain(
const CallBase &CB)
const override {
2974 "No request should be made against an invalid state!");
2975 return {CEDMap.lookup({&CB, PRE}), CEDMap.lookup({&CB, POST})};
2977 ExecutionDomainTy getFunctionExecutionDomain()
const override {
2979 "No request should be made against an invalid state!");
2980 return InterProceduralED;
2986 static bool isInitialThreadOnlyEdge(Attributor &
A, CondBrInst *
Edge,
2987 BasicBlock &SuccessorBB) {
2990 if (
Edge->getSuccessor(0) != &SuccessorBB)
2994 if (!Cmp || !
Cmp->isTrueWhenEqual() || !
Cmp->isEquality())
3002 if (
C->isAllOnesValue()) {
3004 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
3005 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
3006 CB = CB ? OpenMPOpt::getCallIfRegularCall(*CB, &RFI) : nullptr;
3009 ConstantStruct *KernelEnvC =
3011 ConstantInt *ExecModeC =
3012 KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC);
3019 if (
II->getIntrinsicID() == Intrinsic::nvvm_read_ptx_sreg_tid_x)
3024 if (
II->getIntrinsicID() == Intrinsic::amdgcn_workitem_id_x)
3032 ExecutionDomainTy InterProceduralED;
3036 DenseMap<const BasicBlock *, ExecutionDomainTy> BEDMap;
3037 DenseMap<PointerIntPair<const CallBase *, 1, Direction>, ExecutionDomainTy>
3039 SmallSetVector<CallBase *, 16> AlignedBarriers;
3041 ReversePostOrderTraversal<Function *> *RPOT =
nullptr;
3044 static bool setAndRecord(
bool &R,
bool V) {
3052 SmallPtrSet<const FenceInst *, 8> NonNoOpFences;
3055void AAExecutionDomainFunction::mergeInPredecessorBarriersAndAssumptions(
3056 Attributor &
A, ExecutionDomainTy &ED,
const ExecutionDomainTy &PredED) {
3057 for (
auto *EA : PredED.EncounteredAssumes)
3058 ED.addAssumeInst(
A, *EA);
3060 for (
auto *AB : PredED.AlignedBarriers)
3061 ED.addAlignedBarrier(
A, *AB);
3064bool AAExecutionDomainFunction::mergeInPredecessor(
3065 Attributor &
A, ExecutionDomainTy &ED,
const ExecutionDomainTy &PredED,
3066 bool InitialEdgeOnly) {
3070 setAndRecord(ED.IsExecutedByInitialThreadOnly,
3071 InitialEdgeOnly || (PredED.IsExecutedByInitialThreadOnly &&
3072 ED.IsExecutedByInitialThreadOnly));
3074 Changed |= setAndRecord(ED.IsReachedFromAlignedBarrierOnly,
3075 ED.IsReachedFromAlignedBarrierOnly &&
3076 PredED.IsReachedFromAlignedBarrierOnly);
3077 Changed |= setAndRecord(ED.EncounteredNonLocalSideEffect,
3078 ED.EncounteredNonLocalSideEffect |
3079 PredED.EncounteredNonLocalSideEffect);
3081 if (ED.IsReachedFromAlignedBarrierOnly)
3082 mergeInPredecessorBarriersAndAssumptions(
A, ED, PredED);
3084 ED.clearAssumeInstAndAlignedBarriers();
3088bool AAExecutionDomainFunction::handleCallees(Attributor &
A,
3089 ExecutionDomainTy &EntryBBED) {
3091 auto PredForCallSite = [&](AbstractCallSite ACS) {
3092 const auto *EDAA =
A.getAAFor<AAExecutionDomain>(
3094 DepClassTy::OPTIONAL);
3095 if (!EDAA || !EDAA->getState().isValidState())
3098 EDAA->getExecutionDomain(*
cast<CallBase>(ACS.getInstruction())));
3102 ExecutionDomainTy ExitED;
3103 bool AllCallSitesKnown;
3104 if (
A.checkForAllCallSites(PredForCallSite, *
this,
3106 AllCallSitesKnown)) {
3107 for (
const auto &[CSInED, CSOutED] : CallSiteEDs) {
3108 mergeInPredecessor(
A, EntryBBED, CSInED);
3109 ExitED.IsReachingAlignedBarrierOnly &=
3110 CSOutED.IsReachingAlignedBarrierOnly;
3117 EntryBBED.IsExecutedByInitialThreadOnly =
false;
3118 EntryBBED.IsReachedFromAlignedBarrierOnly =
true;
3119 EntryBBED.EncounteredNonLocalSideEffect =
false;
3120 ExitED.IsReachingAlignedBarrierOnly =
false;
3122 EntryBBED.IsExecutedByInitialThreadOnly =
false;
3123 EntryBBED.IsReachedFromAlignedBarrierOnly =
false;
3124 EntryBBED.EncounteredNonLocalSideEffect =
true;
3125 ExitED.IsReachingAlignedBarrierOnly =
false;
3130 auto &FnED = BEDMap[
nullptr];
3131 Changed |= setAndRecord(FnED.IsReachedFromAlignedBarrierOnly,
3132 FnED.IsReachedFromAlignedBarrierOnly &
3133 EntryBBED.IsReachedFromAlignedBarrierOnly);
3134 Changed |= setAndRecord(FnED.IsReachingAlignedBarrierOnly,
3135 FnED.IsReachingAlignedBarrierOnly &
3136 ExitED.IsReachingAlignedBarrierOnly);
3137 Changed |= setAndRecord(FnED.IsExecutedByInitialThreadOnly,
3138 EntryBBED.IsExecutedByInitialThreadOnly);
3142ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &
A) {
3149 auto HandleAlignedBarrier = [&](CallBase &CB, ExecutionDomainTy &ED) {
3150 Changed |= AlignedBarriers.insert(&CB);
3152 auto &CallInED = CEDMap[{&CB, PRE}];
3153 Changed |= mergeInPredecessor(
A, CallInED, ED);
3154 CallInED.IsReachingAlignedBarrierOnly =
true;
3156 ED.EncounteredNonLocalSideEffect =
false;
3157 ED.IsReachedFromAlignedBarrierOnly =
true;
3159 ED.clearAssumeInstAndAlignedBarriers();
3160 ED.addAlignedBarrier(
A, CB);
3161 auto &CallOutED = CEDMap[{&CB, POST}];
3162 Changed |= mergeInPredecessor(
A, CallOutED, ED);
3166 A.getAAFor<AAIsDead>(*
this, getIRPosition(), DepClassTy::OPTIONAL);
3172 SmallVector<Instruction *> SyncInstWorklist;
3173 for (
auto &RIt : *RPOT) {
3176 bool IsEntryBB = &BB == &EntryBB;
3179 bool AlignedBarrierLastInBlock = IsEntryBB && IsKernel;
3180 bool IsExplicitlyAligned = IsEntryBB && IsKernel;
3181 ExecutionDomainTy ED;
3188 if (LivenessAA && LivenessAA->isAssumedDead(&BB))
3192 if (LivenessAA && LivenessAA->isEdgeDead(PredBB, &BB))
3194 bool InitialEdgeOnly = isInitialThreadOnlyEdge(
3196 mergeInPredecessor(
A, ED, BEDMap[PredBB], InitialEdgeOnly);
3202 for (Instruction &
I : BB) {
3203 bool UsedAssumedInformation;
3204 if (
A.isAssumedDead(
I, *
this, LivenessAA, UsedAssumedInformation,
3205 false, DepClassTy::OPTIONAL,
3213 ED.addAssumeInst(
A, *AI);
3217 if (
II->isAssumeLikeIntrinsic())
3222 if (!ED.EncounteredNonLocalSideEffect) {
3224 if (ED.IsReachedFromAlignedBarrierOnly)
3229 case AtomicOrdering::NotAtomic:
3231 case AtomicOrdering::Unordered:
3233 case AtomicOrdering::Monotonic:
3235 case AtomicOrdering::Acquire:
3237 case AtomicOrdering::Release:
3239 case AtomicOrdering::AcquireRelease:
3241 case AtomicOrdering::SequentiallyConsistent:
3245 NonNoOpFences.insert(FI);
3250 bool IsAlignedBarrier =
3254 AlignedBarrierLastInBlock &= IsNoSync;
3255 IsExplicitlyAligned &= IsNoSync;
3261 if (IsAlignedBarrier) {
3262 HandleAlignedBarrier(*CB, ED);
3263 AlignedBarrierLastInBlock =
true;
3264 IsExplicitlyAligned =
true;
3270 if (!ED.EncounteredNonLocalSideEffect &&
3272 ED.EncounteredNonLocalSideEffect =
true;
3274 ED.IsReachedFromAlignedBarrierOnly =
false;
3282 auto &CallInED = CEDMap[{CB, PRE}];
3283 Changed |= mergeInPredecessor(
A, CallInED, ED);
3289 if (!IsNoSync && Callee && !
Callee->isDeclaration()) {
3290 const auto *EDAA =
A.getAAFor<AAExecutionDomain>(
3292 if (EDAA && EDAA->getState().isValidState()) {
3293 const auto &CalleeED = EDAA->getFunctionExecutionDomain();
3294 ED.IsReachedFromAlignedBarrierOnly =
3295 CalleeED.IsReachedFromAlignedBarrierOnly;
3296 AlignedBarrierLastInBlock = ED.IsReachedFromAlignedBarrierOnly;
3297 if (IsNoSync || !CalleeED.IsReachedFromAlignedBarrierOnly)
3298 ED.EncounteredNonLocalSideEffect |=
3299 CalleeED.EncounteredNonLocalSideEffect;
3301 ED.EncounteredNonLocalSideEffect =
3302 CalleeED.EncounteredNonLocalSideEffect;
3303 if (!CalleeED.IsReachingAlignedBarrierOnly) {
3305 setAndRecord(CallInED.IsReachingAlignedBarrierOnly,
false);
3308 if (CalleeED.IsReachedFromAlignedBarrierOnly)
3309 mergeInPredecessorBarriersAndAssumptions(
A, ED, CalleeED);
3310 auto &CallOutED = CEDMap[{CB, POST}];
3311 Changed |= mergeInPredecessor(
A, CallOutED, ED);
3316 ED.IsReachedFromAlignedBarrierOnly =
false;
3317 Changed |= setAndRecord(CallInED.IsReachingAlignedBarrierOnly,
false);
3320 AlignedBarrierLastInBlock &= ED.IsReachedFromAlignedBarrierOnly;
3322 auto &CallOutED = CEDMap[{CB, POST}];
3323 Changed |= mergeInPredecessor(
A, CallOutED, ED);
3326 if (!
I.mayHaveSideEffects() && !
I.mayReadFromMemory())
3332 const auto *MemAA =
A.getAAFor<AAMemoryLocation>(
3340 if (MemAA && MemAA->getState().isValidState() &&
3341 MemAA->checkForAllAccessesToMemoryKind(
3346 auto &InfoCache =
A.getInfoCache();
3347 if (!
I.mayHaveSideEffects() && InfoCache.isOnlyUsedByAssume(
I))
3351 if (LI->hasMetadata(LLVMContext::MD_invariant_load))
3354 if (!ED.EncounteredNonLocalSideEffect &&
3356 ED.EncounteredNonLocalSideEffect =
true;
3359 bool IsEndAndNotReachingAlignedBarriersOnly =
false;
3361 !BB.getTerminator()->getNumSuccessors()) {
3363 Changed |= mergeInPredecessor(
A, InterProceduralED, ED);
3365 auto &FnED = BEDMap[
nullptr];
3366 if (IsKernel && !IsExplicitlyAligned)
3367 FnED.IsReachingAlignedBarrierOnly =
false;
3368 Changed |= mergeInPredecessor(
A, FnED, ED);
3370 if (!FnED.IsReachingAlignedBarrierOnly) {
3371 IsEndAndNotReachingAlignedBarriersOnly =
true;
3372 SyncInstWorklist.
push_back(BB.getTerminator());
3373 auto &BBED = BEDMap[&BB];
3374 Changed |= setAndRecord(BBED.IsReachingAlignedBarrierOnly,
false);
3378 ExecutionDomainTy &StoredED = BEDMap[&BB];
3379 ED.IsReachingAlignedBarrierOnly = StoredED.IsReachingAlignedBarrierOnly &&
3380 !IsEndAndNotReachingAlignedBarriersOnly;
3386 if (ED.IsExecutedByInitialThreadOnly !=
3387 StoredED.IsExecutedByInitialThreadOnly ||
3388 ED.IsReachedFromAlignedBarrierOnly !=
3389 StoredED.IsReachedFromAlignedBarrierOnly ||
3390 ED.EncounteredNonLocalSideEffect !=
3391 StoredED.EncounteredNonLocalSideEffect)
3395 StoredED = std::move(ED);
3400 SmallSetVector<BasicBlock *, 16> Visited;
3401 while (!SyncInstWorklist.
empty()) {
3404 bool HitAlignedBarrierOrKnownEnd =
false;
3409 auto &CallOutED = CEDMap[{CB, POST}];
3410 Changed |= setAndRecord(CallOutED.IsReachingAlignedBarrierOnly,
false);
3411 auto &CallInED = CEDMap[{CB, PRE}];
3412 HitAlignedBarrierOrKnownEnd =
3413 AlignedBarriers.count(CB) || !CallInED.IsReachingAlignedBarrierOnly;
3414 if (HitAlignedBarrierOrKnownEnd)
3416 Changed |= setAndRecord(CallInED.IsReachingAlignedBarrierOnly,
false);
3418 if (HitAlignedBarrierOrKnownEnd)
3422 if (LivenessAA && LivenessAA->isEdgeDead(PredBB, SyncBB))
3424 if (!Visited.
insert(PredBB))
3426 auto &PredED = BEDMap[PredBB];
3427 if (setAndRecord(PredED.IsReachingAlignedBarrierOnly,
false)) {
3429 SyncInstWorklist.
push_back(PredBB->getTerminator());
3432 if (SyncBB != &EntryBB)
3435 setAndRecord(InterProceduralED.IsReachingAlignedBarrierOnly,
false);
3438 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
3443struct AAHeapToShared :
public StateWrapper<BooleanState, AbstractAttribute> {
3444 using Base = StateWrapper<BooleanState, AbstractAttribute>;
3445 AAHeapToShared(
const IRPosition &IRP, Attributor &
A) :
Base(IRP) {}
3448 static AAHeapToShared &createForPosition(
const IRPosition &IRP,
3452 virtual bool isAssumedHeapToShared(CallBase &CB)
const = 0;
3456 virtual bool isAssumedHeapToSharedRemovedFree(CallBase &CB)
const = 0;
3458 StringRef
getName()
const override {
return "AAHeapToShared"; }
3460 const char *getIdAddr()
const override {
return &ID; }
3464 static bool classof(
const AbstractAttribute *AA) {
3469 static const char ID;
3472struct AAHeapToSharedFunction :
public AAHeapToShared {
3473 AAHeapToSharedFunction(
const IRPosition &IRP, Attributor &
A)
3474 : AAHeapToShared(IRP,
A) {}
3476 const std::string getAsStr(Attributor *)
const override {
3477 return "[AAHeapToShared] " + std::to_string(MallocCalls.size()) +
3478 " malloc calls eligible.";
3481 void trackStatistics()
const override {}
3485 void findPotentialRemovedFreeCalls(Attributor &
A) {
3486 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
3487 auto &FreeRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
3489 PotentialRemovedFreeCalls.clear();
3491 for (CallBase *CB : MallocCalls) {
3493 for (
auto *U : CB->
users()) {
3495 if (
C &&
C->getCalledFunction() == FreeRFI.Declaration)
3499 if (FreeCalls.
size() != 1)
3502 PotentialRemovedFreeCalls.insert(FreeCalls.
front());
3507 if (!
A.Opts.openmp_opt_deglobalization) {
3508 indicatePessimisticFixpoint();
3512 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
3513 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3514 if (!RFI.Declaration)
3518 [](
const IRPosition &,
const AbstractAttribute *,
3519 bool &) -> std::optional<Value *> {
return nullptr; };
3522 const OMPInformationCache::RuntimeFunctionInfo::UseVector *
Uses =
3523 RFI.getUseVector(*
F);
3527 for (Use *U : *
Uses)
3529 MallocCalls.insert(CB);
3534 findPotentialRemovedFreeCalls(
A);
3537 bool isAssumedHeapToShared(CallBase &CB)
const override {
3538 return isValidState() && MallocCalls.count(&CB);
3541 bool isAssumedHeapToSharedRemovedFree(CallBase &CB)
const override {
3542 return isValidState() && PotentialRemovedFreeCalls.count(&CB);
3546 if (MallocCalls.empty())
3547 return ChangeStatus::UNCHANGED;
3549 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
3550 auto &FreeCall = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
3554 DepClassTy::OPTIONAL);
3557 for (CallBase *CB : MallocCalls) {
3559 if (HS &&
HS->isAssumedHeapToStack(*CB))
3564 for (
auto *U : CB->
users()) {
3566 if (
C &&
C->getCalledFunction() == FreeCall.Declaration)
3569 if (FreeCalls.
size() != 1)
3574 if (AllocSize->getZExtValue() + SharedMemoryUsed >
3575 A.Opts.openmp_opt_shared_limit) {
3577 <<
" with shared memory."
3578 <<
" Shared memory usage is limited to "
3579 <<
A.Opts.openmp_opt_shared_limit <<
" bytes\n");
3584 <<
" with " << AllocSize->getZExtValue()
3585 <<
" bytes of shared memory\n");
3590 Type *Int8Ty = Type::getInt8Ty(
M->getContext());
3591 Type *Int8ArrTy = ArrayType::get(Int8Ty, AllocSize->getZExtValue());
3592 auto *SharedMem =
new GlobalVariable(
3596 static_cast<unsigned>(AddressSpace::Shared));
3598 SharedMem, PointerType::getUnqual(
M->getContext()));
3600 auto Remark = [&](OptimizationRemark
OR) {
3601 return OR <<
"Replaced globalized variable with "
3602 <<
ore::NV(
"SharedMemory", AllocSize->getZExtValue())
3603 << (AllocSize->isOne() ?
" byte " :
" bytes ")
3604 <<
"of shared memory.";
3606 A.emitRemark<OptimizationRemark>(CB,
"OMP111",
Remark);
3608 MaybeAlign
Alignment = CB->getRetAlign();
3610 "HeapToShared on allocation without alignment attribute");
3614 A.deleteAfterManifest(*CB);
3615 A.deleteAfterManifest(*FreeCalls.
front());
3617 SharedMemoryUsed += AllocSize->getZExtValue();
3618 NumBytesMovedToSharedMemory = SharedMemoryUsed;
3619 Changed = ChangeStatus::CHANGED;
3626 if (MallocCalls.empty())
3627 return indicatePessimisticFixpoint();
3628 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
3629 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3630 if (!RFI.Declaration)
3631 return ChangeStatus::UNCHANGED;
3635 auto NumMallocCalls = MallocCalls.size();
3638 for (User *U : RFI.Declaration->
users()) {
3640 if (CB->getCaller() !=
F)
3642 if (!MallocCalls.count(CB))
3645 MallocCalls.remove(CB);
3648 const auto *ED =
A.getAAFor<AAExecutionDomain>(
3650 if (!ED || !ED->isExecutedByInitialThreadOnly(*CB))
3651 MallocCalls.remove(CB);
3655 findPotentialRemovedFreeCalls(
A);
3657 if (NumMallocCalls != MallocCalls.size())
3658 return ChangeStatus::CHANGED;
3660 return ChangeStatus::UNCHANGED;
3664 SmallSetVector<CallBase *, 4> MallocCalls;
3666 SmallPtrSet<CallBase *, 4> PotentialRemovedFreeCalls;
3668 unsigned SharedMemoryUsed = 0;
3671struct AAKernelInfo :
public StateWrapper<KernelInfoState, AbstractAttribute> {
3672 using Base = StateWrapper<KernelInfoState, AbstractAttribute>;
3673 AAKernelInfo(
const IRPosition &IRP, Attributor &
A) :
Base(IRP) {}
3677 static bool requiresCalleeForCallBase() {
return false; }
3680 void trackStatistics()
const override {}
3682 const std::string getAsStr(Attributor *)
const override {
3683 if (!isValidState())
3685 return std::string(SPMDCompatibilityTracker.isAssumed() ?
"SPMD"
3687 std::string(SPMDCompatibilityTracker.isAtFixpoint() ?
" [FIX]"
3689 std::string(
" #PRs: ") +
3690 (ReachedKnownParallelRegions.isValidState()
3691 ? std::to_string(ReachedKnownParallelRegions.size())
3693 ", #Unknown PRs: " +
3694 (ReachedUnknownParallelRegions.isValidState()
3695 ? std::to_string(ReachedUnknownParallelRegions.size())
3697 ", #Reaching Kernels: " +
3698 (ReachingKernelEntries.isValidState()
3699 ? std::to_string(ReachingKernelEntries.size())
3702 (ParallelLevels.isValidState()
3703 ? std::to_string(ParallelLevels.size())
3705 ", NestedPar: " + (NestedParallelism ?
"yes" :
"no");
3709 static AAKernelInfo &createForPosition(
const IRPosition &IRP, Attributor &
A);
3711 StringRef
getName()
const override {
return "AAKernelInfo"; }
3713 const char *getIdAddr()
const override {
return &ID; }
3716 static bool classof(
const AbstractAttribute *AA) {
3720 static const char ID;
3724static bool isStrictFence(
const Instruction &
I) {
3740 switch (Parsed->first) {
3741 case AtomicScope::Single:
3742 case AtomicScope::Wavefront:
3743 case AtomicScope::Workgroup:
3745 case AtomicScope::Cluster:
3746 case AtomicScope::Device:
3747 case AtomicScope::System:
3755struct AAKernelInfoFunction : AAKernelInfo {
3756 AAKernelInfoFunction(
const IRPosition &IRP, Attributor &
A)
3757 : AAKernelInfo(IRP,
A) {}
3759 SmallPtrSet<Instruction *, 4> GuardedInstructions;
3761 SmallPtrSetImpl<Instruction *> &getGuardedInstructions() {
3762 return GuardedInstructions;
3765 void setConfigurationOfKernelEnvironment(ConstantStruct *ConfigC) {
3767 KernelEnvC, ConfigC, {KernelInfo::ConfigurationIdx});
3768 assert(NewKernelEnvC &&
"Failed to create new kernel environment");
3772#define KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MEMBER) \
3773 void set##MEMBER##OfKernelEnvironment(ConstantInt *NewVal) { \
3774 ConstantStruct *ConfigC = \
3775 KernelInfo::getConfigurationFromKernelEnvironment(KernelEnvC); \
3776 Constant *NewConfigC = ConstantFoldInsertValueInstruction( \
3777 ConfigC, NewVal, {KernelInfo::MEMBER##Idx}); \
3778 assert(NewConfigC && "Failed to create new configuration environment"); \
3779 setConfigurationOfKernelEnvironment(cast<ConstantStruct>(NewConfigC)); \
3790#undef KERNEL_ENVIRONMENT_CONFIGURATION_SETTER
3796 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
3800 OMPInformationCache::RuntimeFunctionInfo &InitRFI =
3801 OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
3802 OMPInformationCache::RuntimeFunctionInfo &DeinitRFI =
3803 OMPInfoCache.RFIs[OMPRTL___kmpc_target_deinit];
3807 auto StoreCallBase = [](
Use &U,
3808 OMPInformationCache::RuntimeFunctionInfo &RFI,
3810 CallBase *CB = OpenMPOpt::getCallIfRegularCall(U, &RFI);
3812 "Unexpected use of __kmpc_target_init or __kmpc_target_deinit!");
3814 "Multiple uses of __kmpc_target_init or __kmpc_target_deinit!");
3820 StoreCallBase(U, InitRFI, KernelInitCB);
3824 DeinitRFI.foreachUse(
3826 StoreCallBase(U, DeinitRFI, KernelDeinitCB);
3832 if (!KernelInitCB || !KernelDeinitCB)
3836 ReachingKernelEntries.insert(Fn);
3837 IsKernelEntry =
true;
3845 KernelConfigurationSimplifyCB =
3847 bool &UsedAssumedInformation) -> std::optional<Constant *> {
3848 if (!isAtFixpoint()) {
3851 UsedAssumedInformation =
true;
3857 A.registerGlobalVariableSimplificationCallback(
3858 *KernelEnvGV, KernelConfigurationSimplifyCB);
3861 bool CanChangeToSPMD = OMPInfoCache.runtimeFnsAvailable(
3862 {OMPRTL___kmpc_get_hardware_thread_id_in_block,
3863 OMPRTL___kmpc_barrier_simple_spmd});
3867 KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC);
3872 SPMDCompatibilityTracker.indicateOptimisticFixpoint();
3873 else if (!
A.Opts.openmp_opt_spmdization || !CanChangeToSPMD)
3876 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
3878 setExecModeOfKernelEnvironment(AssumedExecModeC);
3882 auto [MinThreads, MaxThreads] =
3885 setMinThreadsOfKernelEnvironment(ConstantInt::get(Int32Ty, MinThreads));
3887 setMaxThreadsOfKernelEnvironment(ConstantInt::get(Int32Ty, MaxThreads));
3888 auto [MinTeams, MaxTeams] =
3891 setMinTeamsOfKernelEnvironment(ConstantInt::get(Int32Ty, MinTeams));
3893 setMaxTeamsOfKernelEnvironment(ConstantInt::get(Int32Ty, MaxTeams));
3896 KernelInfo::getMayUseNestedParallelismFromKernelEnvironment(KernelEnvC);
3897 ConstantInt *AssumedMayUseNestedParallelismC = ConstantInt::get(
3899 setMayUseNestedParallelismOfKernelEnvironment(
3900 AssumedMayUseNestedParallelismC);
3902 if (
A.Opts.openmp_opt_state_machine_rewrite) {
3904 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
3907 ConstantInt::get(UseGenericStateMachineC->
getIntegerType(),
false);
3908 setUseGenericStateMachineOfKernelEnvironment(
3909 AssumedUseGenericStateMachineC);
3915 if (!OMPInfoCache.RFIs[RFKind].Declaration)
3917 A.registerVirtualUseCallback(*OMPInfoCache.RFIs[RFKind].Declaration, CB);
3921 auto AddDependence = [](
Attributor &
A,
const AAKernelInfo *KI,
3937 if (SPMDCompatibilityTracker.isValidState())
3938 return AddDependence(
A,
this, QueryingAA);
3940 if (!ReachedKnownParallelRegions.isValidState())
3941 return AddDependence(
A,
this, QueryingAA);
3947 RegisterVirtualUse(OMPRTL___kmpc_get_max_team_threads,
3948 CustomStateMachineUseCB);
3949 RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
3950 CustomStateMachineUseCB);
3951 RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
3952 CustomStateMachineUseCB);
3953 RegisterVirtualUse(OMPRTL___kmpc_kernel_end_parallel,
3954 CustomStateMachineUseCB);
3958 if (SPMDCompatibilityTracker.isAtFixpoint())
3965 if (!SPMDCompatibilityTracker.isValidState())
3966 return AddDependence(
A,
this, QueryingAA);
3969 RegisterVirtualUse(OMPRTL___kmpc_get_hardware_thread_id_in_block,
3978 if (!SPMDCompatibilityTracker.isValidState())
3979 return AddDependence(
A,
this, QueryingAA);
3980 if (SPMDCompatibilityTracker.empty())
3981 return AddDependence(
A,
this, QueryingAA);
3982 if (!mayContainParallelRegion())
3983 return AddDependence(
A,
this, QueryingAA);
3986 RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_spmd, SPMDBarrierUseCB);
3990 static std::string sanitizeForGlobalName(std::string S) {
3994 return !((C >=
'a' && C <=
'z') || (C >=
'A' && C <=
'Z') ||
3995 (C >=
'0' && C <=
'9') || C ==
'_');
4006 if (!KernelInitCB || !KernelDeinitCB)
4007 return ChangeStatus::UNCHANGED;
4011 bool HasBuiltStateMachine =
true;
4012 if (!changeToSPMDMode(
A,
Changed)) {
4014 HasBuiltStateMachine = buildCustomStateMachine(
A,
Changed);
4016 HasBuiltStateMachine =
false;
4020 ConstantStruct *ExistingKernelEnvC =
4022 ConstantInt *OldUseGenericStateMachineVal =
4023 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
4024 ExistingKernelEnvC);
4025 if (!HasBuiltStateMachine)
4026 setUseGenericStateMachineOfKernelEnvironment(
4027 OldUseGenericStateMachineVal);
4030 GlobalVariable *KernelEnvGV =
4034 Changed = ChangeStatus::CHANGED;
4040 void insertInstructionGuardsHelper(Attributor &
A) {
4041 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
4043 auto CreateGuardedRegion = [&](
Instruction *RegionStartI,
4045 LoopInfo *LI =
nullptr;
4046 DominatorTree *DT =
nullptr;
4047 MemorySSAUpdater *MSU =
nullptr;
4076 DT, LI, MSU,
"region.guarded.end");
4079 MSU,
"region.barrier");
4082 DT, LI, MSU,
"region.exit");
4084 SplitBlock(ParentBB, RegionStartI, DT, LI, MSU,
"region.guarded");
4087 "Expected a different CFG");
4090 ParentBB, ParentBB->
getTerminator(), DT, LI, MSU,
"region.check.tid");
4093 A.registerManifestAddedBasicBlock(*RegionEndBB);
4094 A.registerManifestAddedBasicBlock(*RegionBarrierBB);
4095 A.registerManifestAddedBasicBlock(*RegionExitBB);
4096 A.registerManifestAddedBasicBlock(*RegionStartBB);
4097 A.registerManifestAddedBasicBlock(*RegionCheckTidBB);
4099 bool HasBroadcastValues =
false;
4102 for (Instruction &
I : *RegionStartBB) {
4104 for (Use &U :
I.uses()) {
4110 if (OutsideUses.
empty())
4113 HasBroadcastValues =
true;
4117 auto *SharedMem =
new GlobalVariable(
4118 M,
I.getType(),
false,
4120 sanitizeForGlobalName(
4121 (
I.getName() +
".guarded.output.alloc").str()),
4123 static_cast<unsigned>(AddressSpace::Shared));
4126 new StoreInst(&
I, SharedMem,
4129 LoadInst *LoadI =
new LoadInst(
4130 I.getType(), SharedMem,
I.getName() +
".guarded.output.load",
4134 for (Use *U : OutsideUses)
4135 A.changeUseAfterManifest(*U, *LoadI);
4138 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
4143 OpenMPIRBuilder::LocationDescription Loc(ParentBB->
end(),
DL);
4145 uint32_t SrcLocStrSize;
4154 OpenMPIRBuilder::LocationDescription LocRegionCheckTid(
4155 RegionCheckTidBB->
end(),
DL);
4157 FunctionCallee HardwareTidFn =
4159 M, OMPRTL___kmpc_get_hardware_thread_id_in_block);
4163 OMPInfoCache.setCallingConvention(HardwareTidFn, Tid);
4165 OMPInfoCache.OMPBuilder.
Builder
4166 .
CreateCondBr(TidCheck, RegionStartBB, RegionBarrierBB)
4171 FunctionCallee BarrierFn =
4173 M, OMPRTL___kmpc_barrier_simple_spmd);
4178 OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
4181 if (HasBroadcastValues) {
4186 OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
4190 auto &AllocSharedRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
4191 SmallPtrSet<BasicBlock *, 8> Visited;
4192 for (Instruction *GuardedI : SPMDCompatibilityTracker) {
4194 if (!Visited.
insert(BB).second)
4200 while (++IP != IPEnd) {
4201 if (!IP->mayHaveSideEffects() && !IP->mayReadFromMemory())
4204 if (OpenMPOpt::getCallIfRegularCall(*
I, &AllocSharedRFI))
4207 if (isStrictFence(*
I) || !
I->user_empty() ||
4208 !SPMDCompatibilityTracker.contains(
I)) {
4209 LastEffect =
nullptr;
4216 for (
auto &Reorder : Reorders)
4217 Reorder.first->moveBefore(Reorder.second->getIterator());
4222 for (Instruction *GuardedI : SPMDCompatibilityTracker) {
4224 auto *CalleeAA =
A.lookupAAFor<AAKernelInfo>(
4227 assert(CalleeAA !=
nullptr &&
"Expected Callee AAKernelInfo");
4230 if (CalleeAAFunction.getGuardedInstructions().contains(GuardedI))
4233 Instruction *GuardedRegionStart =
nullptr, *GuardedRegionEnd =
nullptr;
4234 for (Instruction &
I : *BB) {
4236 if (isStrictFence(
I)) {
4237 if (SPMDCompatibilityTracker.contains(&
I))
4238 CalleeAAFunction.getGuardedInstructions().insert(&
I);
4239 if (GuardedRegionStart) {
4241 std::make_pair(GuardedRegionStart, GuardedRegionEnd));
4242 GuardedRegionStart =
nullptr;
4243 GuardedRegionEnd =
nullptr;
4250 if (SPMDCompatibilityTracker.contains(&
I)) {
4251 CalleeAAFunction.getGuardedInstructions().insert(&
I);
4252 if (GuardedRegionStart)
4253 GuardedRegionEnd = &
I;
4255 GuardedRegionStart = GuardedRegionEnd = &
I;
4262 if (GuardedRegionStart) {
4264 std::make_pair(GuardedRegionStart, GuardedRegionEnd));
4265 GuardedRegionStart =
nullptr;
4266 GuardedRegionEnd =
nullptr;
4271 for (
auto &GR : GuardedRegions)
4272 CreateGuardedRegion(GR.first, GR.second);
4275 void forceSingleThreadPerWorkgroupHelper(Attributor &
A) {
4284 auto &Ctx = getAnchorValue().getContext();
4291 KernelInitCB->
getNextNode(),
"main.thread.user_code");
4296 A.registerManifestAddedBasicBlock(*InitBB);
4297 A.registerManifestAddedBasicBlock(*UserCodeBB);
4298 A.registerManifestAddedBasicBlock(*ReturnBB);
4307 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
4308 FunctionCallee ThreadIdInBlockFn =
4310 M, OMPRTL___kmpc_get_hardware_thread_id_in_block);
4313 CallInst *ThreadIdInBlock =
4315 OMPInfoCache.setCallingConvention(ThreadIdInBlockFn, ThreadIdInBlock);
4321 ConstantInt::get(ThreadIdInBlock->
getType(), 0),
4322 "thread.is_main", InitBB);
4328 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
4330 if (!SPMDCompatibilityTracker.isAssumed()) {
4331 for (Instruction *NonCompatibleI : SPMDCompatibilityTracker) {
4332 if (!NonCompatibleI)
4337 if (OMPInfoCache.RTLFunctions.contains(CB->getCalledFunction()))
4340 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
4341 ORA <<
"Value has potential side effects preventing SPMD-mode "
4344 ORA <<
". Add `[[omp::assume(\"ompx_spmd_amenable\")]]` to "
4345 "the called function to override";
4349 A.emitRemark<OptimizationRemarkAnalysis>(NonCompatibleI,
"OMP121",
4353 << *NonCompatibleI <<
"\n");
4365 Kernel = CB->getCaller();
4370 ConstantStruct *ExistingKernelEnvC =
4373 KernelInfo::getExecModeFromKernelEnvironment(ExistingKernelEnvC);
4379 Changed = ChangeStatus::CHANGED;
4384 if (mayContainParallelRegion()) {
4385 insertInstructionGuardsHelper(
A);
4387 forceSingleThreadPerWorkgroupHelper(
A);
4388 setMaxThreadsOfKernelEnvironment(
4395 "Initially non-SPMD kernel has SPMD exec mode!");
4396 setExecModeOfKernelEnvironment(
4400 ++NumOpenMPTargetRegionKernelsSPMD;
4404 OMPInfoCache.SPMDizedKernels.insert(
Kernel);
4406 auto Remark = [&](OptimizationRemark
OR) {
4407 return OR <<
"Transformed generic-mode kernel to SPMD-mode.";
4409 A.emitRemark<OptimizationRemark>(KernelInitCB,
"OMP120",
Remark);
4415 if (!
A.Opts.openmp_opt_state_machine_rewrite)
4419 if (!ReachedKnownParallelRegions.isValidState())
4422 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
4423 if (!OMPInfoCache.runtimeFnsAvailable({OMPRTL___kmpc_get_max_team_threads,
4424 OMPRTL___kmpc_barrier_simple_generic,
4425 OMPRTL___kmpc_kernel_parallel,
4426 OMPRTL___kmpc_kernel_end_parallel}))
4429 ConstantStruct *ExistingKernelEnvC =
4436 ConstantInt *UseStateMachineC =
4437 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
4438 ExistingKernelEnvC);
4439 ConstantInt *ModeC =
4440 KernelInfo::getExecModeFromKernelEnvironment(ExistingKernelEnvC);
4445 if (UseStateMachineC->
isZero() ||
4449 Changed = ChangeStatus::CHANGED;
4452 setUseGenericStateMachineOfKernelEnvironment(
4459 if (!mayContainParallelRegion()) {
4460 ++NumOpenMPTargetRegionKernelsWithoutStateMachine;
4462 auto Remark = [&](OptimizationRemark
OR) {
4463 return OR <<
"Removing unused state machine from generic-mode kernel.";
4465 A.emitRemark<OptimizationRemark>(KernelInitCB,
"OMP130",
Remark);
4471 if (ReachedUnknownParallelRegions.empty()) {
4472 ++NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback;
4474 auto Remark = [&](OptimizationRemark
OR) {
4475 return OR <<
"Rewriting generic-mode kernel with a customized state "
4478 A.emitRemark<OptimizationRemark>(KernelInitCB,
"OMP131",
Remark);
4480 ++NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback;
4482 auto Remark = [&](OptimizationRemarkAnalysis
OR) {
4483 return OR <<
"Generic-mode kernel is executed with a customized state "
4484 "machine that requires a fallback.";
4486 A.emitRemark<OptimizationRemarkAnalysis>(KernelInitCB,
"OMP132",
Remark);
4489 for (CallBase *UnknownParallelRegionCB : ReachedUnknownParallelRegions) {
4490 if (!UnknownParallelRegionCB)
4492 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
4493 return ORA <<
"Call may contain unknown parallel regions. Use "
4494 <<
"`[[omp::assume(\"omp_no_parallelism\")]]` to "
4497 A.emitRemark<OptimizationRemarkAnalysis>(UnknownParallelRegionCB,
4530 auto &Ctx = getAnchorValue().getContext();
4534 BasicBlock *InitBB = KernelInitCB->getParent();
4536 KernelInitCB->getNextNode(),
"thread.user_code.check");
4540 Ctx,
"worker_state_machine.begin",
Kernel, UserCodeEntryBB);
4542 Ctx,
"worker_state_machine.finished",
Kernel, UserCodeEntryBB);
4544 Ctx,
"worker_state_machine.is_active.check",
Kernel, UserCodeEntryBB);
4547 Kernel, UserCodeEntryBB);
4550 Kernel, UserCodeEntryBB);
4552 Ctx,
"worker_state_machine.done.barrier",
Kernel, UserCodeEntryBB);
4553 A.registerManifestAddedBasicBlock(*InitBB);
4554 A.registerManifestAddedBasicBlock(*UserCodeEntryBB);
4555 A.registerManifestAddedBasicBlock(*IsWorkerCheckBB);
4556 A.registerManifestAddedBasicBlock(*StateMachineBeginBB);
4557 A.registerManifestAddedBasicBlock(*StateMachineFinishedBB);
4558 A.registerManifestAddedBasicBlock(*StateMachineIsActiveCheckBB);
4559 A.registerManifestAddedBasicBlock(*StateMachineIfCascadeCurrentBB);
4560 A.registerManifestAddedBasicBlock(*StateMachineEndParallelBB);
4561 A.registerManifestAddedBasicBlock(*StateMachineDoneBarrierBB);
4563 const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
4569 ConstantInt::getAllOnesValue(KernelInitCB->getType()),
4570 "thread.is_worker", InitBB);
4579 FunctionCallee MaxTeamThreadsFn =
4581 M, OMPRTL___kmpc_get_max_team_threads);
4582 Constant *IsSPMDArg = ConstantInt::get(OMPInfoCache.OMPBuilder.Int32, 0);
4584 MaxTeamThreadsFn, {IsSPMDArg},
"max_team_threads", IsWorkerCheckBB);
4585 OMPInfoCache.setCallingConvention(MaxTeamThreadsFn, MaxTeamThreads);
4589 "thread.is_main_or_worker", IsWorkerCheckBB);
4592 StateMachineFinishedBB, IsWorkerCheckBB);
4595 const DataLayout &
DL =
M.getDataLayout();
4596 Type *VoidPtrTy = PointerType::getUnqual(Ctx);
4598 new AllocaInst(VoidPtrTy,
DL.getAllocaAddrSpace(),
nullptr,
4603 OpenMPIRBuilder::LocationDescription(StateMachineBeginBB->
end(), DLoc));
4605 Value *Ident = KernelInfo::getIdentFromKernelEnvironment(KernelEnvC);
4606 Value *GTid = KernelInitCB;
4608 FunctionCallee BarrierFn =
4610 M, OMPRTL___kmpc_barrier_simple_generic);
4613 OMPInfoCache.setCallingConvention(BarrierFn, Barrier);
4617 (
unsigned int)AddressSpace::Generic) {
4618 WorkFnAI =
new AddrSpaceCastInst(
4619 WorkFnAI, PointerType::get(Ctx, (
unsigned int)AddressSpace::Generic),
4620 WorkFnAI->
getName() +
".generic", StateMachineBeginBB);
4624 FunctionCallee KernelParallelFn =
4626 M, OMPRTL___kmpc_kernel_parallel);
4628 KernelParallelFn, {WorkFnAI},
"worker.is_active", StateMachineBeginBB);
4629 OMPInfoCache.setCallingConvention(KernelParallelFn, IsActiveWorker);
4631 Instruction *WorkFn =
new LoadInst(VoidPtrTy, WorkFnAI,
"worker.work_fn",
4632 StateMachineBeginBB);
4635 FunctionType *ParallelRegionFnTy = FunctionType::get(
4636 Type::getVoidTy(Ctx), {Type::getInt16Ty(Ctx), Type::getInt32Ty(Ctx)},
4642 StateMachineBeginBB);
4643 IsDone->setDebugLoc(DLoc);
4645 StateMachineIsActiveCheckBB, StateMachineBeginBB)
4649 StateMachineDoneBarrierBB, StateMachineIsActiveCheckBB)
4655 const unsigned int WrapperFunctionArgNo = 6;
4660 for (
int I = 0,
E = ReachedKnownParallelRegions.size();
I <
E; ++
I) {
4661 auto *CB = ReachedKnownParallelRegions[
I];
4663 CB->getArgOperand(WrapperFunctionArgNo)->stripPointerCasts());
4665 Ctx,
"worker_state_machine.parallel_region.execute",
Kernel,
4666 StateMachineEndParallelBB);
4668 ->setDebugLoc(DLoc);
4674 Kernel, StateMachineEndParallelBB);
4675 A.registerManifestAddedBasicBlock(*PRExecuteBB);
4676 A.registerManifestAddedBasicBlock(*PRNextBB);
4681 if (
I + 1 <
E || !ReachedUnknownParallelRegions.empty()) {
4684 "worker.check_parallel_region", StateMachineIfCascadeCurrentBB);
4692 StateMachineIfCascadeCurrentBB)
4694 StateMachineIfCascadeCurrentBB = PRNextBB;
4700 if (!ReachedUnknownParallelRegions.empty()) {
4701 StateMachineIfCascadeCurrentBB->
setName(
4702 "worker_state_machine.parallel_region.fallback.execute");
4704 StateMachineIfCascadeCurrentBB)
4705 ->setDebugLoc(DLoc);
4708 StateMachineIfCascadeCurrentBB)
4711 FunctionCallee EndParallelFn =
4713 M, OMPRTL___kmpc_kernel_end_parallel);
4714 CallInst *EndParallel =
4716 OMPInfoCache.setCallingConvention(EndParallelFn, EndParallel);
4722 ->setDebugLoc(DLoc);
4732 KernelInfoState StateBefore = getState();
4738 struct UpdateKernelEnvCRAII {
4739 AAKernelInfoFunction &AA;
4741 UpdateKernelEnvCRAII(AAKernelInfoFunction &AA) : AA(AA) {}
4743 ~UpdateKernelEnvCRAII() {
4747 ConstantStruct *ExistingKernelEnvC =
4750 if (!AA.isValidState()) {
4751 AA.KernelEnvC = ExistingKernelEnvC;
4755 if (!AA.ReachedKnownParallelRegions.isValidState())
4756 AA.setUseGenericStateMachineOfKernelEnvironment(
4757 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
4758 ExistingKernelEnvC));
4760 if (!AA.SPMDCompatibilityTracker.isValidState())
4761 AA.setExecModeOfKernelEnvironment(
4762 KernelInfo::getExecModeFromKernelEnvironment(ExistingKernelEnvC));
4764 ConstantInt *MayUseNestedParallelismC =
4765 KernelInfo::getMayUseNestedParallelismFromKernelEnvironment(
4767 ConstantInt *NewMayUseNestedParallelismC = ConstantInt::get(
4768 MayUseNestedParallelismC->
getIntegerType(), AA.NestedParallelism);
4769 AA.setMayUseNestedParallelismOfKernelEnvironment(
4770 NewMayUseNestedParallelismC);
4780 if (!
I.mayWriteToMemory())
4783 const auto *UnderlyingObjsAA =
A.getAAFor<AAUnderlyingObjects>(
4785 DepClassTy::OPTIONAL);
4786 auto *
HS =
A.getAAFor<AAHeapToStack>(
4788 DepClassTy::OPTIONAL);
4789 if (UnderlyingObjsAA &&
4790 UnderlyingObjsAA->forallUnderlyingObjects([&](
Value &Obj) {
4791 if (AA::isAssumedThreadLocalObject(A, Obj, *this))
4795 auto *CB = dyn_cast<CallBase>(&Obj);
4796 return CB && HS && HS->isAssumedHeapToStack(*CB);
4802 SPMDCompatibilityTracker.insert(&
I);
4806 bool UsedAssumedInformationInCheckRWInst =
false;
4807 if (!SPMDCompatibilityTracker.isAtFixpoint())
4808 if (!
A.checkForAllReadWriteInstructions(
4809 CheckRWInst, *
this, UsedAssumedInformationInCheckRWInst))
4810 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4812 bool UsedAssumedInformationFromReachingKernels =
false;
4813 if (!IsKernelEntry) {
4814 updateParallelLevels(
A);
4816 bool AllReachingKernelsKnown =
true;
4817 updateReachingKernelEntries(
A, AllReachingKernelsKnown);
4818 UsedAssumedInformationFromReachingKernels = !AllReachingKernelsKnown;
4820 if (!SPMDCompatibilityTracker.empty()) {
4821 if (!ParallelLevels.isValidState())
4822 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4823 else if (!ReachingKernelEntries.isValidState())
4824 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4830 for (
auto *
Kernel : ReachingKernelEntries) {
4831 auto *CBAA =
A.getAAFor<AAKernelInfo>(
4833 if (CBAA && CBAA->SPMDCompatibilityTracker.isValidState() &&
4834 CBAA->SPMDCompatibilityTracker.isAssumed())
4838 if (!CBAA || !CBAA->SPMDCompatibilityTracker.isAtFixpoint())
4839 UsedAssumedInformationFromReachingKernels =
true;
4841 if (SPMD != 0 &&
Generic != 0)
4842 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4848 bool AllParallelRegionStatesWereFixed =
true;
4849 bool AllSPMDStatesWereFixed =
true;
4856 if (
Function *Callback = OMPInformationCache::getAnalyzableCallback(CB)) {
4858 <<
Callback->getName() <<
" of " << CB <<
"\n");
4859 if (
auto *CallbackAA =
A.getAAFor<AAKernelInfo>(
4861 getState() ^= CallbackAA->getState();
4862 AllSPMDStatesWereFixed &=
4863 CallbackAA->SPMDCompatibilityTracker.isAtFixpoint();
4864 AllParallelRegionStatesWereFixed &=
4865 CallbackAA->ReachedKnownParallelRegions.isAtFixpoint();
4866 AllParallelRegionStatesWereFixed &=
4867 CallbackAA->ReachedUnknownParallelRegions.isAtFixpoint();
4870 auto *CBAA =
A.getAAFor<AAKernelInfo>(
4874 getState() ^= CBAA->getState();
4875 AllSPMDStatesWereFixed &= CBAA->SPMDCompatibilityTracker.isAtFixpoint();
4876 AllParallelRegionStatesWereFixed &=
4877 CBAA->ReachedKnownParallelRegions.isAtFixpoint();
4878 AllParallelRegionStatesWereFixed &=
4879 CBAA->ReachedUnknownParallelRegions.isAtFixpoint();
4883 bool UsedAssumedInformationInCheckCallInst =
false;
4884 if (!
A.checkForAllCallLikeInstructions(
4885 CheckCallInst, *
this, UsedAssumedInformationInCheckCallInst)) {
4887 <<
"Failed to visit all call-like instructions!\n";);
4888 return indicatePessimisticFixpoint();
4893 if (!UsedAssumedInformationInCheckCallInst &&
4894 AllParallelRegionStatesWereFixed) {
4895 ReachedKnownParallelRegions.indicateOptimisticFixpoint();
4896 ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
4901 if (!UsedAssumedInformationInCheckRWInst &&
4902 !UsedAssumedInformationInCheckCallInst &&
4903 !UsedAssumedInformationFromReachingKernels && AllSPMDStatesWereFixed)
4904 SPMDCompatibilityTracker.indicateOptimisticFixpoint();
4906 return StateBefore == getState() ? ChangeStatus::UNCHANGED
4907 : ChangeStatus::CHANGED;
4912 void updateReachingKernelEntries(Attributor &
A,
4913 bool &AllReachingKernelsKnown) {
4914 auto PredCallSite = [&](AbstractCallSite ACS) {
4917 assert(Caller &&
"Caller is nullptr");
4919 auto *CAA =
A.getOrCreateAAFor<AAKernelInfo>(
4921 if (CAA && CAA->ReachingKernelEntries.isValidState()) {
4922 ReachingKernelEntries ^= CAA->ReachingKernelEntries;
4928 ReachingKernelEntries.indicatePessimisticFixpoint();
4933 if (!
A.checkForAllCallSites(PredCallSite, *
this,
4935 AllReachingKernelsKnown))
4936 ReachingKernelEntries.indicatePessimisticFixpoint();
4940 void updateParallelLevels(Attributor &
A) {
4941 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
4942 OMPInformationCache::RuntimeFunctionInfo &Parallel60RFI =
4943 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
4945 auto PredCallSite = [&](AbstractCallSite ACS) {
4948 assert(Caller &&
"Caller is nullptr");
4952 if (CAA && CAA->ParallelLevels.isValidState()) {
4958 if (Caller == Parallel60RFI.Declaration) {
4959 ParallelLevels.indicatePessimisticFixpoint();
4963 ParallelLevels ^= CAA->ParallelLevels;
4970 ParallelLevels.indicatePessimisticFixpoint();
4975 bool AllCallSitesKnown =
true;
4976 if (!
A.checkForAllCallSites(PredCallSite, *
this,
4979 ParallelLevels.indicatePessimisticFixpoint();
4986struct AAKernelInfoCallSite : AAKernelInfo {
4987 AAKernelInfoCallSite(
const IRPosition &IRP, Attributor &
A)
4988 : AAKernelInfo(IRP,
A) {}
4991 AAKernelInfo::initialize(
A);
4994 auto *AssumptionAA =
A.getAAFor<AAAssumptionInfo>(
4998 if (AssumptionAA && AssumptionAA->hasAssumption(
"ompx_spmd_amenable")) {
4999 indicateOptimisticFixpoint();
5007 indicateOptimisticFixpoint();
5016 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
5017 const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Callee);
5018 if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
5020 if (!Callee || !
A.isFunctionIPOAmendable(*Callee)) {
5024 if (!AssumptionAA ||
5025 !(AssumptionAA->hasAssumption(
"omp_no_openmp") ||
5026 AssumptionAA->hasAssumption(
"omp_no_parallelism")))
5027 ReachedUnknownParallelRegions.insert(&CB);
5031 if (!SPMDCompatibilityTracker.isAtFixpoint()) {
5032 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5033 SPMDCompatibilityTracker.insert(&CB);
5038 indicateOptimisticFixpoint();
5047 if (NumCallees > 1 && !
Callee->hasMetadata(LLVMContext::MD_callback)) {
5048 indicatePessimisticFixpoint();
5055 case OMPRTL___kmpc_is_spmd_exec_mode:
5056 case OMPRTL___kmpc_distribute_static_fini:
5057 case OMPRTL___kmpc_for_static_fini:
5058 case OMPRTL___kmpc_global_thread_num:
5059 case OMPRTL___kmpc_get_hardware_num_threads_in_block:
5060 case OMPRTL___kmpc_get_hardware_num_blocks:
5061 case OMPRTL___kmpc_single:
5062 case OMPRTL___kmpc_end_single:
5063 case OMPRTL___kmpc_master:
5064 case OMPRTL___kmpc_end_master:
5065 case OMPRTL___kmpc_barrier:
5066 case OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2:
5067 case OMPRTL___kmpc_gpu_xteam_reduce_nowait:
5068 case OMPRTL___kmpc_error:
5069 case OMPRTL___kmpc_flush:
5070 case OMPRTL___kmpc_get_hardware_thread_id_in_block:
5071 case OMPRTL___kmpc_get_warp_size:
5072 case OMPRTL_omp_get_thread_num:
5073 case OMPRTL_omp_get_num_threads:
5074 case OMPRTL_omp_get_max_threads:
5075 case OMPRTL_omp_in_parallel:
5076 case OMPRTL_omp_get_dynamic:
5077 case OMPRTL_omp_get_cancellation:
5078 case OMPRTL_omp_get_nested:
5079 case OMPRTL_omp_get_schedule:
5080 case OMPRTL_omp_get_thread_limit:
5081 case OMPRTL_omp_get_supported_active_levels:
5082 case OMPRTL_omp_get_max_active_levels:
5083 case OMPRTL_omp_get_level:
5084 case OMPRTL_omp_get_ancestor_thread_num:
5085 case OMPRTL_omp_get_team_size:
5086 case OMPRTL_omp_get_active_level:
5087 case OMPRTL_omp_in_final:
5088 case OMPRTL_omp_get_proc_bind:
5089 case OMPRTL_omp_get_num_places:
5090 case OMPRTL_omp_get_num_procs:
5091 case OMPRTL_omp_get_place_proc_ids:
5092 case OMPRTL_omp_get_place_num:
5093 case OMPRTL_omp_get_partition_num_places:
5094 case OMPRTL_omp_get_partition_place_nums:
5095 case OMPRTL_omp_get_wtime:
5097 case OMPRTL___kmpc_distribute_static_init_4:
5098 case OMPRTL___kmpc_distribute_static_init_4u:
5099 case OMPRTL___kmpc_distribute_static_init_8:
5100 case OMPRTL___kmpc_distribute_static_init_8u:
5101 case OMPRTL___kmpc_for_static_init_4:
5102 case OMPRTL___kmpc_for_static_init_4u:
5103 case OMPRTL___kmpc_for_static_init_8:
5104 case OMPRTL___kmpc_for_static_init_8u: {
5106 unsigned ScheduleArgOpNo = 2;
5107 auto *ScheduleTypeCI =
5109 unsigned ScheduleTypeVal =
5110 ScheduleTypeCI ? ScheduleTypeCI->getZExtValue() : 0;
5112 case OMPScheduleType::UnorderedStatic:
5113 case OMPScheduleType::UnorderedStaticChunked:
5114 case OMPScheduleType::OrderedDistribute:
5115 case OMPScheduleType::OrderedDistributeChunked:
5118 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5119 SPMDCompatibilityTracker.insert(&CB);
5123 case OMPRTL___kmpc_target_init:
5126 case OMPRTL___kmpc_target_deinit:
5127 KernelDeinitCB = &CB;
5129 case OMPRTL___kmpc_parallel_60:
5130 if (!handleParallel60(
A, CB))
5131 indicatePessimisticFixpoint();
5133 case OMPRTL___kmpc_omp_task:
5135 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5136 SPMDCompatibilityTracker.insert(&CB);
5137 ReachedUnknownParallelRegions.insert(&CB);
5139 case OMPRTL___kmpc_alloc_shared:
5140 case OMPRTL___kmpc_free_shared:
5148 case OMPRTL___kmpc_distribute_static_loop_4:
5149 case OMPRTL___kmpc_distribute_static_loop_4u:
5150 case OMPRTL___kmpc_distribute_static_loop_8:
5151 case OMPRTL___kmpc_distribute_static_loop_8u:
5159 if (!OMPInformationCache::getAnalyzableCallback(CB))
5160 ReachedUnknownParallelRegions.insert(&CB);
5161 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5162 SPMDCompatibilityTracker.insert(&CB);
5164 case OMPRTL___kmpc_distribute_for_static_loop_4:
5165 case OMPRTL___kmpc_distribute_for_static_loop_4u:
5166 case OMPRTL___kmpc_distribute_for_static_loop_8:
5167 case OMPRTL___kmpc_distribute_for_static_loop_8u:
5168 case OMPRTL___kmpc_for_static_loop_4:
5169 case OMPRTL___kmpc_for_static_loop_4u:
5170 case OMPRTL___kmpc_for_static_loop_8:
5171 case OMPRTL___kmpc_for_static_loop_8u:
5180 if (!OMPInformationCache::getAnalyzableCallback(CB))
5181 ReachedUnknownParallelRegions.insert(&CB);
5182 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5183 SPMDCompatibilityTracker.insert(&CB);
5188 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5189 SPMDCompatibilityTracker.insert(&CB);
5195 indicateOptimisticFixpoint();
5199 A.getAAFor<AACallEdges>(*
this, getIRPosition(), DepClassTy::OPTIONAL);
5200 if (!AACE || !AACE->getState().isValidState() || AACE->hasUnknownCallee()) {
5201 CheckCallee(getAssociatedFunction(), 1);
5204 const auto &OptimisticEdges = AACE->getOptimisticEdges();
5205 for (
auto *Callee : OptimisticEdges) {
5206 CheckCallee(Callee, OptimisticEdges.size());
5217 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
5218 KernelInfoState StateBefore = getState();
5220 auto CheckCallee = [&](
Function *
F,
int NumCallees) {
5221 const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(
F);
5225 if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
5228 A.getAAFor<AAKernelInfo>(*
this, FnPos, DepClassTy::REQUIRED);
5230 return indicatePessimisticFixpoint();
5231 if (getState() == FnAA->getState())
5232 return ChangeStatus::UNCHANGED;
5233 getState() = FnAA->getState();
5234 return ChangeStatus::CHANGED;
5238 if (NumCallees > 1 && !
F->hasMetadata(LLVMContext::MD_callback))
5239 return indicatePessimisticFixpoint();
5242 if (It->getSecond() == OMPRTL___kmpc_parallel_60) {
5243 if (!handleParallel60(
A, CB))
5244 return indicatePessimisticFixpoint();
5245 return StateBefore == getState() ? ChangeStatus::UNCHANGED
5246 : ChangeStatus::CHANGED;
5252 (It->getSecond() == OMPRTL___kmpc_alloc_shared ||
5253 It->getSecond() == OMPRTL___kmpc_free_shared) &&
5254 "Expected a __kmpc_alloc_shared or __kmpc_free_shared runtime call");
5256 auto *HeapToStackAA =
A.getAAFor<AAHeapToStack>(
5258 auto *HeapToSharedAA =
A.getAAFor<AAHeapToShared>(
5266 case OMPRTL___kmpc_alloc_shared:
5267 if ((!HeapToStackAA || !HeapToStackAA->isAssumedHeapToStack(CB)) &&
5268 (!HeapToSharedAA || !HeapToSharedAA->isAssumedHeapToShared(CB)))
5269 SPMDCompatibilityTracker.insert(&CB);
5271 case OMPRTL___kmpc_free_shared:
5272 if ((!HeapToStackAA ||
5273 !HeapToStackAA->isAssumedHeapToStackRemovedFree(CB)) &&
5275 !HeapToSharedAA->isAssumedHeapToSharedRemovedFree(CB)))
5276 SPMDCompatibilityTracker.insert(&CB);
5279 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5280 SPMDCompatibilityTracker.insert(&CB);
5282 return ChangeStatus::CHANGED;
5286 A.getAAFor<AACallEdges>(*
this, getIRPosition(), DepClassTy::OPTIONAL);
5287 if (!AACE || !AACE->getState().isValidState() || AACE->hasUnknownCallee()) {
5288 if (
Function *
F = getAssociatedFunction())
5291 const auto &OptimisticEdges = AACE->getOptimisticEdges();
5292 for (
auto *Callee : OptimisticEdges) {
5293 CheckCallee(Callee, OptimisticEdges.size());
5299 return StateBefore == getState() ? ChangeStatus::UNCHANGED
5300 : ChangeStatus::CHANGED;
5305 bool handleParallel60(Attributor &
A, CallBase &CB) {
5306 const unsigned int NonWrapperFunctionArgNo = 5;
5307 const unsigned int WrapperFunctionArgNo = 6;
5308 auto ParallelRegionOpArgNo = SPMDCompatibilityTracker.isAssumed()
5309 ? NonWrapperFunctionArgNo
5310 : WrapperFunctionArgNo;
5314 if (!ParallelRegion)
5317 ReachedKnownParallelRegions.insert(&CB);
5319 auto *FnAA =
A.getAAFor<AAKernelInfo>(
5321 NestedParallelism |= !FnAA || !FnAA->getState().isValidState() ||
5322 !FnAA->ReachedKnownParallelRegions.empty() ||
5323 !FnAA->ReachedKnownParallelRegions.isValidState() ||
5324 !FnAA->ReachedUnknownParallelRegions.isValidState() ||
5325 !FnAA->ReachedUnknownParallelRegions.empty();
5330struct AAFoldRuntimeCall
5331 :
public StateWrapper<BooleanState, AbstractAttribute> {
5332 using Base = StateWrapper<BooleanState, AbstractAttribute>;
5334 AAFoldRuntimeCall(
const IRPosition &IRP, Attributor &
A) :
Base(IRP) {}
5337 void trackStatistics()
const override {}
5340 static AAFoldRuntimeCall &createForPosition(
const IRPosition &IRP,
5343 StringRef
getName()
const override {
return "AAFoldRuntimeCall"; }
5345 const char *getIdAddr()
const override {
return &ID; }
5349 static bool classof(
const AbstractAttribute *AA) {
5353 static const char ID;
5356struct AAFoldRuntimeCallCallSiteReturned : AAFoldRuntimeCall {
5357 AAFoldRuntimeCallCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
5358 : AAFoldRuntimeCall(IRP,
A) {}
5360 const std::string getAsStr(Attributor *)
const override {
5361 if (!isValidState())
5364 std::string Str(
"simplified value: ");
5366 if (!SimplifiedValue)
5367 return Str + std::string(
"none");
5369 if (!*SimplifiedValue)
5370 return Str + std::string(
"nullptr");
5373 return Str + std::to_string(CI->getSExtValue());
5375 return Str + std::string(
"unknown");
5379 if (!
A.Opts.openmp_opt_folding)
5380 indicatePessimisticFixpoint();
5384 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
5385 const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Callee);
5386 assert(It != OMPInfoCache.RuntimeFunctionIDMap.end() &&
5387 "Expected a known OpenMP runtime function");
5389 RFKind = It->getSecond();
5392 A.registerSimplificationCallback(
5394 [&](
const IRPosition &IRP,
const AbstractAttribute *AA,
5395 bool &UsedAssumedInformation) -> std::optional<Value *> {
5396 assert((isValidState() || SimplifiedValue ==
nullptr) &&
5397 "Unexpected invalid state!");
5399 if (!isAtFixpoint()) {
5400 UsedAssumedInformation =
true;
5402 A.recordDependence(*
this, *AA, DepClassTy::OPTIONAL);
5404 return SimplifiedValue;
5411 case OMPRTL___kmpc_is_spmd_exec_mode:
5414 case OMPRTL___kmpc_parallel_level:
5417 case OMPRTL___kmpc_get_hardware_num_threads_in_block:
5418 Changed =
Changed | foldKernelFnAttribute(
A,
"omp_target_thread_limit");
5420 case OMPRTL___kmpc_get_hardware_num_blocks:
5433 if (SimplifiedValue && *SimplifiedValue) {
5436 A.deleteAfterManifest(
I);
5439 auto Remark = [&](OptimizationRemark
OR) {
5441 return OR <<
"Replacing OpenMP runtime call "
5443 <<
ore::NV(
"FoldedValue",
C->getZExtValue()) <<
".";
5444 return OR <<
"Replacing OpenMP runtime call "
5448 if (CB &&
A.Opts.openmp_opt_verbose_remarks)
5449 A.emitRemark<OptimizationRemark>(CB,
"OMP180",
Remark);
5452 << **SimplifiedValue <<
"\n");
5454 Changed = ChangeStatus::CHANGED;
5461 SimplifiedValue =
nullptr;
5462 return AAFoldRuntimeCall::indicatePessimisticFixpoint();
5468 std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
5470 unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
5471 unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
5472 auto *CallerKernelInfoAA =
A.getAAFor<AAKernelInfo>(
5475 if (!CallerKernelInfoAA ||
5476 !CallerKernelInfoAA->ReachingKernelEntries.isValidState())
5477 return indicatePessimisticFixpoint();
5479 for (
Kernel K : CallerKernelInfoAA->ReachingKernelEntries) {
5481 DepClassTy::REQUIRED);
5483 if (!AA || !AA->isValidState()) {
5484 SimplifiedValue =
nullptr;
5485 return indicatePessimisticFixpoint();
5488 if (AA->SPMDCompatibilityTracker.isAssumed()) {
5489 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5494 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5495 ++KnownNonSPMDCount;
5497 ++AssumedNonSPMDCount;
5501 if ((AssumedSPMDCount + KnownSPMDCount) &&
5502 (AssumedNonSPMDCount + KnownNonSPMDCount))
5503 return indicatePessimisticFixpoint();
5505 auto &Ctx = getAnchorValue().getContext();
5506 if (KnownSPMDCount || AssumedSPMDCount) {
5507 assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
5508 "Expected only SPMD kernels!");
5511 SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx),
true);
5512 }
else if (KnownNonSPMDCount || AssumedNonSPMDCount) {
5513 assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
5514 "Expected only non-SPMD kernels!");
5517 SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx),
false);
5522 assert(!SimplifiedValue &&
"SimplifiedValue should be none");
5525 return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
5526 : ChangeStatus::CHANGED;
5531 std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
5533 auto *CallerKernelInfoAA =
A.getAAFor<AAKernelInfo>(
5536 if (!CallerKernelInfoAA ||
5537 !CallerKernelInfoAA->ParallelLevels.isValidState())
5538 return indicatePessimisticFixpoint();
5540 if (!CallerKernelInfoAA->ReachingKernelEntries.isValidState())
5541 return indicatePessimisticFixpoint();
5543 if (CallerKernelInfoAA->ReachingKernelEntries.empty()) {
5544 assert(!SimplifiedValue &&
5545 "SimplifiedValue should keep none at this point");
5546 return ChangeStatus::UNCHANGED;
5549 unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
5550 unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
5551 for (
Kernel K : CallerKernelInfoAA->ReachingKernelEntries) {
5553 DepClassTy::REQUIRED);
5554 if (!AA || !AA->SPMDCompatibilityTracker.isValidState())
5555 return indicatePessimisticFixpoint();
5557 if (AA->SPMDCompatibilityTracker.isAssumed()) {
5558 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5563 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5564 ++KnownNonSPMDCount;
5566 ++AssumedNonSPMDCount;
5570 if ((AssumedSPMDCount + KnownSPMDCount) &&
5571 (AssumedNonSPMDCount + KnownNonSPMDCount))
5572 return indicatePessimisticFixpoint();
5574 auto &Ctx = getAnchorValue().getContext();
5578 if (AssumedSPMDCount || KnownSPMDCount) {
5579 assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
5580 "Expected only SPMD kernels!");
5581 SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), 1);
5583 assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
5584 "Expected only non-SPMD kernels!");
5585 SimplifiedValue = ConstantInt::get(Type::getInt8Ty(Ctx), 0);
5587 return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
5588 : ChangeStatus::CHANGED;
5591 ChangeStatus foldKernelFnAttribute(Attributor &
A, llvm::StringRef Attr) {
5593 int32_t CurrentAttrValue = -1;
5594 std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
5596 auto *CallerKernelInfoAA =
A.getAAFor<AAKernelInfo>(
5599 if (!CallerKernelInfoAA ||
5600 !CallerKernelInfoAA->ReachingKernelEntries.isValidState())
5601 return indicatePessimisticFixpoint();
5604 for (
Kernel K : CallerKernelInfoAA->ReachingKernelEntries) {
5605 int32_t NextAttrVal =
K->getFnAttributeAsParsedInteger(Attr, -1);
5607 if (NextAttrVal == -1 ||
5608 (CurrentAttrValue != -1 && CurrentAttrValue != NextAttrVal))
5609 return indicatePessimisticFixpoint();
5610 CurrentAttrValue = NextAttrVal;
5613 if (CurrentAttrValue != -1) {
5614 auto &Ctx = getAnchorValue().getContext();
5616 ConstantInt::get(Type::getInt32Ty(Ctx), CurrentAttrValue);
5618 return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
5619 : ChangeStatus::CHANGED;
5625 std::optional<Value *> SimplifiedValue;
5635 auto &RFI = OMPInfoCache.RFIs[RF];
5636 RFI.foreachUse(SCC, [&](Use &U,
Function &
F) {
5637 CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, &RFI);
5640 A.getOrCreateAAFor<AAFoldRuntimeCall>(
5642 DepClassTy::NONE,
false,
5648void OpenMPOpt::registerAAs(
bool IsModulePass) {
5658 A.getOrCreateAAFor<AAKernelInfo>(
5660 DepClassTy::NONE,
false,
5664 OMPInformationCache::RuntimeFunctionInfo &InitRFI =
5665 OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
5666 InitRFI.foreachUse(SCC, CreateKernelInfoCB);
5668 registerFoldRuntimeCall(OMPRTL___kmpc_is_spmd_exec_mode);
5669 registerFoldRuntimeCall(OMPRTL___kmpc_parallel_level);
5670 registerFoldRuntimeCall(OMPRTL___kmpc_get_hardware_num_threads_in_block);
5671 registerFoldRuntimeCall(OMPRTL___kmpc_get_hardware_num_blocks);
5675 if (
A.Opts.openmp_deduce_icv_values) {
5676 for (
int Idx = 0;
Idx < OMPInfoCache.ICVs.size() - 1; ++
Idx) {
5679 auto &GetterRFI = OMPInfoCache.RFIs[ICVInfo.Getter];
5682 CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, &GetterRFI);
5689 A.getOrCreateAAFor<AAICVTracker>(CBPos);
5693 GetterRFI.foreachUse(SCC, CreateAA);
5702 for (
auto *
F : SCC) {
5703 if (
F->isDeclaration())
5709 if (
F->hasLocalLinkage()) {
5711 const auto *CB = dyn_cast<CallBase>(U.getUser());
5712 return CB && CB->isCallee(&U) &&
5713 A.isRunOn(const_cast<Function *>(CB->getCaller()));
5717 registerAAsForFunction(
A, *
F);
5721void OpenMPOpt::registerAAsForFunction(Attributor &
A,
const Function &
F) {
5722 auto &OMPInfoCache =
static_cast<OMPInformationCache &
>(
A.getInfoCache());
5725 A.getOrCreateAAFor<AAExecutionDomain>(FPos);
5726 if (
F.hasFnAttribute(Attribute::Convergent))
5727 A.getOrCreateAAFor<AANonConvergent>(FPos);
5729 bool FunctionUsesSharedAlloc =
false;
5730 if (
A.Opts.openmp_opt_deglobalization) {
5731 const OMPInformationCache::RuntimeFunctionInfo::UseVector *SharedAllocUses =
5732 OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared].getUseVector(
5734 FunctionUsesSharedAlloc = SharedAllocUses && !SharedAllocUses->
empty();
5736 bool HasHeapToStackCandidate =
false;
5737 const TargetLibraryInfo *
TLI =
nullptr;
5741 bool UsedAssumedInformation =
false;
5744 A.getOrCreateAAFor<AAAddressSpace>(
5749 if (
A.Opts.openmp_opt_deglobalization && !HasHeapToStackCandidate) {
5751 TLI =
A.getInfoCache().getTargetLibraryInfoForFunction(
F);
5752 HasHeapToStackCandidate =
5756 A.getOrCreateAAFor<AAIndirectCallInfo>(
5761 A.getOrCreateAAFor<AAAddressSpace>(
5770 if (
II->getIntrinsicID() == Intrinsic::assume) {
5771 A.getOrCreateAAFor<AAPotentialValues>(
5778 if (FunctionUsesSharedAlloc)
5779 A.getOrCreateAAFor<AAHeapToShared>(FPos);
5780 if (HasHeapToStackCandidate)
5781 A.getOrCreateAAFor<AAHeapToStack>(FPos);
5784const char AAICVTracker::ID = 0;
5785const char AAKernelInfo::ID = 0;
5787const char AAHeapToShared::ID = 0;
5788const char AAFoldRuntimeCall::ID = 0;
5790AAICVTracker &AAICVTracker::createForPosition(
const IRPosition &IRP,
5792 AAICVTracker *AA =
nullptr;
5800 AA =
new (
A.Allocator) AAICVTrackerFunctionReturned(IRP,
A);
5803 AA =
new (
A.Allocator) AAICVTrackerCallSiteReturned(IRP,
A);
5806 AA =
new (
A.Allocator) AAICVTrackerCallSite(IRP,
A);
5809 AA =
new (
A.Allocator) AAICVTrackerFunction(IRP,
A);
5818 AAExecutionDomainFunction *
AA =
nullptr;
5828 "AAExecutionDomain can only be created for function position!");
5830 AA =
new (
A.Allocator) AAExecutionDomainFunction(IRP,
A);
5837AAHeapToShared &AAHeapToShared::createForPosition(
const IRPosition &IRP,
5839 AAHeapToSharedFunction *
AA =
nullptr;
5849 "AAHeapToShared can only be created for function position!");
5851 AA =
new (
A.Allocator) AAHeapToSharedFunction(IRP,
A);
5858AAKernelInfo &AAKernelInfo::createForPosition(
const IRPosition &IRP,
5860 AAKernelInfo *AA =
nullptr;
5870 AA =
new (
A.Allocator) AAKernelInfoCallSite(IRP,
A);
5873 AA =
new (
A.Allocator) AAKernelInfoFunction(IRP,
A);
5880AAFoldRuntimeCall &AAFoldRuntimeCall::createForPosition(
const IRPosition &IRP,
5882 AAFoldRuntimeCall *AA =
nullptr;
5891 llvm_unreachable(
"KernelInfo can only be created for call site position!");
5893 AA =
new (
A.Allocator) AAFoldRuntimeCallCallSiteReturned(IRP,
A);
5913 unsigned NumAssumedCallees) {
5914 return NumAssumedCallees <=
A.Opts.openmp_opt_max_callees_for_specialization;
5921 if (!Opts.openmp_opt)
5928 if (Opts.openmp_opt_print_module_before)
5932 if (Kernels.contains(&
F))
5934 return !
F.use_empty();
5941 return ORA <<
"Could not internalize function. "
5942 <<
"Some optimizations may not be possible. [OMP140]";
5954 if (!
F.isDeclaration() && !Kernels.contains(&
F) && IsCalled(
F) &&
5955 Opts.openmp_opt_internalization) {
5958 }
else if (!
F.hasLocalLinkage() && !
F.hasFnAttribute(Attribute::Cold)) {
5971 if (!
F.isDeclaration() && !InternalizedMap.
lookup(&
F)) {
5973 Functions.insert(&
F);
5991 OMPInformationCache InfoCache(M, AG, Allocator,
nullptr, PostLink);
5993 unsigned MaxFixpointIterations =
6006 return F.hasFnAttribute(
"kernel");
6011 OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache,
A);
6017 if (!
F.isDeclaration() && !Kernels.contains(&
F) && !
F.isNoInline())
6018 F.addFnAttr(Attribute::AlwaysInline);
6020 if (Opts.openmp_opt_print_module_after)
6035 const IPOOptions &Opts =
getIPOOptions(
C.begin()->getFunction());
6036 if (!Opts.openmp_opt)
6049 Module &M = *
C.begin()->getFunction().getParent();
6051 if (Opts.openmp_opt_print_module_before)
6071 OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator,
6072 &Functions, PostLink);
6074 unsigned MaxFixpointIterations =
6089 OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache,
A);
6090 bool Changed = OMPOpt.run(
false);
6092 if (Opts.openmp_opt_print_module_after)
6109 if (
F.hasKernelCallingConv()) {
6114 ++NumOpenMPTargetRegionKernels;
6117 ++NumNonOpenMPTargetRegionKernels;
6124 Metadata *MD = M.getModuleFlag(
"openmp");
6132 Metadata *MD = M.getModuleFlag(
"openmp-device");
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
amdgpu next use AMDGPU Next Use Analysis Printer
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static const Function * getParent(const Value *V)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseSet and SmallDenseSet classes.
This file defines an array type that can be indexed using scoped enum values.
static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE, bool Skip)
Loop::LoopBounds::Direction Direction
Machine Check Debug Module
This file provides utility analysis objects describing memory locations.
uint64_t IntrinsicInst * II
This file defines constans and helpers used when dealing with OpenMP.
This file defines constans that will be used by both host and device compilation.
static constexpr auto TAG
#define KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MEMBER)
#define KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MEMBER, IDX)
#define KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MEMBER)
static bool shouldSpecializeIndirectCallee(Attributor &A, const AbstractAttribute &, CallBase &, Function &, unsigned NumAssumedCallees)
Bound the if-cascade AAIndirectCallInfo builds for an indirect call.
#define KERNEL_ENVIRONMENT_IDX(MEMBER, IDX)
#define KERNEL_ENVIRONMENT_GETTER(MEMBER, RETURNTYPE)
FunctionAnalysisManager FAM
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
This file contains the declarations for profiling metadata utility functions.
static StringRef getName(Value *V)
Remove Loads Into Fake Uses
std::pair< BasicBlock *, BasicBlock * > Edge
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
iterator begin()
Instruction iterator methods.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
reverse_iterator rbegin()
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
LLVM_ABI const BasicBlock * getUniqueSuccessor() const
Return the successor of this block if it has a unique successor.
InstListType::reverse_iterator reverse_iterator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool doesNotAccessMemory(unsigned OpNo) const
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
bool isArgOperand(const Use *U) const
bool hasOperandBundles() const
Return true if this User has any operand bundles.
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
Wrapper to unify "old style" CallGraph and "new style" LazyCallGraph.
void initialize(LazyCallGraph &LCG, LazyCallGraph::SCC &SCC, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR)
Initializers for usage outside of a CGSCC pass, inside a CGSCC pass in the old and new pass manager (...
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
@ ICMP_SLT
signed less than
static CondBrInst * Create(Value *Cond, BasicBlock *IfTrue, BasicBlock *IfFalse, InsertPosition InsertBefore=nullptr)
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static LLVM_ABI Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
This is the shared class of boolean and integer constants.
IntegerType * getIntegerType() const
Variant of the getType() method to always return an IntegerType, which reduces the amount of casting ...
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
This is an important base class in LLVM.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
LLVM_ABI Instruction * findNearestCommonDominator(Instruction *I1, Instruction *I2) const
Find the nearest instruction I that dominates both I1 and I2, in the sense that a result produced bef...
static ErrorSuccess success()
Create a success value.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
A proxy from a FunctionAnalysisManager to an SCC.
const BasicBlock & getEntryBlock() const
const BasicBlock & front() const
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void setEntryCount(uint64_t Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
Argument * getArg(unsigned i) const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
bool hasLocalLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
LLVM_ABI void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
CondBrInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
LLVM_ABI bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
LLVM_ABI bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
LLVM_ABI bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
iterator_range< user_iterator > users()
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void setSuccessor(unsigned Idx, BasicBlock *BB)
Update the specified successor to point at the provided block.
LLVM_ABI std::optional< StringRef > getSyncScopeName(SyncScope::ID Id) const
getSyncScopeName - Returns the name of a SyncScope::ID registered with LLVMContext,...
LLVM_ABI const DiagnosticHandler * getDiagHandlerPtr() const
getDiagHandlerPtr - Returns const raw pointer of DiagnosticHandler set by setDiagnosticHandler.
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
LLVM_ABI void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
LLVM_ABI StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
A Module instance is used to store all the information related to an LLVM module.
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
LLVM_ABI Constant * getOrCreateIdent(Constant *SrcLocStr, uint32_t SrcLocStrSize, omp::IdentFlag Flags=omp::IdentFlag(0), unsigned Reserve2Flags=0)
Return an ident_t* encoding the source location SrcLocStr and Flags.
LLVM_ABI FunctionCallee getOrCreateRuntimeFunction(Module &M, omp::RuntimeFunction FnID)
Return the function declaration for the runtime function with FnID.
static LLVM_ABI std::pair< int32_t, int32_t > readThreadBoundsForKernel(const Triple &T, Function &Kernel)
}
LLVM_ABI Constant * getOrCreateSrcLocStr(StringRef LocStr, uint32_t &SrcLocStrSize)
Return the (LLVM-IR) string describing the source location LocStr.
IRBuilder<>::InsertPoint InsertPointTy
Type used throughout for insertion points.
IRBuilder Builder
The LLVM-IR Builder used to create IR.
static LLVM_ABI std::pair< int32_t, int32_t > readTeamBoundsForKernel(const Triple &T, Function &Kernel)
Read/write a bounds on teams for Kernel.
bool updateToLocation(const LocationDescription &Loc)
Update the internal location to Loc.
LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
static LLVM_ABI ProfileSummary * getFromMD(Metadata *MD)
Construct profile summary from metdata.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Triple - Helper class for working with autoconf configuration names.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static UncondBrInst * Create(BasicBlock *Target, InsertPosition InsertBefore=nullptr)
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< user_iterator > users()
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
const ParentTy * getParent() const
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
GlobalVariable * getKernelEnvironementGVFromKernelInitCB(CallBase *KernelInitCB)
ConstantStruct * getKernelEnvironementFromKernelInitCB(CallBase *KernelInitCB)
Abstract Attribute helper functions.
LLVM_ABI bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache)
Return true if the value of VAC is a valid at the position of VAC, that is a constant,...
LLVM_ABI bool isPotentiallyAffectedByBarrier(Attributor &A, const Instruction &I, const AbstractAttribute &QueryingAA)
Return true if I is potentially affected by a barrier.
LLVM_ABI bool isNoSyncInst(Attributor &A, const Instruction &I, const AbstractAttribute &QueryingAA)
Return true if I is a nosync instruction.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr E & operator^=(E &LHS, E RHS)
@ BasicBlock
Various leaf nodes.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
constexpr uint64_t PointerSize
aarch64 pointer size.
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
LLVM_ABI bool isOpenMPDevice(Module &M)
Helper to determine if M is a OpenMP target offloading device module.
LLVM_ABI bool containsOpenMP(Module &M)
Helper to determine if M contains OpenMP.
InternalControlVar
IDs for all Internal Control Variables (ICVs).
RuntimeFunction
IDs for all omp runtime library (RTL) functions.
LLVM_ABI KernelSet getDeviceKernels(Module &M)
Get OpenMP device kernels in M.
@ OMP_TGT_EXEC_MODE_GENERIC_SPMD
@ OMP_TGT_EXEC_MODE_GENERIC
SetVector< Kernel > KernelSet
Set of kernels in the module.
Function * Kernel
Summary of a kernel (=entry point for target offloading).
LLVM_ABI bool isOpenMPKernel(Function &Fn)
Return true iff Fn is an OpenMP GPU kernel; Fn has the "kernel" attribute.
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< UseNode * > Use
friend class Instruction
Iterator for Instructions in a `BasicBlock.
LLVM_ABI iterator begin() const
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
bool succ_empty(const Instruction *I)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
bool operator!=(uint64_t V1, const APInt &V2)
constexpr from_range_t from_range
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
AnalysisManager< LazyCallGraph::SCC, LazyCallGraph & > CGSCCAnalysisManager
The CGSCC analysis manager.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
auto dyn_cast_or_null(const Y &Val)
const IPOOptions & getIPOOptions(const LLVMContext &C)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
std::optional< std::pair< AtomicScope, bool > > parseAtomicScopeIRString(const Triple &T, StringRef Name)
Parses a target syncscope string into its abstract scope, the inverse of getAtomicScopeIRString.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
void cantFail(Error Err, const char *Msg=nullptr)
Report a fatal error if Err is a failure value.
bool operator&=(SparseBitVector< ElementSize > *LHS, const SparseBitVector< ElementSize > &RHS)
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
std::string toString(const APInt &I, unsigned Radix, bool Signed, bool formatAsCLiteral=false, bool UpperCase=true, bool InsertSeparators=false)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto predecessors(const MachineBasicBlock *BB)
LLVM_ABI Constant * ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, ArrayRef< unsigned > Idxs)
Attempt to constant fold an insertvalue instruction with the specified operands and indices.
@ OPTIONAL
The target may be valid if the source is not.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
BumpPtrAllocatorImpl<> BumpPtrAllocator
The standard BumpPtrAllocator which just uses the default template parameters.
@ End
Run the machine verifier once, at the end of the pipeline.
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)
Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
static LLVM_ABI AAExecutionDomain & createForPosition(const IRPosition &IRP, Attributor &A)
Create an abstract attribute view for the position IRP.
AAExecutionDomain(const IRPosition &IRP, Attributor &A)
static LLVM_ABI const char ID
Unique ID (due to the unique address)
AccessKind
Simple enum to distinguish read/write/read-write accesses.
StateType::base_t MemoryLocationsKind
static LLVM_ABI bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned)
Helper function to determine if CB is an aligned (GPU) barrier.
Base struct for all "concrete attribute" deductions.
virtual const char * getIdAddr() const =0
This function should return the address of the ID of the AbstractAttribute.
An interface to query the internal state of an abstract attribute.
Wrapper for FunctionAnalysisManager.
Configuration for the Attributor.
std::function< void(Attributor &A, const Function &F)> InitializationCallback
Callback function to be invoked on internal functions marked live.
std::optional< unsigned > MaxFixpointIterations
Maximum number of iterations to run until fixpoint.
bool RewriteSignatures
Flag to determine if we rewrite function signatures.
OptimizationRemarkGetter OREGetter
IPOAmendableCBTy IPOAmendableCB
bool IsModulePass
Is the user of the Attributor a module pass or not.
std::function< bool(Attributor &A, const AbstractAttribute &AA, CallBase &CB, Function &AssumedCallee, unsigned NumAssumedCallees)> IndirectCalleeSpecializationCallback
Callback function to determine if an indirect call targets should be made direct call targets (with a...
bool DefaultInitializeLiveInternals
Flag to determine if we want to initialize all default AAs for an internal function marked live.
The fixpoint analysis framework that orchestrates the attribute deduction.
static LLVM_ABI bool isInternalizable(Function &F)
Returns true if the function F can be internalized.
std::function< std::optional< Value * >( const IRPosition &, const AbstractAttribute *, bool &)> SimplifictionCallbackTy
Register CB as a simplification callback.
std::function< std::optional< Constant * >( const GlobalVariable &, const AbstractAttribute *, bool &)> GlobalVariableSimplifictionCallbackTy
Register CB as a simplification callback.
std::function< bool(Attributor &, const AbstractAttribute *)> VirtualUseCallbackTy
static LLVM_ABI bool internalizeFunctions(SmallPtrSetImpl< Function * > &FnSet, DenseMap< Function *, Function * > &FnMap)
Make copies of each function in the set FnSet such that the copied version has internal linkage after...
Simple wrapper for a single bit (boolean) state.
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
bool isAnyRemarkEnabled(StringRef PassName) const
Return true if any type of remarks are enabled for this pass.
Helper to describe and deal with positions in the LLVM-IR.
static const IRPosition callsite_returned(const CallBase &CB)
Create a position describing the returned value of CB.
static const IRPosition returned(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the returned value of F.
static const IRPosition value(const Value &V, const CallBaseContext *CBContext=nullptr)
Create a position describing the value of V.
static const IRPosition inst(const Instruction &I, const CallBaseContext *CBContext=nullptr)
Create a position describing the instruction I.
@ IRP_ARGUMENT
An attribute for a function argument.
@ IRP_RETURNED
An attribute for the function return value.
@ IRP_CALL_SITE
An attribute for a call site (function scope).
@ IRP_CALL_SITE_RETURNED
An attribute for a call site return value.
@ IRP_FUNCTION
An attribute for a function (scope).
@ IRP_FLOAT
A position that is not associated with a spot suitable for attributes.
@ IRP_CALL_SITE_ARGUMENT
An attribute for a call site argument.
@ IRP_INVALID
An invalid position.
static const IRPosition function(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the function scope of F.
Kind getPositionKind() const
Return the associated position kind.
static const IRPosition callsite_function(const CallBase &CB)
Create a position describing the function scope of CB.
Defines various target-specific GPU grid values that must be consistent between host RTL (plugin),...