63#define DEBUG_TYPE "instrprof"
68 cl::desc(
"Use debug info to correlate profiles."),
75 "hash-based-counter-split",
76 cl::desc(
"Rename counter variable of a comdat function based on cfg hash"),
80 RuntimeCounterRelocation(
"runtime-counter-relocation",
81 cl::desc(
"Enable relocating counters at runtime."),
86 cl::desc(
"Do static counter allocation for value profiler"),
90 "vp-counters-per-site",
91 cl::desc(
"The average number of profile counters allocated "
92 "per value profiling site."),
100 "instrprof-atomic-counter-update-all",
101 cl::desc(
"Make all profile counter updates atomic (for testing only)"),
105 "atomic-counter-update-promoted",
106 cl::desc(
"Do counter update using atomic fetch add "
107 " for promoted counters only"),
111 "atomic-first-counter",
112 cl::desc(
"Use atomic fetch add for first counter in a function (usually "
113 "the entry counter)"),
122 cl::desc(
"Do counter register promotion"),
125 "max-counter-promotions-per-loop",
cl::init(20),
126 cl::desc(
"Max number counter promotions per loop to avoid"
127 " increasing register pressure too much"));
131 MaxNumOfPromotions(
"max-counter-promotions",
cl::init(-1),
132 cl::desc(
"Max number of allowed counter promotions"));
135 "speculative-counter-promotion-max-exiting",
cl::init(3),
136 cl::desc(
"The max number of exiting blocks of a loop to allow "
137 " speculative counter promotion"));
140 "speculative-counter-promotion-to-loop",
141 cl::desc(
"When the option is false, if the target block is in a loop, "
142 "the promotion will be disallowed unless the promoted counter "
143 " update can be further/iteratively promoted into an acyclic "
147 "iterative-counter-promotion",
cl::init(
true),
148 cl::desc(
"Allow counter promotion across the whole loop nest."));
151 "skip-ret-exit-block",
cl::init(
true),
152 cl::desc(
"Suppress counter promotion if exit blocks contain ret."));
163 PGOCounterPromoterHelper(
170 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
172 assert(isa<StoreInst>(S));
173 SSA.AddAvailableValue(PH,
Init);
177 for (
unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
183 Value *LiveInValue =
SSA.GetValueInMiddleOfBlock(ExitBlock);
184 Value *
Addr = cast<StoreInst>(Store)->getPointerOperand();
187 if (
auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(
Addr)) {
194 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
195 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
196 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
197 Addr = Builder.CreateIntToPtr(BiasInst,
200 if (AtomicCounterUpdatePromoted)
205 AtomicOrdering::SequentiallyConsistent);
207 LoadInst *OldVal = Builder.CreateLoad(Ty,
Addr,
"pgocount.promoted");
208 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
209 auto *NewStore = Builder.CreateStore(NewVal,
Addr);
212 if (IterativeCounterPromotion) {
213 auto *TargetLoop = LI.getLoopFor(ExitBlock);
215 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
232class PGOCounterPromoter {
237 : LoopToCandidates(LoopToCands),
L(CurLoop), LI(LI),
BFI(
BFI) {
244 L.getExitBlocks(LoopExitBlocks);
245 if (!isPromotionPossible(&L, LoopExitBlocks))
248 for (
BasicBlock *ExitBlock : LoopExitBlocks) {
249 if (BlockSet.
insert(ExitBlock).second &&
253 ExitBlocks.push_back(ExitBlock);
259 bool run(int64_t *NumPromoted) {
261 if (ExitBlocks.size() == 0)
269 if (SkipRetExitBlock) {
270 for (
auto *BB : ExitBlocks)
271 if (isa<ReturnInst>(BB->getTerminator()))
275 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
279 unsigned Promoted = 0;
280 for (
auto &Cand : LoopToCandidates[&L]) {
288 auto *BB = Cand.first->getParent();
292 auto PreheaderCount =
BFI->getBlockProfileCount(
L.getLoopPreheader());
295 if (PreheaderCount && (*PreheaderCount * 3) >= (*
InstrCount * 2))
299 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second,
SSA, InitVal,
300 L.getLoopPreheader(), ExitBlocks,
301 InsertPts, LoopToCandidates, LI);
304 if (Promoted >= MaxProm)
308 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
312 LLVM_DEBUG(
dbgs() << Promoted <<
" counters promoted for loop (depth="
313 <<
L.getLoopDepth() <<
")\n");
314 return Promoted != 0;
318 bool allowSpeculativeCounterPromotion(
Loop *LP) {
320 L.getExitingBlocks(ExitingBlocks);
322 if (ExitingBlocks.
size() == 1)
324 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
332 isPromotionPossible(
Loop *LP,
351 unsigned getMaxNumOfPromotionsInLoop(
Loop *LP) {
354 if (!isPromotionPossible(LP, LoopExitBlocks))
365 if (ExitingBlocks.
size() == 1)
366 return MaxNumOfPromotionsPerLoop;
368 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
372 if (SpeculativeCounterPromotionToLoop)
373 return MaxNumOfPromotionsPerLoop;
376 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
377 for (
auto *TargetBlock : LoopExitBlocks) {
378 auto *TargetLoop = LI.
getLoopFor(TargetBlock);
381 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
382 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
384 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
385 PendingCandsInTarget);
398enum class ValueProfilingCallType {
421bool InstrProfiling::lowerIntrinsics(
Function *
F) {
422 bool MadeChange =
false;
423 PromotionCandidates.clear();
426 if (
auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
427 lowerIncrement(IPIS);
429 }
else if (
auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
432 }
else if (
auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
435 }
else if (
auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
438 }
else if (
auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
439 lowerValueProfileInst(IPVP);
441 }
else if (
auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
442 IPMP->eraseFromParent();
444 }
else if (
auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
445 lowerMCDCTestVectorBitmapUpdate(IPBU);
447 }
else if (
auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(&Instr)) {
448 lowerMCDCCondBitmapUpdate(IPTU);
457 promoteCounterLoadStores(
F);
461bool InstrProfiling::isRuntimeCounterRelocationEnabled()
const {
466 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
467 return RuntimeCounterRelocation;
473bool InstrProfiling::isCounterPromotionEnabled()
const {
474 if (DoCounterPromotion.getNumOccurrences() > 0)
475 return DoCounterPromotion;
480void InstrProfiling::promoteCounterLoadStores(
Function *
F) {
481 if (!isCounterPromotionEnabled())
488 std::unique_ptr<BlockFrequencyInfo>
BFI;
490 std::unique_ptr<BranchProbabilityInfo> BPI;
495 for (
const auto &LoadStore : PromotionCandidates) {
502 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
510 PGOCounterPromoter Promoter(LoopPromotionCandidates, *
Loop, LI,
BFI.get());
511 Promoter.run(&TotalCountersPromoted);
517 if (TT.isOSFuchsia())
525 auto containsIntrinsic = [&](
int ID) {
527 return !
F->use_empty();
530 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
531 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
532 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
533 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
534 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
540 this->GetTLI = std::move(GetTLI);
543 ProfileDataMap.clear();
544 CompilerUsedVars.clear();
546 TT =
Triple(M.getTargetTriple());
548 bool MadeChange =
false;
550 if (NeedsRuntimeHook)
551 MadeChange = emitRuntimeHook();
557 if (!ContainsProfiling && !CoverageNamesVar)
567 if (
auto *Ind = dyn_cast<InstrProfValueProfileInst>(
I))
568 computeNumValueSiteCounts(Ind);
570 if (FirstProfInst ==
nullptr &&
571 (isa<InstrProfIncrementInst>(
I) || isa<InstrProfCoverInst>(
I)))
572 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(
I);
574 if (
const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(
I))
575 static_cast<void>(getOrCreateRegionBitmaps(Params));
582 if (FirstProfInst !=
nullptr) {
583 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
588 MadeChange |= lowerIntrinsics(&
F);
590 if (CoverageNamesVar) {
591 lowerCoverageData(CoverageNamesVar);
605 if (!NeedsRuntimeHook && ContainsProfiling)
610 emitInitialization();
616 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
621 if (
auto AK = TLI.getExtAttrForI32Param(
false))
622 AL = AL.addParamAttribute(M.getContext(), 2, AK);
624 assert((CallType == ValueProfilingCallType::Default ||
625 CallType == ValueProfilingCallType::MemOp) &&
626 "Must be Default or MemOp");
627 Type *ParamTypes[] = {
628#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
631 auto *ValueProfilingCallTy =
633 StringRef FuncName = CallType == ValueProfilingCallType::Default
636 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
643 auto &
PD = ProfileDataMap[
Name];
654 "Value profiling is not yet supported with lightweight instrumentation");
656 auto It = ProfileDataMap.find(
Name);
657 assert(It != ProfileDataMap.end() && It->second.DataVar &&
658 "value profiling detected in function with no counter incerement");
664 Index += It->second.NumValueSites[Kind];
668 llvm::InstrProfValueKind::IPVK_MemOPSize);
684 Call = Builder.CreateCall(
688 if (
auto AK = TLI->getExtAttrForI32Param(
false))
689 Call->addParamAttr(2, AK);
695 auto *Counters = getOrCreateRegionCounters(
I);
698 if (isa<InstrProfTimestampInst>(
I))
699 Counters->setAlignment(
Align(8));
701 auto *
Addr = Builder.CreateConstInBoundsGEP2_32(
702 Counters->getValueType(), Counters, 0,
I->getIndex()->getZExtValue());
704 if (!isRuntimeCounterRelocationEnabled())
708 Function *Fn =
I->getParent()->getParent();
709 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
726 Bias->setComdat(
M->getOrInsertComdat(Bias->getName()));
728 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
730 auto *
Add = Builder.CreateAdd(Builder.CreatePtrToInt(
Addr, Int64Ty), BiasLI);
731 return Builder.CreateIntToPtr(
Add,
Addr->getType());
735 auto *Bitmaps = getOrCreateRegionBitmaps(
I);
738 auto *
Addr = Builder.CreateConstInBoundsGEP2_32(
739 Bitmaps->getValueType(), Bitmaps, 0,
I->getBitmapIndex()->getZExtValue());
741 if (isRuntimeCounterRelocationEnabled()) {
745 Twine(
"Runtime counter relocation is presently not supported for MC/DC "
754 auto *
Addr = getCounterAddress(CoverInstruction);
757 Builder.CreateStore(Builder.getInt8(0),
Addr);
761void InstrProfiling::lowerTimestamp(
764 "timestamp probes are always the first probe for a function");
765 auto &Ctx =
M->getContext();
766 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
770 auto Callee =
M->getOrInsertFunction(
771 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
772 Builder.CreateCall(Callee, {TimestampAddr});
777 auto *
Addr = getCounterAddress(Inc);
780 if (Options.
Atomic || AtomicCounterUpdateAll ||
787 auto *Count = Builder.CreateAdd(Load, Inc->
getStep());
788 auto *
Store = Builder.CreateStore(Count,
Addr);
789 if (isCounterPromotionEnabled())
790 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
795void InstrProfiling::lowerCoverageData(
GlobalVariable *CoverageNamesVar) {
801 assert(isa<GlobalVariable>(V) &&
"Missing reference to function name");
805 ReferencedNames.push_back(
Name);
806 if (isa<ConstantExpr>(
NC))
807 NC->dropAllReferences();
812void InstrProfiling::lowerMCDCTestVectorBitmapUpdate(
820 auto *BitmapAddr = getBitmapAddress(Update);
824 auto *Temp = Builder.CreateLoad(
Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp");
828 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
833 auto *BitmapByteAddr =
834 Builder.CreateAdd(Builder.CreatePtrToInt(BitmapAddr, Int64Ty),
835 Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty));
839 BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy);
844 auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
848 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
852 auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr,
"mcdc.bits");
856 auto *
Result = Builder.CreateOr(Bitmap, ShiftedVal);
860 Builder.CreateStore(Result, BitmapByteAddr);
864void InstrProfiling::lowerMCDCCondBitmapUpdate(
872 auto *Temp = Builder.CreateLoad(
Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp");
880 auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->
getCondID());
884 auto *
Result = Builder.CreateOr(Temp, ShiftedVal);
888 Builder.CreateStore(Result, MCDCCondBitmapAddr);
902 return (Prefix +
Name).str();
908 return (Prefix +
Name).str();
909 return (Prefix +
Name +
"." +
Twine(FuncHash)).str();
913 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
919 return cast<ConstantInt>(MD->getValue())->getZExtValue();
941 bool HasAvailableExternallyLinkage =
F->hasAvailableExternallyLinkage();
942 if (!
F->hasLinkOnceLinkage() && !
F->hasLocalLinkage() &&
943 !HasAvailableExternallyLinkage)
949 if (HasAvailableExternallyLinkage &&
950 F->hasFnAttribute(Attribute::AlwaysInline))
956 if (
F->hasLocalLinkage() &&
F->hasComdat())
966 return F->hasAddressTaken() ||
F->hasLinkOnceLinkage();
1013 Fn->
getName() +
".local", Fn);
1037 if (TT.isOSDarwin())
1040 if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
1041 TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows())
1058 Comdat *
C =
M->getOrInsertComdat(GroupName);
1114 if (IPSK == IPSK_cnts) {
1118 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1119 }
else if (IPSK == IPSK_bitmap) {
1123 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1124 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1129 Ptr->setVisibility(Visibility);
1133 Ptr->setLinkage(Linkage);
1134 maybeSetComdat(
Ptr, Fn, VarName);
1153 auto &
PD = ProfileDataMap[NamePtr];
1154 if (
PD.RegionBitmaps)
1155 return PD.RegionBitmaps;
1159 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1160 PD.RegionBitmaps = BitmapPtr;
1162 return PD.RegionBitmaps;
1169 auto &Ctx =
M->getContext();
1171 if (isa<InstrProfCoverInst>(Inc)) {
1175 std::vector<Constant *> InitialValues(NumCounters,
1193 auto &
PD = ProfileDataMap[NamePtr];
1194 if (
PD.RegionCounters)
1195 return PD.RegionCounters;
1199 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1200 PD.RegionCounters = CounterPtr;
1207 Metadata *FunctionNameAnnotation[] = {
1215 Metadata *NumCountersAnnotation[] = {
1224 auto *DICounter =
DB.createGlobalVariableExpression(
1225 SP, CounterPtr->getName(),
StringRef(), SP->getFile(),
1226 0,
DB.createUnspecifiedType(
"Profile Data Type"),
1227 CounterPtr->hasLocalLinkage(),
true,
nullptr,
1228 nullptr,
nullptr, 0,
1230 CounterPtr->addDebugInfo(DICounter);
1235 CompilerUsedVars.push_back(
PD.RegionCounters);
1239 createDataVariable(Inc);
1241 return PD.RegionCounters;
1251 auto &
PD = ProfileDataMap[NamePtr];
1278 std::string CntsVarName =
1280 std::string DataVarName =
1289 NS +=
PD.NumValueSites[Kind];
1290 if (NS > 0 && ValueProfileStaticAlloc &&
1296 ValuesVar->setVisibility(Visibility);
1297 ValuesVar->setSection(
1299 ValuesVar->setAlignment(
Align(8));
1300 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1301 ValuesPtrExpr = ValuesVar;
1305 auto *CounterPtr =
PD.RegionCounters;
1310 auto *IntPtrTy =
M->getDataLayout().getIntPtrType(
M->getContext());
1313 Type *DataTypes[] = {
1314#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1321 Constant *Int16ArrayVals[IPVK_Last + 1];
1336 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1343 new GlobalVariable(*M, DataTy,
false, Linkage,
nullptr, DataVarName);
1346 auto *RelativeCounterPtr =
1354 if (BitmapPtr !=
nullptr) {
1361#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1366 Data->setVisibility(Visibility);
1368 Data->setAlignment(
Align(INSTR_PROF_DATA_ALIGNMENT));
1369 maybeSetComdat(
Data, Fn, CntsVarName);
1374 CompilerUsedVars.push_back(
Data);
1380 ReferencedNames.push_back(NamePtr);
1383void InstrProfiling::emitVNodes() {
1384 if (!ValueProfileStaticAlloc)
1394 for (
auto &PD : ProfileDataMap) {
1396 TotalNS +=
PD.second.NumValueSites[Kind];
1402 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1410#define INSTR_PROF_MIN_VAL_COUNTS 10
1414 auto &Ctx =
M->getContext();
1415 Type *VNodeTypes[] = {
1416#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1425 VNodesVar->setSection(
1427 VNodesVar->setAlignment(
M->getDataLayout().getABITypeAlign(VNodesTy));
1430 UsedVars.push_back(VNodesVar);
1433void InstrProfiling::emitNameData() {
1434 std::string UncompressedData;
1436 if (ReferencedNames.empty())
1439 std::string CompressedNameStr;
1445 auto &Ctx =
M->getContext();
1451 NamesSize = CompressedNameStr.size();
1460 UsedVars.push_back(NamesVar);
1462 for (
auto *NamePtr : ReferencedNames)
1466void InstrProfiling::emitRegistration() {
1479 RegisterF->addFnAttr(Attribute::NoRedZone);
1482 auto *RuntimeRegisterF =
1488 if (!isa<Function>(
Data))
1489 IRB.CreateCall(RuntimeRegisterF,
Data);
1491 if (
Data != NamesVar && !isa<Function>(
Data))
1492 IRB.CreateCall(RuntimeRegisterF,
Data);
1495 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1496 auto *NamesRegisterTy =
1498 auto *NamesRegisterF =
1501 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
1504 IRB.CreateRetVoid();
1507bool InstrProfiling::emitRuntimeHook() {
1526 CompilerUsedVars.push_back(Var);
1532 User->addFnAttr(Attribute::NoInline);
1534 User->addFnAttr(Attribute::NoRedZone);
1541 IRB.CreateRet(Load);
1544 CompilerUsedVars.push_back(
User);
1549void InstrProfiling::emitUses() {
1571void InstrProfiling::emitInitialization() {
1588 F->addFnAttr(Attribute::NoInline);
1590 F->addFnAttr(Attribute::NoRedZone);
1594 IRB.CreateCall(RegisterF, {});
1595 IRB.CreateRetVoid();
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool enablesValueProfiling(const Module &M)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag)
static bool profDataReferencedByCode(const Module &M)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
Module.h This file contains the declarations for the Module class.
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Annotations lets you mark points and ranges inside source code, for tests:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
@ NoDeduplicate
No deduplication is performed.
ConstantArray - Constant Array Declarations.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
bool isZeroValue() const
Return true if the value is negative zero or null value.
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Lightweight error class with error context and mandatory checking.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
void setComdat(Comdat *C)
void setSection(StringRef S)
Change the section for this global.
VisibilityTypes getVisibility() const
LinkageTypes getLinkage() const
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
void setLinkage(LinkageTypes LT)
bool isDeclarationForLinker() const
VisibilityTypes
An enumeration for the kinds of visibility of global values.
@ DefaultVisibility
The GV is visible.
@ HiddenVisibility
The GV is hidden.
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
ConstantInt * getNumBitmapBytes() const
This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic.
ConstantInt * getCondID() const
Value * getCondBool() const
Value * getMCDCCondBitmapAddr() const
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
Value * getMCDCCondBitmapAddr() const
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
Value * getTargetValue() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
An instruction for reading from memory.
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Helper class for SSA formation on a set of values defined in multiple blocks.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
constexpr size_t size() const
size - Get the string size.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
bool isOSBinFormatXCOFF() const
Tests whether the OS uses the XCOFF binary format.
bool isOSLinux() const
Tests whether the OS is Linux.
bool isOSAIX() const
Tests whether the OS is AIX.
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
Linkage
Describes symbol linkage. This can be used to resolve definition clashes.
This is an optimization pass for GlobalISel generic memory operations.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
cl::opt< bool > DoInstrProfNameCompression
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
StringRef getInstrProfCounterBiasVarName()
auto reverse(ContainerTy &&C)
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
cl::opt< bool > DebugInfoCorrelate
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
This struct is a compact representation of a valid (non-zero power of two) alignment.
std::string InstrProfileOutput
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.