64#define DEBUG_TYPE "instrprof"
70 "debug-info-correlate",
71 cl::desc(
"Use debug info to correlate profiles. (Deprecated, use "
72 "-profile-correlate=debug-info)"),
77 cl::desc(
"Use debug info or binary file to correlate profiles."),
80 "No profile correlation"),
82 "Use debug info to correlate"),
84 "Use binary to correlate")));
90 "hash-based-counter-split",
91 cl::desc(
"Rename counter variable of a comdat function based on cfg hash"),
95 RuntimeCounterRelocation(
"runtime-counter-relocation",
96 cl::desc(
"Enable relocating counters at runtime."),
101 cl::desc(
"Do static counter allocation for value profiler"),
105 "vp-counters-per-site",
106 cl::desc(
"The average number of profile counters allocated "
107 "per value profiling site."),
115 "instrprof-atomic-counter-update-all",
116 cl::desc(
"Make all profile counter updates atomic (for testing only)"),
120 "atomic-counter-update-promoted",
121 cl::desc(
"Do counter update using atomic fetch add "
122 " for promoted counters only"),
126 "atomic-first-counter",
127 cl::desc(
"Use atomic fetch add for first counter in a function (usually "
128 "the entry counter)"),
137 cl::desc(
"Do counter register promotion"),
140 "max-counter-promotions-per-loop",
cl::init(20),
141 cl::desc(
"Max number counter promotions per loop to avoid"
142 " increasing register pressure too much"));
146 MaxNumOfPromotions(
"max-counter-promotions",
cl::init(-1),
147 cl::desc(
"Max number of allowed counter promotions"));
150 "speculative-counter-promotion-max-exiting",
cl::init(3),
151 cl::desc(
"The max number of exiting blocks of a loop to allow "
152 " speculative counter promotion"));
155 "speculative-counter-promotion-to-loop",
156 cl::desc(
"When the option is false, if the target block is in a loop, "
157 "the promotion will be disallowed unless the promoted counter "
158 " update can be further/iteratively promoted into an acyclic "
162 "iterative-counter-promotion",
cl::init(
true),
163 cl::desc(
"Allow counter promotion across the whole loop nest."));
166 "skip-ret-exit-block",
cl::init(
true),
167 cl::desc(
"Suppress counter promotion if exit blocks contain ret."));
169using LoadStorePair = std::pair<Instruction *, Instruction *>;
172 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
178 return cast<ConstantInt>(MD->getValue())->getZExtValue();
181static bool enablesValueProfiling(
const Module &M) {
183 getIntModuleFlagOrZero(M,
"EnableValueProfiling") != 0;
187static bool profDataReferencedByCode(
const Module &M) {
188 return enablesValueProfiling(M);
191class InstrLowerer final {
197 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(
M)) {}
210 const bool DataReferencedByCode;
212 struct PerFunctionProfileData {
213 uint32_t NumValueSites[IPVK_Last + 1] = {};
219 PerFunctionProfileData() =
default;
225 std::vector<GlobalValue *> CompilerUsedVars;
226 std::vector<GlobalValue *> UsedVars;
227 std::vector<GlobalVariable *> ReferencedNames;
229 size_t NamesSize = 0;
232 std::vector<LoadStorePair> PromotionCandidates;
234 int64_t TotalCountersPromoted = 0;
241 void promoteCounterLoadStores(
Function *
F);
244 bool isRuntimeCounterRelocationEnabled()
const;
247 bool isCounterPromotionEnabled()
const;
327 void emitRegistration();
331 bool emitRuntimeHook();
338 void emitInitialization();
350 PGOCounterPromoterHelper(
357 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
359 assert(isa<StoreInst>(S));
360 SSA.AddAvailableValue(PH,
Init);
364 for (
unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
370 Value *LiveInValue =
SSA.GetValueInMiddleOfBlock(ExitBlock);
371 Value *
Addr = cast<StoreInst>(Store)->getPointerOperand();
374 if (
auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(
Addr)) {
381 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
382 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
383 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
384 Addr = Builder.CreateIntToPtr(BiasInst,
387 if (AtomicCounterUpdatePromoted)
392 AtomicOrdering::SequentiallyConsistent);
394 LoadInst *OldVal = Builder.CreateLoad(Ty,
Addr,
"pgocount.promoted");
395 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
396 auto *NewStore = Builder.CreateStore(NewVal,
Addr);
399 if (IterativeCounterPromotion) {
400 auto *TargetLoop = LI.getLoopFor(ExitBlock);
402 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
419class PGOCounterPromoter {
424 : LoopToCandidates(LoopToCands),
L(CurLoop), LI(LI),
BFI(
BFI) {
431 L.getExitBlocks(LoopExitBlocks);
432 if (!isPromotionPossible(&L, LoopExitBlocks))
435 for (
BasicBlock *ExitBlock : LoopExitBlocks) {
436 if (BlockSet.
insert(ExitBlock).second &&
440 ExitBlocks.push_back(ExitBlock);
446 bool run(int64_t *NumPromoted) {
448 if (ExitBlocks.size() == 0)
456 if (SkipRetExitBlock) {
457 for (
auto *BB : ExitBlocks)
458 if (isa<ReturnInst>(BB->getTerminator()))
462 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
466 unsigned Promoted = 0;
467 for (
auto &Cand : LoopToCandidates[&L]) {
471 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
475 auto *BB = Cand.first->getParent();
479 auto PreheaderCount =
BFI->getBlockProfileCount(
L.getLoopPreheader());
482 if (PreheaderCount && (*PreheaderCount * 3) >= (*
InstrCount * 2))
486 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second,
SSA, InitVal,
487 L.getLoopPreheader(), ExitBlocks,
488 InsertPts, LoopToCandidates, LI);
491 if (Promoted >= MaxProm)
495 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
499 LLVM_DEBUG(
dbgs() << Promoted <<
" counters promoted for loop (depth="
500 <<
L.getLoopDepth() <<
")\n");
501 return Promoted != 0;
505 bool allowSpeculativeCounterPromotion(
Loop *LP) {
507 L.getExitingBlocks(ExitingBlocks);
509 if (ExitingBlocks.
size() == 1)
511 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
519 isPromotionPossible(
Loop *LP,
523 return isa<CatchSwitchInst>(Exit->getTerminator());
538 unsigned getMaxNumOfPromotionsInLoop(
Loop *LP) {
541 if (!isPromotionPossible(LP, LoopExitBlocks))
552 if (ExitingBlocks.
size() == 1)
553 return MaxNumOfPromotionsPerLoop;
555 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
559 if (SpeculativeCounterPromotionToLoop)
560 return MaxNumOfPromotionsPerLoop;
563 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
564 for (
auto *TargetBlock : LoopExitBlocks) {
565 auto *TargetLoop = LI.
getLoopFor(TargetBlock);
568 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
569 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
571 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
572 PendingCandsInTarget);
585enum class ValueProfilingCallType {
603 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
604 if (!Lowerer.lower())
610bool InstrLowerer::lowerIntrinsics(
Function *
F) {
611 bool MadeChange =
false;
612 PromotionCandidates.clear();
615 if (
auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
616 lowerIncrement(IPIS);
618 }
else if (
auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
621 }
else if (
auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
624 }
else if (
auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
627 }
else if (
auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
628 lowerValueProfileInst(IPVP);
630 }
else if (
auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
631 IPMP->eraseFromParent();
633 }
else if (
auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
634 lowerMCDCTestVectorBitmapUpdate(IPBU);
636 }
else if (
auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(&Instr)) {
637 lowerMCDCCondBitmapUpdate(IPTU);
646 promoteCounterLoadStores(
F);
650bool InstrLowerer::isRuntimeCounterRelocationEnabled()
const {
652 if (
TT.isOSBinFormatMachO())
655 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
656 return RuntimeCounterRelocation;
659 return TT.isOSFuchsia();
662bool InstrLowerer::isCounterPromotionEnabled()
const {
663 if (DoCounterPromotion.getNumOccurrences() > 0)
664 return DoCounterPromotion;
666 return Options.DoCounterPromotion;
669void InstrLowerer::promoteCounterLoadStores(
Function *
F) {
670 if (!isCounterPromotionEnabled())
677 std::unique_ptr<BlockFrequencyInfo>
BFI;
678 if (
Options.UseBFIInPromotion) {
679 std::unique_ptr<BranchProbabilityInfo> BPI;
684 for (
const auto &LoadStore : PromotionCandidates) {
691 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
699 PGOCounterPromoter Promoter(LoopPromotionCandidates, *
Loop, LI,
BFI.get());
700 Promoter.run(&TotalCountersPromoted);
706 if (TT.isOSFuchsia())
714 auto containsIntrinsic = [&](
int ID) {
716 return !
F->use_empty();
719 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
720 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
721 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
722 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
723 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
726bool InstrLowerer::lower() {
727 bool MadeChange =
false;
729 if (NeedsRuntimeHook)
730 MadeChange = emitRuntimeHook();
736 if (!ContainsProfiling && !CoverageNamesVar)
746 if (
auto *Ind = dyn_cast<InstrProfValueProfileInst>(
I))
747 computeNumValueSiteCounts(Ind);
749 if (FirstProfInst ==
nullptr &&
750 (isa<InstrProfIncrementInst>(
I) || isa<InstrProfCoverInst>(
I)))
751 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(
I);
753 if (
const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(
I))
754 static_cast<void>(getOrCreateRegionBitmaps(Params));
761 if (FirstProfInst !=
nullptr) {
762 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
769 if (CoverageNamesVar) {
770 lowerCoverageData(CoverageNamesVar);
784 if (!NeedsRuntimeHook && ContainsProfiling)
789 emitInitialization();
795 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
800 if (
auto AK = TLI.getExtAttrForI32Param(
false))
801 AL = AL.addParamAttribute(M.getContext(), 2, AK);
803 assert((CallType == ValueProfilingCallType::Default ||
804 CallType == ValueProfilingCallType::MemOp) &&
805 "Must be Default or MemOp");
806 Type *ParamTypes[] = {
807#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
810 auto *ValueProfilingCallTy =
812 StringRef FuncName = CallType == ValueProfilingCallType::Default
815 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
822 auto &
PD = ProfileDataMap[
Name];
833 "Value profiling is not yet supported with lightweight instrumentation");
835 auto It = ProfileDataMap.find(
Name);
836 assert(It != ProfileDataMap.end() && It->second.DataVar &&
837 "value profiling detected in function with no counter incerement");
843 Index += It->second.NumValueSites[Kind];
847 llvm::InstrProfValueKind::IPVK_MemOPSize);
863 Call = Builder.CreateCall(
867 if (
auto AK = TLI->getExtAttrForI32Param(
false))
868 Call->addParamAttr(2, AK);
874 auto *Counters = getOrCreateRegionCounters(
I);
877 if (isa<InstrProfTimestampInst>(
I))
878 Counters->setAlignment(
Align(8));
880 auto *
Addr = Builder.CreateConstInBoundsGEP2_32(
881 Counters->getValueType(), Counters, 0,
I->getIndex()->getZExtValue());
883 if (!isRuntimeCounterRelocationEnabled())
887 Function *Fn =
I->getParent()->getParent();
888 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
904 if (
TT.supportsCOMDAT())
905 Bias->setComdat(
M.getOrInsertComdat(Bias->getName()));
907 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
909 auto *
Add = Builder.CreateAdd(Builder.CreatePtrToInt(
Addr, Int64Ty), BiasLI);
910 return Builder.CreateIntToPtr(
Add,
Addr->getType());
914 auto *Bitmaps = getOrCreateRegionBitmaps(
I);
917 auto *
Addr = Builder.CreateConstInBoundsGEP2_32(
918 Bitmaps->getValueType(), Bitmaps, 0,
I->getBitmapIndex()->getZExtValue());
920 if (isRuntimeCounterRelocationEnabled()) {
924 Twine(
"Runtime counter relocation is presently not supported for MC/DC "
933 auto *
Addr = getCounterAddress(CoverInstruction);
936 Builder.CreateStore(Builder.getInt8(0),
Addr);
940void InstrLowerer::lowerTimestamp(
943 "timestamp probes are always the first probe for a function");
944 auto &Ctx =
M.getContext();
945 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
949 auto Callee =
M.getOrInsertFunction(
950 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
951 Builder.CreateCall(Callee, {TimestampAddr});
956 auto *
Addr = getCounterAddress(Inc);
959 if (
Options.Atomic || AtomicCounterUpdateAll ||
966 auto *Count = Builder.CreateAdd(Load, Inc->
getStep());
967 auto *
Store = Builder.CreateStore(Count,
Addr);
968 if (isCounterPromotionEnabled())
969 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
974void InstrLowerer::lowerCoverageData(
GlobalVariable *CoverageNamesVar) {
980 assert(isa<GlobalVariable>(V) &&
"Missing reference to function name");
984 ReferencedNames.push_back(
Name);
985 if (isa<ConstantExpr>(
NC))
986 NC->dropAllReferences();
991void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
997 auto *BitmapAddr = getBitmapAddress(Update);
1001 auto *Temp = Builder.CreateLoad(
Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp");
1005 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
1009 auto *BitmapByteAddr =
1010 Builder.CreateInBoundsPtrAdd(BitmapAddr, BitmapByteOffset);
1015 auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
1019 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
1023 auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr,
"mcdc.bits");
1027 auto *
Result = Builder.CreateOr(Bitmap, ShiftedVal);
1031 Builder.CreateStore(Result, BitmapByteAddr);
1035void InstrLowerer::lowerMCDCCondBitmapUpdate(
1043 auto *Temp = Builder.CreateLoad(
Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp");
1051 auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->
getCondID());
1055 auto *
Result = Builder.CreateOr(Temp, ShiftedVal);
1059 Builder.CreateStore(Result, MCDCCondBitmapAddr);
1073 return (Prefix +
Name).str();
1079 return (Prefix +
Name).str();
1080 return (Prefix +
Name +
"." +
Twine(FuncHash)).str();
1088 if (!profDataReferencedByCode(*
F->getParent()))
1092 bool HasAvailableExternallyLinkage =
F->hasAvailableExternallyLinkage();
1093 if (!
F->hasLinkOnceLinkage() && !
F->hasLocalLinkage() &&
1094 !HasAvailableExternallyLinkage)
1100 if (HasAvailableExternallyLinkage &&
1101 F->hasFnAttribute(Attribute::AlwaysInline))
1107 if (
F->hasLocalLinkage() &&
F->hasComdat())
1117 return F->hasAddressTaken() ||
F->hasLinkOnceLinkage();
1164 Fn->
getName() +
".local", Fn);
1189 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1190 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
1203 bool UseComdat = (NeedComdat ||
TT.isOSBinFormatELF());
1218 StringRef GroupName =
TT.isOSBinFormatCOFF() && DataReferencedByCode
1221 Comdat *
C =
M.getOrInsertComdat(GroupName);
1261 if (
TT.isOSBinFormatXCOFF()) {
1270 if (IPSK == IPSK_cnts) {
1274 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1275 }
else if (IPSK == IPSK_bitmap) {
1279 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1280 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1285 Ptr->setVisibility(Visibility);
1289 Ptr->setLinkage(Linkage);
1290 maybeSetComdat(
Ptr, Fn, VarName);
1309 auto &
PD = ProfileDataMap[NamePtr];
1310 if (
PD.RegionBitmaps)
1311 return PD.RegionBitmaps;
1315 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1316 PD.RegionBitmaps = BitmapPtr;
1318 return PD.RegionBitmaps;
1325 auto &Ctx =
M.getContext();
1327 if (isa<InstrProfCoverInst>(Inc)) {
1331 std::vector<Constant *> InitialValues(NumCounters,
1349 auto &
PD = ProfileDataMap[NamePtr];
1350 if (
PD.RegionCounters)
1351 return PD.RegionCounters;
1355 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1356 PD.RegionCounters = CounterPtr;
1364 Metadata *FunctionNameAnnotation[] = {
1372 Metadata *NumCountersAnnotation[] = {
1381 auto *DICounter =
DB.createGlobalVariableExpression(
1382 SP, CounterPtr->getName(),
StringRef(), SP->getFile(),
1383 0,
DB.createUnspecifiedType(
"Profile Data Type"),
1384 CounterPtr->hasLocalLinkage(),
true,
nullptr,
1385 nullptr,
nullptr, 0,
1387 CounterPtr->addDebugInfo(DICounter);
1392 CompilerUsedVars.push_back(
PD.RegionCounters);
1396 createDataVariable(Inc);
1398 return PD.RegionCounters;
1408 auto &
PD = ProfileDataMap[NamePtr];
1425 if (
TT.isOSBinFormatXCOFF()) {
1434 std::string CntsVarName =
1436 std::string DataVarName =
1445 NS +=
PD.NumValueSites[Kind];
1446 if (NS > 0 && ValueProfileStaticAlloc &&
1452 ValuesVar->setVisibility(Visibility);
1454 ValuesVar->setSection(
1456 ValuesVar->setAlignment(
Align(8));
1457 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1458 ValuesPtrExpr = ValuesVar;
1462 auto *CounterPtr =
PD.RegionCounters;
1467 auto *IntPtrTy =
M.getDataLayout().getIntPtrType(
M.getContext());
1470 Type *DataTypes[] = {
1471#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1478 Constant *Int16ArrayVals[IPVK_Last + 1];
1480 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty,
PD.NumValueSites[Kind]);
1493 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1494 (
TT.isOSBinFormatELF() ||
1495 (!DataReferencedByCode &&
TT.isOSBinFormatCOFF()))) {
1500 new GlobalVariable(M, DataTy,
false, Linkage,
nullptr, DataVarName);
1503 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
1508 DataSectionKind = IPSK_covdata;
1510 if (BitmapPtr !=
nullptr)
1515 DataSectionKind = IPSK_data;
1516 RelativeCounterPtr =
1519 if (BitmapPtr !=
nullptr)
1526#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1531 Data->setVisibility(Visibility);
1534 Data->setAlignment(
Align(INSTR_PROF_DATA_ALIGNMENT));
1535 maybeSetComdat(
Data, Fn, CntsVarName);
1540 CompilerUsedVars.push_back(
Data);
1546 ReferencedNames.push_back(NamePtr);
1549void InstrLowerer::emitVNodes() {
1550 if (!ValueProfileStaticAlloc)
1560 for (
auto &PD : ProfileDataMap) {
1562 TotalNS +=
PD.second.NumValueSites[Kind];
1568 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1576#define INSTR_PROF_MIN_VAL_COUNTS 10
1580 auto &Ctx =
M.getContext();
1581 Type *VNodeTypes[] = {
1582#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1592 VNodesVar->setSection(
1594 VNodesVar->setAlignment(
M.getDataLayout().getABITypeAlign(VNodesTy));
1597 UsedVars.push_back(VNodesVar);
1600void InstrLowerer::emitNameData() {
1601 std::string UncompressedData;
1603 if (ReferencedNames.empty())
1606 std::string CompressedNameStr;
1612 auto &Ctx =
M.getContext();
1618 NamesSize = CompressedNameStr.size();
1620 NamesVar->setSection(
1627 NamesVar->setAlignment(
Align(1));
1630 UsedVars.push_back(NamesVar);
1632 for (
auto *NamePtr : ReferencedNames)
1636void InstrLowerer::emitRegistration() {
1649 RegisterF->addFnAttr(Attribute::NoRedZone);
1652 auto *RuntimeRegisterF =
1658 if (!isa<Function>(
Data))
1659 IRB.CreateCall(RuntimeRegisterF,
Data);
1661 if (
Data != NamesVar && !isa<Function>(
Data))
1662 IRB.CreateCall(RuntimeRegisterF,
Data);
1665 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1666 auto *NamesRegisterTy =
1668 auto *NamesRegisterF =
1671 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
1674 IRB.CreateRetVoid();
1677bool InstrLowerer::emitRuntimeHook() {
1680 if (
TT.isOSLinux() ||
TT.isOSAIX())
1694 if (
TT.isOSBinFormatELF() && !
TT.isPS()) {
1696 CompilerUsedVars.push_back(Var);
1702 User->addFnAttr(Attribute::NoInline);
1704 User->addFnAttr(Attribute::NoRedZone);
1706 if (
TT.supportsCOMDAT())
1711 IRB.CreateRet(Load);
1714 CompilerUsedVars.push_back(
User);
1719void InstrLowerer::emitUses() {
1729 if (
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO() ||
1730 (
TT.isOSBinFormatCOFF() && !DataReferencedByCode))
1741void InstrLowerer::emitInitialization() {
1758 F->addFnAttr(Attribute::NoInline);
1760 F->addFnAttr(Attribute::NoRedZone);
1764 IRB.CreateCall(RegisterF, {});
1765 IRB.CreateRetVoid();
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
static bool lowerIntrinsics(Module &M)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
Module.h This file contains the declarations for the Module class.
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Annotations lets you mark points and ranges inside source code, for tests:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
const Function * getParent() const
Return the enclosing method, or null if none.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
@ NoDeduplicate
No deduplication is performed.
ConstantArray - Constant Array Declarations.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
bool isZeroValue() const
Return true if the value is negative zero or null value.
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Lightweight error class with error context and mandatory checking.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
void setComdat(Comdat *C)
VisibilityTypes getVisibility() const
LinkageTypes getLinkage() const
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
void setLinkage(LinkageTypes LT)
bool isDeclarationForLinker() const
VisibilityTypes
An enumeration for the kinds of visibility of global values.
@ DefaultVisibility
The GV is visible.
@ HiddenVisibility
The GV is hidden.
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
ConstantInt * getNumBitmapBytes() const
This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic.
ConstantInt * getCondID() const
Value * getCondBool() const
Value * getMCDCCondBitmapAddr() const
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
Value * getMCDCCondBitmapAddr() const
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
Value * getTargetValue() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
An instruction for reading from memory.
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Helper class for SSA formation on a set of values defined in multiple blocks.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
constexpr size_t size() const
size - Get the string size.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
Linkage
Describes symbol linkage. This can be used to resolve definition clashes.
This is an optimization pass for GlobalISel generic memory operations.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
cl::opt< bool > DoInstrProfNameCompression
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
cl::opt< bool > DebugInfoCorrelate
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
StringRef getInstrProfCounterBiasVarName()
auto reverse(ContainerTy &&C)
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
void setGlobalVariableLargeSection(const Triple &TargetTriple, GlobalVariable &GV)
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
This struct is a compact representation of a valid (non-zero power of two) alignment.
Options for the frontend instrumentation based profiling pass.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.