64#define DEBUG_TYPE "instrprof"
73 "debug-info-correlate",
74 cl::desc(
"Use debug info to correlate profiles. (Deprecated, use "
75 "-profile-correlate=debug-info)"),
80 cl::desc(
"Use debug info or binary file to correlate profiles."),
83 "No profile correlation"),
85 "Use debug info to correlate"),
87 "Use binary to correlate")));
93 "hash-based-counter-split",
94 cl::desc(
"Rename counter variable of a comdat function based on cfg hash"),
98 RuntimeCounterRelocation(
"runtime-counter-relocation",
99 cl::desc(
"Enable relocating counters at runtime."),
104 cl::desc(
"Do static counter allocation for value profiler"),
108 "vp-counters-per-site",
109 cl::desc(
"The average number of profile counters allocated "
110 "per value profiling site."),
118 "instrprof-atomic-counter-update-all",
119 cl::desc(
"Make all profile counter updates atomic (for testing only)"),
123 "atomic-counter-update-promoted",
124 cl::desc(
"Do counter update using atomic fetch add "
125 " for promoted counters only"),
129 "atomic-first-counter",
130 cl::desc(
"Use atomic fetch add for first counter in a function (usually "
131 "the entry counter)"),
140 cl::desc(
"Do counter register promotion"),
143 "max-counter-promotions-per-loop",
cl::init(20),
144 cl::desc(
"Max number counter promotions per loop to avoid"
145 " increasing register pressure too much"));
149 MaxNumOfPromotions(
"max-counter-promotions",
cl::init(-1),
150 cl::desc(
"Max number of allowed counter promotions"));
153 "speculative-counter-promotion-max-exiting",
cl::init(3),
154 cl::desc(
"The max number of exiting blocks of a loop to allow "
155 " speculative counter promotion"));
158 "speculative-counter-promotion-to-loop",
159 cl::desc(
"When the option is false, if the target block is in a loop, "
160 "the promotion will be disallowed unless the promoted counter "
161 " update can be further/iteratively promoted into an acyclic "
165 "iterative-counter-promotion",
cl::init(
true),
166 cl::desc(
"Allow counter promotion across the whole loop nest."));
169 "skip-ret-exit-block",
cl::init(
true),
170 cl::desc(
"Suppress counter promotion if exit blocks contain ret."));
172using LoadStorePair = std::pair<Instruction *, Instruction *>;
175 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
181 return cast<ConstantInt>(MD->getValue())->getZExtValue();
184static bool enablesValueProfiling(
const Module &M) {
186 getIntModuleFlagOrZero(M,
"EnableValueProfiling") != 0;
190static bool profDataReferencedByCode(
const Module &M) {
191 return enablesValueProfiling(M);
194class InstrLowerer final {
200 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(
M)) {}
213 const bool DataReferencedByCode;
215 struct PerFunctionProfileData {
216 uint32_t NumValueSites[IPVK_Last + 1] = {};
222 PerFunctionProfileData() =
default;
231 std::vector<GlobalValue *> CompilerUsedVars;
232 std::vector<GlobalValue *> UsedVars;
233 std::vector<GlobalVariable *> ReferencedNames;
236 std::vector<GlobalVariable *> ReferencedVTables;
238 size_t NamesSize = 0;
241 std::vector<LoadStorePair> PromotionCandidates;
243 int64_t TotalCountersPromoted = 0;
250 void promoteCounterLoadStores(
Function *
F);
253 bool isRuntimeCounterRelocationEnabled()
const;
256 bool isCounterPromotionEnabled()
const;
336 void emitVTableNames();
342 void emitRegistration();
346 bool emitRuntimeHook();
353 void emitInitialization();
365 PGOCounterPromoterHelper(
372 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
374 assert(isa<StoreInst>(S));
375 SSA.AddAvailableValue(PH,
Init);
379 for (
unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
385 Value *LiveInValue =
SSA.GetValueInMiddleOfBlock(ExitBlock);
386 Value *
Addr = cast<StoreInst>(Store)->getPointerOperand();
389 if (
auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(
Addr)) {
396 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
397 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
398 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
399 Addr = Builder.CreateIntToPtr(BiasInst,
402 if (AtomicCounterUpdatePromoted)
407 AtomicOrdering::SequentiallyConsistent);
409 LoadInst *OldVal = Builder.CreateLoad(Ty,
Addr,
"pgocount.promoted");
410 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
411 auto *NewStore = Builder.CreateStore(NewVal,
Addr);
414 if (IterativeCounterPromotion) {
415 auto *TargetLoop = LI.getLoopFor(ExitBlock);
417 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
434class PGOCounterPromoter {
439 : LoopToCandidates(LoopToCands),
L(CurLoop), LI(LI),
BFI(
BFI) {
446 L.getExitBlocks(LoopExitBlocks);
447 if (!isPromotionPossible(&L, LoopExitBlocks))
450 for (
BasicBlock *ExitBlock : LoopExitBlocks) {
451 if (BlockSet.
insert(ExitBlock).second &&
455 ExitBlocks.push_back(ExitBlock);
461 bool run(int64_t *NumPromoted) {
463 if (ExitBlocks.size() == 0)
471 if (SkipRetExitBlock) {
472 for (
auto *BB : ExitBlocks)
473 if (isa<ReturnInst>(BB->getTerminator()))
477 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
481 unsigned Promoted = 0;
482 for (
auto &Cand : LoopToCandidates[&L]) {
486 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
490 auto *BB = Cand.first->getParent();
494 auto PreheaderCount =
BFI->getBlockProfileCount(
L.getLoopPreheader());
497 if (PreheaderCount && (*PreheaderCount * 3) >= (*
InstrCount * 2))
501 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second,
SSA, InitVal,
502 L.getLoopPreheader(), ExitBlocks,
503 InsertPts, LoopToCandidates, LI);
506 if (Promoted >= MaxProm)
510 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
514 LLVM_DEBUG(
dbgs() << Promoted <<
" counters promoted for loop (depth="
515 <<
L.getLoopDepth() <<
")\n");
516 return Promoted != 0;
520 bool allowSpeculativeCounterPromotion(
Loop *LP) {
522 L.getExitingBlocks(ExitingBlocks);
524 if (ExitingBlocks.
size() == 1)
526 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
534 isPromotionPossible(
Loop *LP,
538 return isa<CatchSwitchInst>(
Exit->getTerminator());
553 unsigned getMaxNumOfPromotionsInLoop(
Loop *LP) {
556 if (!isPromotionPossible(LP, LoopExitBlocks))
567 if (ExitingBlocks.
size() == 1)
568 return MaxNumOfPromotionsPerLoop;
570 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
574 if (SpeculativeCounterPromotionToLoop)
575 return MaxNumOfPromotionsPerLoop;
578 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
579 for (
auto *TargetBlock : LoopExitBlocks) {
580 auto *TargetLoop = LI.
getLoopFor(TargetBlock);
583 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
584 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
586 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
587 PendingCandsInTarget);
600enum class ValueProfilingCallType {
618 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
619 if (!Lowerer.lower())
625bool InstrLowerer::lowerIntrinsics(
Function *
F) {
626 bool MadeChange =
false;
627 PromotionCandidates.clear();
630 if (
auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
631 lowerIncrement(IPIS);
633 }
else if (
auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
636 }
else if (
auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
639 }
else if (
auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
642 }
else if (
auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
643 lowerValueProfileInst(IPVP);
645 }
else if (
auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
646 IPMP->eraseFromParent();
648 }
else if (
auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
649 lowerMCDCTestVectorBitmapUpdate(IPBU);
651 }
else if (
auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(&Instr)) {
652 lowerMCDCCondBitmapUpdate(IPTU);
661 promoteCounterLoadStores(
F);
665bool InstrLowerer::isRuntimeCounterRelocationEnabled()
const {
667 if (
TT.isOSBinFormatMachO())
670 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
671 return RuntimeCounterRelocation;
674 return TT.isOSFuchsia();
677bool InstrLowerer::isCounterPromotionEnabled()
const {
678 if (DoCounterPromotion.getNumOccurrences() > 0)
679 return DoCounterPromotion;
681 return Options.DoCounterPromotion;
684void InstrLowerer::promoteCounterLoadStores(
Function *
F) {
685 if (!isCounterPromotionEnabled())
692 std::unique_ptr<BlockFrequencyInfo>
BFI;
693 if (
Options.UseBFIInPromotion) {
694 std::unique_ptr<BranchProbabilityInfo> BPI;
699 for (
const auto &LoadStore : PromotionCandidates) {
706 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
714 PGOCounterPromoter Promoter(LoopPromotionCandidates, *
Loop, LI,
BFI.get());
715 Promoter.run(&TotalCountersPromoted);
721 if (TT.isOSFuchsia())
729 auto containsIntrinsic = [&](
int ID) {
731 return !
F->use_empty();
734 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
735 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
736 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
737 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
738 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
741bool InstrLowerer::lower() {
742 bool MadeChange =
false;
744 if (NeedsRuntimeHook)
745 MadeChange = emitRuntimeHook();
751 if (!ContainsProfiling && !CoverageNamesVar)
760 for (
auto I = BB.
begin(), E = BB.
end();
I != E;
I++) {
761 if (
auto *Ind = dyn_cast<InstrProfValueProfileInst>(
I))
762 computeNumValueSiteCounts(Ind);
764 if (FirstProfInst ==
nullptr &&
765 (isa<InstrProfIncrementInst>(
I) || isa<InstrProfCoverInst>(
I)))
766 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(
I);
768 if (
const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(
I))
769 static_cast<void>(getOrCreateRegionBitmaps(Params));
776 if (FirstProfInst !=
nullptr) {
777 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
784 if (GV.hasMetadata(LLVMContext::MD_type))
785 getOrCreateVTableProfData(&GV);
790 if (CoverageNamesVar) {
791 lowerCoverageData(CoverageNamesVar);
806 if (!NeedsRuntimeHook && ContainsProfiling)
811 emitInitialization();
817 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
822 if (
auto AK = TLI.getExtAttrForI32Param(
false))
823 AL = AL.addParamAttribute(M.getContext(), 2, AK);
825 assert((CallType == ValueProfilingCallType::Default ||
826 CallType == ValueProfilingCallType::MemOp) &&
827 "Must be Default or MemOp");
828 Type *ParamTypes[] = {
829#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
832 auto *ValueProfilingCallTy =
834 StringRef FuncName = CallType == ValueProfilingCallType::Default
837 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
844 auto &
PD = ProfileDataMap[
Name];
855 "Value profiling is not yet supported with lightweight instrumentation");
857 auto It = ProfileDataMap.find(
Name);
858 assert(It != ProfileDataMap.end() && It->second.DataVar &&
859 "value profiling detected in function with no counter incerement");
865 Index += It->second.NumValueSites[Kind];
869 llvm::InstrProfValueKind::IPVK_MemOPSize);
885 Call = Builder.CreateCall(
889 if (
auto AK = TLI->getExtAttrForI32Param(
false))
890 Call->addParamAttr(2, AK);
896 auto *Counters = getOrCreateRegionCounters(
I);
899 if (isa<InstrProfTimestampInst>(
I))
900 Counters->setAlignment(
Align(8));
902 auto *
Addr = Builder.CreateConstInBoundsGEP2_32(
903 Counters->getValueType(), Counters, 0,
I->getIndex()->getZExtValue());
905 if (!isRuntimeCounterRelocationEnabled())
909 Function *Fn =
I->getParent()->getParent();
910 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
926 if (
TT.supportsCOMDAT())
927 Bias->setComdat(
M.getOrInsertComdat(Bias->getName()));
929 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
931 auto *
Add = Builder.CreateAdd(Builder.CreatePtrToInt(
Addr, Int64Ty), BiasLI);
932 return Builder.CreateIntToPtr(
Add,
Addr->getType());
936 auto *Bitmaps = getOrCreateRegionBitmaps(
I);
939 auto *
Addr = Builder.CreateConstInBoundsGEP2_32(
940 Bitmaps->getValueType(), Bitmaps, 0,
I->getBitmapIndex()->getZExtValue());
942 if (isRuntimeCounterRelocationEnabled()) {
946 Twine(
"Runtime counter relocation is presently not supported for MC/DC "
955 auto *
Addr = getCounterAddress(CoverInstruction);
958 Builder.CreateStore(Builder.getInt8(0),
Addr);
962void InstrLowerer::lowerTimestamp(
965 "timestamp probes are always the first probe for a function");
966 auto &Ctx =
M.getContext();
967 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
971 auto Callee =
M.getOrInsertFunction(
972 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
973 Builder.CreateCall(Callee, {TimestampAddr});
978 auto *
Addr = getCounterAddress(Inc);
981 if (
Options.Atomic || AtomicCounterUpdateAll ||
988 auto *Count = Builder.CreateAdd(Load, Inc->
getStep());
989 auto *
Store = Builder.CreateStore(Count,
Addr);
990 if (isCounterPromotionEnabled())
991 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
996void InstrLowerer::lowerCoverageData(
GlobalVariable *CoverageNamesVar) {
1001 Value *
V =
NC->stripPointerCasts();
1002 assert(isa<GlobalVariable>(V) &&
"Missing reference to function name");
1006 ReferencedNames.push_back(
Name);
1007 if (isa<ConstantExpr>(
NC))
1008 NC->dropAllReferences();
1013void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1019 auto *BitmapAddr = getBitmapAddress(Update);
1023 auto *Temp = Builder.CreateLoad(
Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp");
1027 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
1031 auto *BitmapByteAddr =
1032 Builder.CreateInBoundsPtrAdd(BitmapAddr, BitmapByteOffset);
1037 auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
1041 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
1045 auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr,
"mcdc.bits");
1049 auto *
Result = Builder.CreateOr(Bitmap, ShiftedVal);
1053 Builder.CreateStore(Result, BitmapByteAddr);
1057void InstrLowerer::lowerMCDCCondBitmapUpdate(
1065 auto *Temp = Builder.CreateLoad(
Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp");
1073 auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->
getCondID());
1077 auto *
Result = Builder.CreateOr(Temp, ShiftedVal);
1081 Builder.CreateStore(Result, MCDCCondBitmapAddr);
1095 return (Prefix +
Name).str();
1101 return (Prefix +
Name).str();
1102 return (Prefix +
Name +
"." +
Twine(FuncHash)).str();
1110 if (!profDataReferencedByCode(*
F->getParent()))
1114 bool HasAvailableExternallyLinkage =
F->hasAvailableExternallyLinkage();
1115 if (!
F->hasLinkOnceLinkage() && !
F->hasLocalLinkage() &&
1116 !HasAvailableExternallyLinkage)
1122 if (HasAvailableExternallyLinkage &&
1123 F->hasFnAttribute(Attribute::AlwaysInline))
1129 if (
F->hasLocalLinkage() &&
F->hasComdat())
1139 return F->hasAddressTaken() ||
F->hasLinkOnceLinkage();
1186 Fn->
getName() +
".local", Fn);
1211 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1212 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
1225 bool UseComdat = (NeedComdat ||
TT.isOSBinFormatELF());
1240 StringRef GroupName =
TT.isOSBinFormatCOFF() && DataReferencedByCode
1243 Comdat *
C =
M.getOrInsertComdat(GroupName);
1263 if (!profDataReferencedByCode(*GV->
getParent()))
1290void InstrLowerer::getOrCreateVTableProfData(
GlobalVariable *GV) {
1292 "Value profiling is not supported with lightweight instrumentation");
1303 auto It = VTableDataMap.find(GV);
1304 if (It != VTableDataMap.end() && It->second)
1312 if (
TT.isOSBinFormatXCOFF()) {
1318 Type *DataTypes[] = {
1319#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1321#undef INSTR_PROF_VTABLE_DATA
1328 const std::string PGOVTableName =
getPGOName(*GV);
1335#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1337#undef INSTR_PROF_VTABLE_DATA
1345 Data->setVisibility(Visibility);
1349 maybeSetComdat(
Data, GV,
Data->getName());
1351 VTableDataMap[GV] =
Data;
1353 ReferencedVTables.push_back(GV);
1357 UsedVars.push_back(
Data);
1381 if (
TT.isOSBinFormatXCOFF()) {
1390 if (IPSK == IPSK_cnts) {
1394 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1395 }
else if (IPSK == IPSK_bitmap) {
1399 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1400 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1405 Ptr->setVisibility(Visibility);
1409 Ptr->setLinkage(Linkage);
1410 maybeSetComdat(
Ptr, Fn, VarName);
1429 auto &
PD = ProfileDataMap[NamePtr];
1430 if (
PD.RegionBitmaps)
1431 return PD.RegionBitmaps;
1435 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1436 PD.RegionBitmaps = BitmapPtr;
1438 return PD.RegionBitmaps;
1445 auto &Ctx =
M.getContext();
1447 if (isa<InstrProfCoverInst>(Inc)) {
1451 std::vector<Constant *> InitialValues(NumCounters,
1469 auto &
PD = ProfileDataMap[NamePtr];
1470 if (
PD.RegionCounters)
1471 return PD.RegionCounters;
1475 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1476 PD.RegionCounters = CounterPtr;
1484 Metadata *FunctionNameAnnotation[] = {
1492 Metadata *NumCountersAnnotation[] = {
1501 auto *DICounter =
DB.createGlobalVariableExpression(
1502 SP, CounterPtr->getName(),
StringRef(), SP->getFile(),
1503 0,
DB.createUnspecifiedType(
"Profile Data Type"),
1504 CounterPtr->hasLocalLinkage(),
true,
nullptr,
1505 nullptr,
nullptr, 0,
1507 CounterPtr->addDebugInfo(DICounter);
1512 CompilerUsedVars.push_back(
PD.RegionCounters);
1516 createDataVariable(Inc);
1518 return PD.RegionCounters;
1528 auto &
PD = ProfileDataMap[NamePtr];
1545 if (
TT.isOSBinFormatXCOFF()) {
1554 std::string CntsVarName =
1556 std::string DataVarName =
1565 NS +=
PD.NumValueSites[Kind];
1566 if (NS > 0 && ValueProfileStaticAlloc &&
1572 ValuesVar->setVisibility(Visibility);
1574 ValuesVar->setSection(
1576 ValuesVar->setAlignment(
Align(8));
1577 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1578 ValuesPtrExpr = ValuesVar;
1582 auto *CounterPtr =
PD.RegionCounters;
1587 auto *IntPtrTy =
M.getDataLayout().getIntPtrType(
M.getContext());
1590 Type *DataTypes[] = {
1591#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1598 Constant *Int16ArrayVals[IPVK_Last + 1];
1600 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty,
PD.NumValueSites[Kind]);
1613 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1614 (
TT.isOSBinFormatELF() ||
1615 (!DataReferencedByCode &&
TT.isOSBinFormatCOFF()))) {
1620 new GlobalVariable(M, DataTy,
false, Linkage,
nullptr, DataVarName);
1623 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
1628 DataSectionKind = IPSK_covdata;
1630 if (BitmapPtr !=
nullptr)
1635 DataSectionKind = IPSK_data;
1636 RelativeCounterPtr =
1639 if (BitmapPtr !=
nullptr)
1646#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1651 Data->setVisibility(Visibility);
1654 Data->setAlignment(
Align(INSTR_PROF_DATA_ALIGNMENT));
1655 maybeSetComdat(
Data, Fn, CntsVarName);
1660 CompilerUsedVars.push_back(
Data);
1666 ReferencedNames.push_back(NamePtr);
1669void InstrLowerer::emitVNodes() {
1670 if (!ValueProfileStaticAlloc)
1680 for (
auto &PD : ProfileDataMap) {
1682 TotalNS +=
PD.second.NumValueSites[Kind];
1688 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1696#define INSTR_PROF_MIN_VAL_COUNTS 10
1700 auto &Ctx =
M.getContext();
1701 Type *VNodeTypes[] = {
1702#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1712 VNodesVar->setSection(
1714 VNodesVar->setAlignment(
M.getDataLayout().getABITypeAlign(VNodesTy));
1717 UsedVars.push_back(VNodesVar);
1720void InstrLowerer::emitNameData() {
1721 std::string UncompressedData;
1723 if (ReferencedNames.empty())
1726 std::string CompressedNameStr;
1732 auto &Ctx =
M.getContext();
1738 NamesSize = CompressedNameStr.size();
1740 NamesVar->setSection(
1747 NamesVar->setAlignment(
Align(1));
1750 UsedVars.push_back(NamesVar);
1752 for (
auto *NamePtr : ReferencedNames)
1756void InstrLowerer::emitVTableNames() {
1761 std::string CompressedVTableNames;
1767 auto &Ctx =
M.getContext();
1769 Ctx,
StringRef(CompressedVTableNames),
false );
1778 UsedVars.push_back(VTableNamesVar);
1781void InstrLowerer::emitRegistration() {
1794 RegisterF->addFnAttr(Attribute::NoRedZone);
1797 auto *RuntimeRegisterF =
1803 if (!isa<Function>(
Data))
1804 IRB.CreateCall(RuntimeRegisterF,
Data);
1806 if (
Data != NamesVar && !isa<Function>(
Data))
1807 IRB.CreateCall(RuntimeRegisterF,
Data);
1810 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1811 auto *NamesRegisterTy =
1813 auto *NamesRegisterF =
1816 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
1819 IRB.CreateRetVoid();
1822bool InstrLowerer::emitRuntimeHook() {
1825 if (
TT.isOSLinux() ||
TT.isOSAIX())
1839 if (
TT.isOSBinFormatELF() && !
TT.isPS()) {
1841 CompilerUsedVars.push_back(Var);
1847 User->addFnAttr(Attribute::NoInline);
1849 User->addFnAttr(Attribute::NoRedZone);
1851 if (
TT.supportsCOMDAT())
1856 IRB.CreateRet(Load);
1859 CompilerUsedVars.push_back(
User);
1864void InstrLowerer::emitUses() {
1874 if (
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO() ||
1875 (
TT.isOSBinFormatCOFF() && !DataReferencedByCode))
1886void InstrLowerer::emitInitialization() {
1903 F->addFnAttr(Attribute::NoInline);
1905 F->addFnAttr(Attribute::NoRedZone);
1909 IRB.CreateCall(RegisterF, {});
1910 IRB.CreateRetVoid();
This file contains the simple types necessary to represent the attributes associated with functions a...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
static bool lowerIntrinsics(Module &M)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool shouldRecordVTableAddr(GlobalVariable *GV)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static Constant * getVTableAddrForProfData(GlobalVariable *GV)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
Module.h This file contains the declarations for the Module class.
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Annotations lets you mark points and ranges inside source code, for tests:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
const Function * getParent() const
Return the enclosing method, or null if none.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
@ NoDeduplicate
No deduplication is performed.
ConstantArray - Constant Array Declarations.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
bool isZeroValue() const
Return true if the value is negative zero or null value.
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Lightweight error class with error context and mandatory checking.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
void setComdat(Comdat *C)
void setSection(StringRef S)
Change the section for this global.
bool hasLinkOnceLinkage() const
VisibilityTypes getVisibility() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LinkageTypes getLinkage() const
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
void setLinkage(LinkageTypes LT)
bool isDeclarationForLinker() const
Module * getParent()
Get the module that this global value is contained inside of...
VisibilityTypes
An enumeration for the kinds of visibility of global values.
@ DefaultVisibility
The GV is visible.
@ HiddenVisibility
The GV is hidden.
bool hasAvailableExternallyLinkage() const
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
ConstantInt * getNumBitmapBytes() const
This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic.
ConstantInt * getCondID() const
Value * getCondBool() const
Value * getMCDCCondBitmapAddr() const
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
Value * getMCDCCondBitmapAddr() const
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
Value * getTargetValue() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
An instruction for reading from memory.
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Helper class for SSA formation on a set of values defined in multiple blocks.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
size - Get the string size.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
Linkage
Describes symbol linkage. This can be used to resolve definition clashes.
This is an optimization pass for GlobalISel generic memory operations.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
cl::opt< bool > DoInstrProfNameCompression
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
StringRef getInstrProfVTableNamesVarName()
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
cl::opt< bool > DebugInfoCorrelate
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
StringRef getInstrProfCounterBiasVarName()
auto reverse(ContainerTy &&C)
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
void setGlobalVariableLargeSection(const Triple &TargetTriple, GlobalVariable &GV)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
StringRef getInstrProfVTableVarPrefix()
Return the name prefix of variables containing virtual table profile data.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Options for the frontend instrumentation based profiling pass.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.