65#define DEBUG_TYPE "instrprof"
74 "debug-info-correlate",
75 cl::desc(
"Use debug info to correlate profiles. (Deprecated, use "
76 "-profile-correlate=debug-info)"),
81 cl::desc(
"Use debug info or binary file to correlate profiles."),
84 "No profile correlation"),
86 "Use debug info to correlate"),
88 "Use binary to correlate")));
94 "hash-based-counter-split",
95 cl::desc(
"Rename counter variable of a comdat function based on cfg hash"),
99 RuntimeCounterRelocation(
"runtime-counter-relocation",
100 cl::desc(
"Enable relocating counters at runtime."),
105 cl::desc(
"Do static counter allocation for value profiler"),
109 "vp-counters-per-site",
110 cl::desc(
"The average number of profile counters allocated "
111 "per value profiling site."),
119 "instrprof-atomic-counter-update-all",
120 cl::desc(
"Make all profile counter updates atomic (for testing only)"),
124 "atomic-counter-update-promoted",
125 cl::desc(
"Do counter update using atomic fetch add "
126 " for promoted counters only"),
130 "atomic-first-counter",
131 cl::desc(
"Use atomic fetch add for first counter in a function (usually "
132 "the entry counter)"),
141 cl::desc(
"Do counter register promotion"),
144 "max-counter-promotions-per-loop",
cl::init(20),
145 cl::desc(
"Max number counter promotions per loop to avoid"
146 " increasing register pressure too much"));
150 MaxNumOfPromotions(
"max-counter-promotions",
cl::init(-1),
151 cl::desc(
"Max number of allowed counter promotions"));
154 "speculative-counter-promotion-max-exiting",
cl::init(3),
155 cl::desc(
"The max number of exiting blocks of a loop to allow "
156 " speculative counter promotion"));
159 "speculative-counter-promotion-to-loop",
160 cl::desc(
"When the option is false, if the target block is in a loop, "
161 "the promotion will be disallowed unless the promoted counter "
162 " update can be further/iteratively promoted into an acyclic "
166 "iterative-counter-promotion",
cl::init(
true),
167 cl::desc(
"Allow counter promotion across the whole loop nest."));
170 "skip-ret-exit-block",
cl::init(
true),
171 cl::desc(
"Suppress counter promotion if exit blocks contain ret."));
173using LoadStorePair = std::pair<Instruction *, Instruction *>;
176 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
182 return cast<ConstantInt>(MD->getValue())->getZExtValue();
185static bool enablesValueProfiling(
const Module &M) {
187 getIntModuleFlagOrZero(M,
"EnableValueProfiling") != 0;
191static bool profDataReferencedByCode(
const Module &M) {
192 return enablesValueProfiling(M);
195class InstrLowerer final {
201 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(
M)) {}
214 const bool DataReferencedByCode;
216 struct PerFunctionProfileData {
217 uint32_t NumValueSites[IPVK_Last + 1] = {};
223 PerFunctionProfileData() =
default;
232 std::vector<GlobalValue *> CompilerUsedVars;
233 std::vector<GlobalValue *> UsedVars;
234 std::vector<GlobalVariable *> ReferencedNames;
237 std::vector<GlobalVariable *> ReferencedVTables;
239 size_t NamesSize = 0;
246 std::vector<LoadStorePair> PromotionCandidates;
248 int64_t TotalCountersPromoted = 0;
255 void promoteCounterLoadStores(
Function *
F);
258 bool isRuntimeCounterRelocationEnabled()
const;
261 bool isCounterPromotionEnabled()
const;
349 void emitVTableNames();
355 void emitRegistration();
359 bool emitRuntimeHook();
366 void emitInitialization();
378 PGOCounterPromoterHelper(
385 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
387 assert(isa<StoreInst>(S));
388 SSA.AddAvailableValue(PH,
Init);
392 for (
unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
398 Value *LiveInValue =
SSA.GetValueInMiddleOfBlock(ExitBlock);
399 Value *
Addr = cast<StoreInst>(Store)->getPointerOperand();
402 if (
auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(
Addr)) {
409 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
410 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
411 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
412 Addr = Builder.CreateIntToPtr(BiasInst,
415 if (AtomicCounterUpdatePromoted)
420 AtomicOrdering::SequentiallyConsistent);
422 LoadInst *OldVal = Builder.CreateLoad(Ty,
Addr,
"pgocount.promoted");
423 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
424 auto *NewStore = Builder.CreateStore(NewVal,
Addr);
427 if (IterativeCounterPromotion) {
428 auto *TargetLoop = LI.getLoopFor(ExitBlock);
430 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
447class PGOCounterPromoter {
452 : LoopToCandidates(LoopToCands),
L(CurLoop), LI(LI),
BFI(
BFI) {
459 L.getExitBlocks(LoopExitBlocks);
460 if (!isPromotionPossible(&L, LoopExitBlocks))
463 for (
BasicBlock *ExitBlock : LoopExitBlocks) {
464 if (BlockSet.
insert(ExitBlock).second &&
468 ExitBlocks.push_back(ExitBlock);
474 bool run(int64_t *NumPromoted) {
476 if (ExitBlocks.size() == 0)
484 if (SkipRetExitBlock) {
485 for (
auto *BB : ExitBlocks)
486 if (isa<ReturnInst>(BB->getTerminator()))
490 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
494 unsigned Promoted = 0;
495 for (
auto &Cand : LoopToCandidates[&L]) {
499 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
503 auto *BB = Cand.first->getParent();
507 auto PreheaderCount =
BFI->getBlockProfileCount(
L.getLoopPreheader());
510 if (PreheaderCount && (*PreheaderCount * 3) >= (*
InstrCount * 2))
514 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second,
SSA, InitVal,
515 L.getLoopPreheader(), ExitBlocks,
516 InsertPts, LoopToCandidates, LI);
519 if (Promoted >= MaxProm)
523 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
527 LLVM_DEBUG(
dbgs() << Promoted <<
" counters promoted for loop (depth="
528 <<
L.getLoopDepth() <<
")\n");
529 return Promoted != 0;
533 bool allowSpeculativeCounterPromotion(
Loop *LP) {
535 L.getExitingBlocks(ExitingBlocks);
537 if (ExitingBlocks.
size() == 1)
539 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
547 isPromotionPossible(
Loop *LP,
551 return isa<CatchSwitchInst>(
Exit->getTerminator());
566 unsigned getMaxNumOfPromotionsInLoop(
Loop *LP) {
569 if (!isPromotionPossible(LP, LoopExitBlocks))
580 if (ExitingBlocks.
size() == 1)
581 return MaxNumOfPromotionsPerLoop;
583 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
587 if (SpeculativeCounterPromotionToLoop)
588 return MaxNumOfPromotionsPerLoop;
591 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
592 for (
auto *TargetBlock : LoopExitBlocks) {
593 auto *TargetLoop = LI.
getLoopFor(TargetBlock);
596 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
597 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
599 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
600 PendingCandsInTarget);
613enum class ValueProfilingCallType {
631 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
632 if (!Lowerer.lower())
638bool InstrLowerer::lowerIntrinsics(
Function *
F) {
639 bool MadeChange =
false;
640 PromotionCandidates.clear();
643 if (
auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
644 lowerIncrement(IPIS);
646 }
else if (
auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
649 }
else if (
auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
652 }
else if (
auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
655 }
else if (
auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
656 lowerValueProfileInst(IPVP);
658 }
else if (
auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
659 IPMP->eraseFromParent();
661 }
else if (
auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
662 lowerMCDCTestVectorBitmapUpdate(IPBU);
671 promoteCounterLoadStores(
F);
675bool InstrLowerer::isRuntimeCounterRelocationEnabled()
const {
677 if (
TT.isOSBinFormatMachO())
680 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
681 return RuntimeCounterRelocation;
684 return TT.isOSFuchsia();
687bool InstrLowerer::isCounterPromotionEnabled()
const {
688 if (DoCounterPromotion.getNumOccurrences() > 0)
689 return DoCounterPromotion;
691 return Options.DoCounterPromotion;
694void InstrLowerer::promoteCounterLoadStores(
Function *
F) {
695 if (!isCounterPromotionEnabled())
702 std::unique_ptr<BlockFrequencyInfo>
BFI;
703 if (
Options.UseBFIInPromotion) {
704 std::unique_ptr<BranchProbabilityInfo> BPI;
709 for (
const auto &LoadStore : PromotionCandidates) {
716 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
724 PGOCounterPromoter Promoter(LoopPromotionCandidates, *
Loop, LI,
BFI.get());
725 Promoter.run(&TotalCountersPromoted);
731 if (TT.isOSFuchsia())
739 auto containsIntrinsic = [&](
int ID) {
741 return !
F->use_empty();
744 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
745 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
746 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
747 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
748 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
751bool InstrLowerer::lower() {
752 bool MadeChange =
false;
754 if (NeedsRuntimeHook)
755 MadeChange = emitRuntimeHook();
761 if (!ContainsProfiling && !CoverageNamesVar)
770 for (
auto I = BB.
begin(), E = BB.
end();
I != E;
I++) {
771 if (
auto *Ind = dyn_cast<InstrProfValueProfileInst>(
I))
772 computeNumValueSiteCounts(Ind);
774 if (FirstProfInst ==
nullptr &&
775 (isa<InstrProfIncrementInst>(
I) || isa<InstrProfCoverInst>(
I)))
776 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(
I);
778 if (
const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(
I))
779 static_cast<void>(getOrCreateRegionBitmaps(Params));
786 if (FirstProfInst !=
nullptr) {
787 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
794 if (GV.hasMetadata(LLVMContext::MD_type))
795 getOrCreateVTableProfData(&GV);
800 if (CoverageNamesVar) {
801 lowerCoverageData(CoverageNamesVar);
816 if (!NeedsRuntimeHook && ContainsProfiling)
821 emitInitialization();
827 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
832 if (
auto AK = TLI.getExtAttrForI32Param(
false))
833 AL = AL.addParamAttribute(M.getContext(), 2, AK);
835 assert((CallType == ValueProfilingCallType::Default ||
836 CallType == ValueProfilingCallType::MemOp) &&
837 "Must be Default or MemOp");
838 Type *ParamTypes[] = {
839#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
842 auto *ValueProfilingCallTy =
844 StringRef FuncName = CallType == ValueProfilingCallType::Default
847 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
854 auto &
PD = ProfileDataMap[
Name];
865 "Value profiling is not yet supported with lightweight instrumentation");
867 auto It = ProfileDataMap.find(
Name);
868 assert(It != ProfileDataMap.end() && It->second.DataVar &&
869 "value profiling detected in function with no counter incerement");
875 Index += It->second.NumValueSites[Kind];
879 llvm::InstrProfValueKind::IPVK_MemOPSize);
895 Call = Builder.CreateCall(
899 if (
auto AK = TLI->getExtAttrForI32Param(
false))
900 Call->addParamAttr(2, AK);
922 if (
TT.supportsCOMDAT())
923 Bias->
setComdat(
M.getOrInsertComdat(VarName));
929 auto *
Counters = getOrCreateRegionCounters(
I);
932 if (isa<InstrProfTimestampInst>(
I))
935 auto *
Addr = Builder.CreateConstInBoundsGEP2_32(
938 if (!isRuntimeCounterRelocationEnabled())
942 Function *Fn =
I->getParent()->getParent();
943 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
947 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias,
"profc_bias");
949 auto *
Add = Builder.CreateAdd(Builder.CreatePtrToInt(
Addr, Int64Ty), BiasLI);
950 return Builder.CreateIntToPtr(
Add,
Addr->getType());
955Function *InstrLowerer::createRMWOrFunc() {
956 auto &Ctx =
M.getContext();
963 auto *ArgAddr = Fn->
getArg(0);
964 auto *ArgVal = Fn->
getArg(1);
969 auto *Bitmap = Builder.CreateLoad(Int8Ty, ArgAddr,
"mcdc.bits");
971 if (
Options.Atomic || AtomicCounterUpdateAll) {
975 auto *Masked = Builder.CreateAnd(Bitmap, ArgVal);
976 auto *ShouldStore = Builder.CreateICmpNE(Masked, ArgVal);
981 Builder.CreateCondBr(ShouldStore, ThenTerm, ElseTerm, Unlikely);
986 ThenBuilder.CreateRetVoid();
989 ElseBuilder.CreateRetVoid();
996 auto *
Result = Builder.CreateOr(Bitmap, ArgVal);
1000 Builder.CreateStore(Result, ArgAddr);
1003 Builder.CreateRetVoid();
1010 RMWOrFunc = createRMWOrFunc();
1016 auto *Bitmaps = getOrCreateRegionBitmaps(
I);
1019 if (isRuntimeCounterRelocationEnabled()) {
1023 Twine(
"Runtime counter relocation is presently not supported for MC/DC "
1032 auto *
Addr = getCounterAddress(CoverInstruction);
1035 Builder.CreateStore(Builder.getInt8(0),
Addr);
1039void InstrLowerer::lowerTimestamp(
1042 "timestamp probes are always the first probe for a function");
1043 auto &Ctx =
M.getContext();
1044 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
1048 auto Callee =
M.getOrInsertFunction(
1049 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
1050 Builder.CreateCall(Callee, {TimestampAddr});
1055 auto *
Addr = getCounterAddress(Inc);
1058 if (
Options.Atomic || AtomicCounterUpdateAll ||
1065 auto *Count = Builder.CreateAdd(Load, Inc->
getStep());
1066 auto *
Store = Builder.CreateStore(Count,
Addr);
1067 if (isCounterPromotionEnabled())
1068 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
1073void InstrLowerer::lowerCoverageData(
GlobalVariable *CoverageNamesVar) {
1078 Value *
V =
NC->stripPointerCasts();
1079 assert(isa<GlobalVariable>(V) &&
"Missing reference to function name");
1083 ReferencedNames.push_back(
Name);
1084 if (isa<ConstantExpr>(
NC))
1085 NC->dropAllReferences();
1090void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1096 auto *BitmapAddr = getBitmapAddress(Update);
1100 auto *Temp = Builder.CreateAdd(
1101 Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp"),
1106 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
1110 auto *BitmapByteAddr =
1111 Builder.CreateInBoundsPtrAdd(BitmapAddr, BitmapByteOffset);
1116 auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
1120 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
1122 Builder.Insert(getRMWOrCall(BitmapByteAddr, ShiftedVal));
1136 return (Prefix +
Name).str();
1142 return (Prefix +
Name).str();
1143 return (Prefix +
Name +
"." +
Twine(FuncHash)).str();
1151 if (!profDataReferencedByCode(*
F->getParent()))
1155 bool HasAvailableExternallyLinkage =
F->hasAvailableExternallyLinkage();
1156 if (!
F->hasLinkOnceLinkage() && !
F->hasLocalLinkage() &&
1157 !HasAvailableExternallyLinkage)
1163 if (HasAvailableExternallyLinkage &&
1164 F->hasFnAttribute(Attribute::AlwaysInline))
1170 if (
F->hasLocalLinkage() &&
F->hasComdat())
1180 return F->hasAddressTaken() ||
F->hasLinkOnceLinkage();
1227 Fn->
getName() +
".local", Fn);
1252 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1253 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
1266 bool UseComdat = (NeedComdat ||
TT.isOSBinFormatELF());
1281 StringRef GroupName =
TT.isOSBinFormatCOFF() && DataReferencedByCode
1284 Comdat *
C =
M.getOrInsertComdat(GroupName);
1304 if (!profDataReferencedByCode(*GV->
getParent()))
1331void InstrLowerer::getOrCreateVTableProfData(
GlobalVariable *GV) {
1333 "Value profiling is not supported with lightweight instrumentation");
1344 auto It = VTableDataMap.find(GV);
1345 if (It != VTableDataMap.end() && It->second)
1353 if (
TT.isOSBinFormatXCOFF()) {
1359 Type *DataTypes[] = {
1360#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1362#undef INSTR_PROF_VTABLE_DATA
1369 const std::string PGOVTableName =
getPGOName(*GV);
1376#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1378#undef INSTR_PROF_VTABLE_DATA
1386 Data->setVisibility(Visibility);
1390 maybeSetComdat(
Data, GV,
Data->getName());
1392 VTableDataMap[GV] =
Data;
1394 ReferencedVTables.push_back(GV);
1398 UsedVars.push_back(
Data);
1422 if (
TT.isOSBinFormatXCOFF()) {
1431 if (IPSK == IPSK_cnts) {
1435 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1436 }
else if (IPSK == IPSK_bitmap) {
1440 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1441 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1446 Ptr->setVisibility(Visibility);
1450 Ptr->setLinkage(Linkage);
1451 maybeSetComdat(
Ptr, Fn, VarName);
1470 auto &
PD = ProfileDataMap[NamePtr];
1471 if (
PD.RegionBitmaps)
1472 return PD.RegionBitmaps;
1476 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1477 PD.RegionBitmaps = BitmapPtr;
1479 return PD.RegionBitmaps;
1486 auto &Ctx =
M.getContext();
1488 if (isa<InstrProfCoverInst>(Inc)) {
1492 std::vector<Constant *> InitialValues(NumCounters,
1510 auto &
PD = ProfileDataMap[NamePtr];
1511 if (
PD.RegionCounters)
1512 return PD.RegionCounters;
1516 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1517 PD.RegionCounters = CounterPtr;
1525 Metadata *FunctionNameAnnotation[] = {
1533 Metadata *NumCountersAnnotation[] = {
1542 auto *DICounter =
DB.createGlobalVariableExpression(
1543 SP, CounterPtr->getName(),
StringRef(), SP->getFile(),
1544 0,
DB.createUnspecifiedType(
"Profile Data Type"),
1545 CounterPtr->hasLocalLinkage(),
true,
nullptr,
1546 nullptr,
nullptr, 0,
1548 CounterPtr->addDebugInfo(DICounter);
1553 CompilerUsedVars.push_back(
PD.RegionCounters);
1557 createDataVariable(Inc);
1559 return PD.RegionCounters;
1569 auto &
PD = ProfileDataMap[NamePtr];
1586 if (
TT.isOSBinFormatXCOFF()) {
1595 std::string CntsVarName =
1597 std::string DataVarName =
1606 NS +=
PD.NumValueSites[Kind];
1607 if (NS > 0 && ValueProfileStaticAlloc &&
1613 ValuesVar->setVisibility(Visibility);
1615 ValuesVar->setSection(
1617 ValuesVar->setAlignment(
Align(8));
1618 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1619 ValuesPtrExpr = ValuesVar;
1623 auto *CounterPtr =
PD.RegionCounters;
1628 auto *IntPtrTy =
M.getDataLayout().getIntPtrType(
M.getContext());
1631 Type *DataTypes[] = {
1632#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1639 Constant *Int16ArrayVals[IPVK_Last + 1];
1641 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty,
PD.NumValueSites[Kind]);
1654 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1655 (
TT.isOSBinFormatELF() ||
1656 (!DataReferencedByCode &&
TT.isOSBinFormatCOFF()))) {
1661 new GlobalVariable(M, DataTy,
false, Linkage,
nullptr, DataVarName);
1664 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
1669 DataSectionKind = IPSK_covdata;
1671 if (BitmapPtr !=
nullptr)
1676 DataSectionKind = IPSK_data;
1677 RelativeCounterPtr =
1680 if (BitmapPtr !=
nullptr)
1687#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1692 Data->setVisibility(Visibility);
1695 Data->setAlignment(
Align(INSTR_PROF_DATA_ALIGNMENT));
1696 maybeSetComdat(
Data, Fn, CntsVarName);
1701 CompilerUsedVars.push_back(
Data);
1707 ReferencedNames.push_back(NamePtr);
1710void InstrLowerer::emitVNodes() {
1711 if (!ValueProfileStaticAlloc)
1721 for (
auto &PD : ProfileDataMap) {
1723 TotalNS +=
PD.second.NumValueSites[Kind];
1729 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1737#define INSTR_PROF_MIN_VAL_COUNTS 10
1741 auto &Ctx =
M.getContext();
1742 Type *VNodeTypes[] = {
1743#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1753 VNodesVar->setSection(
1755 VNodesVar->setAlignment(
M.getDataLayout().getABITypeAlign(VNodesTy));
1758 UsedVars.push_back(VNodesVar);
1761void InstrLowerer::emitNameData() {
1762 std::string UncompressedData;
1764 if (ReferencedNames.empty())
1767 std::string CompressedNameStr;
1773 auto &Ctx =
M.getContext();
1779 NamesSize = CompressedNameStr.size();
1781 NamesVar->setSection(
1788 NamesVar->setAlignment(
Align(1));
1791 UsedVars.push_back(NamesVar);
1793 for (
auto *NamePtr : ReferencedNames)
1797void InstrLowerer::emitVTableNames() {
1802 std::string CompressedVTableNames;
1808 auto &Ctx =
M.getContext();
1810 Ctx,
StringRef(CompressedVTableNames),
false );
1819 UsedVars.push_back(VTableNamesVar);
1822void InstrLowerer::emitRegistration() {
1835 RegisterF->addFnAttr(Attribute::NoRedZone);
1838 auto *RuntimeRegisterF =
1844 if (!isa<Function>(
Data))
1845 IRB.CreateCall(RuntimeRegisterF,
Data);
1847 if (
Data != NamesVar && !isa<Function>(
Data))
1848 IRB.CreateCall(RuntimeRegisterF,
Data);
1851 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1852 auto *NamesRegisterTy =
1854 auto *NamesRegisterF =
1857 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
1860 IRB.CreateRetVoid();
1863bool InstrLowerer::emitRuntimeHook() {
1866 if (
TT.isOSLinux() ||
TT.isOSAIX())
1880 if (
TT.isOSBinFormatELF() && !
TT.isPS()) {
1882 CompilerUsedVars.push_back(Var);
1888 User->addFnAttr(Attribute::NoInline);
1890 User->addFnAttr(Attribute::NoRedZone);
1892 if (
TT.supportsCOMDAT())
1896 auto *
Load = IRB.CreateLoad(Int32Ty, Var);
1897 IRB.CreateRet(Load);
1900 CompilerUsedVars.push_back(
User);
1905void InstrLowerer::emitUses() {
1915 if (
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO() ||
1916 (
TT.isOSBinFormatCOFF() && !DataReferencedByCode))
1927void InstrLowerer::emitInitialization() {
1944 F->addFnAttr(Attribute::NoInline);
1946 F->addFnAttr(Attribute::NoRedZone);
1950 IRB.CreateCall(RegisterF, {});
1951 IRB.CreateRetVoid();
This file contains the simple types necessary to represent the attributes associated with functions a...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
static bool lowerIntrinsics(Module &M)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool shouldRecordVTableAddr(GlobalVariable *GV)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static Constant * getVTableAddrForProfData(GlobalVariable *GV)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
Module.h This file contains the declarations for the Module class.
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Annotations lets you mark points and ranges inside source code, for tests:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
const Function * getParent() const
Return the enclosing method, or null if none.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
@ NoDeduplicate
No deduplication is performed.
ConstantArray - Constant Array Declarations.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
bool isZeroValue() const
Return true if the value is negative zero or null value.
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Lightweight error class with error context and mandatory checking.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
void setComdat(Comdat *C)
void setSection(StringRef S)
Change the section for this global.
bool hasLinkOnceLinkage() const
VisibilityTypes getVisibility() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LinkageTypes getLinkage() const
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
void setLinkage(LinkageTypes LT)
bool isDeclarationForLinker() const
Module * getParent()
Get the module that this global value is contained inside of...
VisibilityTypes
An enumeration for the kinds of visibility of global values.
@ DefaultVisibility
The GV is visible.
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
bool hasAvailableExternallyLinkage() const
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
auto getNumBitmapBytes() const
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
Value * getMCDCCondBitmapAddr() const
ConstantInt * getBitmapIndex() const
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
Value * getTargetValue() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
An instruction for reading from memory.
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Helper class for SSA formation on a set of values defined in multiple blocks.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
size - Get the string size.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
Linkage
Describes symbol linkage. This can be used to resolve definition clashes.
This is an optimization pass for GlobalISel generic memory operations.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
cl::opt< bool > DoInstrProfNameCompression
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
StringRef getInstrProfVTableNamesVarName()
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
cl::opt< bool > DebugInfoCorrelate
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
StringRef getInstrProfCounterBiasVarName()
auto reverse(ContainerTy &&C)
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
void setGlobalVariableLargeSection(const Triple &TargetTriple, GlobalVariable &GV)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
StringRef getInstrProfVTableVarPrefix()
Return the name prefix of variables containing virtual table profile data.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Options for the frontend instrumentation based profiling pass.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.