65#define DEBUG_TYPE "instrprof"
74 "debug-info-correlate",
75 cl::desc(
"Use debug info to correlate profiles. (Deprecated, use "
76 "-profile-correlate=debug-info)"),
81 cl::desc(
"Use debug info or binary file to correlate profiles."),
84 "No profile correlation"),
86 "Use debug info to correlate"),
88 "Use binary to correlate")));
94 "hash-based-counter-split",
95 cl::desc(
"Rename counter variable of a comdat function based on cfg hash"),
99 RuntimeCounterRelocation(
"runtime-counter-relocation",
100 cl::desc(
"Enable relocating counters at runtime."),
105 cl::desc(
"Do static counter allocation for value profiler"),
109 "vp-counters-per-site",
110 cl::desc(
"The average number of profile counters allocated "
111 "per value profiling site."),
119 "instrprof-atomic-counter-update-all",
120 cl::desc(
"Make all profile counter updates atomic (for testing only)"),
124 "atomic-counter-update-promoted",
125 cl::desc(
"Do counter update using atomic fetch add "
126 " for promoted counters only"),
130 "atomic-first-counter",
131 cl::desc(
"Use atomic fetch add for first counter in a function (usually "
132 "the entry counter)"),
141 cl::desc(
"Do counter register promotion"),
144 "max-counter-promotions-per-loop",
cl::init(20),
145 cl::desc(
"Max number counter promotions per loop to avoid"
146 " increasing register pressure too much"));
150 MaxNumOfPromotions(
"max-counter-promotions",
cl::init(-1),
151 cl::desc(
"Max number of allowed counter promotions"));
154 "speculative-counter-promotion-max-exiting",
cl::init(3),
155 cl::desc(
"The max number of exiting blocks of a loop to allow "
156 " speculative counter promotion"));
159 "speculative-counter-promotion-to-loop",
160 cl::desc(
"When the option is false, if the target block is in a loop, "
161 "the promotion will be disallowed unless the promoted counter "
162 " update can be further/iteratively promoted into an acyclic "
166 "iterative-counter-promotion",
cl::init(
true),
167 cl::desc(
"Allow counter promotion across the whole loop nest."));
170 "skip-ret-exit-block",
cl::init(
true),
171 cl::desc(
"Suppress counter promotion if exit blocks contain ret."));
175 cl::desc(
"Do PGO instrumentation sampling"));
178 "sampled-instr-period",
179 cl::desc(
"Set the profile instrumentation sample period. For each sample "
180 "period, a fixed number of consecutive samples will be recorded. "
181 "The number is controlled by 'sampled-instr-burst-duration' flag. "
182 "The default sample period of 65535 is optimized for generating "
183 "efficient code that leverages unsigned integer wrapping in "
188 "sampled-instr-burst-duration",
189 cl::desc(
"Set the profile instrumentation burst duration, which can range "
190 "from 0 to one less than the value of 'sampled-instr-period'. "
191 "This number of samples will be recorded for each "
192 "'sampled-instr-period' count update. Setting to 1 enables "
193 "simple sampling, in which case it is recommended to set "
194 "'sampled-instr-period' to a prime number."),
197using LoadStorePair = std::pair<Instruction *, Instruction *>;
200 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
206 return cast<ConstantInt>(MD->getValue())->getZExtValue();
209static bool enablesValueProfiling(
const Module &M) {
211 getIntModuleFlagOrZero(M,
"EnableValueProfiling") != 0;
215static bool profDataReferencedByCode(
const Module &M) {
216 return enablesValueProfiling(M);
219class InstrLowerer final {
225 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(
M)) {}
238 const bool DataReferencedByCode;
240 struct PerFunctionProfileData {
241 uint32_t NumValueSites[IPVK_Last + 1] = {};
247 PerFunctionProfileData() =
default;
256 std::vector<GlobalValue *> CompilerUsedVars;
257 std::vector<GlobalValue *> UsedVars;
258 std::vector<GlobalVariable *> ReferencedNames;
261 std::vector<GlobalVariable *> ReferencedVTables;
263 size_t NamesSize = 0;
270 std::vector<LoadStorePair> PromotionCandidates;
272 int64_t TotalCountersPromoted = 0;
279 void promoteCounterLoadStores(
Function *
F);
282 bool isRuntimeCounterRelocationEnabled()
const;
285 bool isCounterPromotionEnabled()
const;
288 bool isSamplingEnabled()
const;
379 void emitVTableNames();
385 void emitRegistration();
389 bool emitRuntimeHook();
396 void emitInitialization();
408 PGOCounterPromoterHelper(
415 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
417 assert(isa<StoreInst>(S));
418 SSA.AddAvailableValue(PH,
Init);
422 for (
unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
428 Value *LiveInValue =
SSA.GetValueInMiddleOfBlock(ExitBlock);
429 Value *
Addr = cast<StoreInst>(Store)->getPointerOperand();
432 if (
auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(
Addr)) {
439 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
440 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
441 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
442 Addr = Builder.CreateIntToPtr(BiasInst,
445 if (AtomicCounterUpdatePromoted)
450 AtomicOrdering::SequentiallyConsistent);
452 LoadInst *OldVal = Builder.CreateLoad(Ty,
Addr,
"pgocount.promoted");
453 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
454 auto *NewStore = Builder.CreateStore(NewVal,
Addr);
457 if (IterativeCounterPromotion) {
458 auto *TargetLoop = LI.getLoopFor(ExitBlock);
460 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
477class PGOCounterPromoter {
482 : LoopToCandidates(LoopToCands),
L(CurLoop), LI(LI),
BFI(
BFI) {
489 L.getExitBlocks(LoopExitBlocks);
490 if (!isPromotionPossible(&L, LoopExitBlocks))
493 for (
BasicBlock *ExitBlock : LoopExitBlocks) {
494 if (BlockSet.
insert(ExitBlock).second &&
498 ExitBlocks.push_back(ExitBlock);
504 bool run(int64_t *NumPromoted) {
506 if (ExitBlocks.size() == 0)
514 if (SkipRetExitBlock) {
515 for (
auto *BB : ExitBlocks)
516 if (isa<ReturnInst>(BB->getTerminator()))
520 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
524 unsigned Promoted = 0;
525 for (
auto &Cand : LoopToCandidates[&L]) {
529 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
533 auto *BB = Cand.first->getParent();
537 auto PreheaderCount =
BFI->getBlockProfileCount(
L.getLoopPreheader());
540 if (PreheaderCount && (*PreheaderCount * 3) >= (*
InstrCount * 2))
544 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second,
SSA, InitVal,
545 L.getLoopPreheader(), ExitBlocks,
546 InsertPts, LoopToCandidates, LI);
549 if (Promoted >= MaxProm)
553 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
557 LLVM_DEBUG(
dbgs() << Promoted <<
" counters promoted for loop (depth="
558 <<
L.getLoopDepth() <<
")\n");
559 return Promoted != 0;
563 bool allowSpeculativeCounterPromotion(
Loop *LP) {
565 L.getExitingBlocks(ExitingBlocks);
567 if (ExitingBlocks.
size() == 1)
569 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
577 isPromotionPossible(
Loop *LP,
581 return isa<CatchSwitchInst>(
Exit->getTerminator());
596 unsigned getMaxNumOfPromotionsInLoop(
Loop *LP) {
599 if (!isPromotionPossible(LP, LoopExitBlocks))
610 if (ExitingBlocks.
size() == 1)
611 return MaxNumOfPromotionsPerLoop;
613 if (ExitingBlocks.
size() > SpeculativeCounterPromotionMaxExiting)
617 if (SpeculativeCounterPromotionToLoop)
618 return MaxNumOfPromotionsPerLoop;
621 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
622 for (
auto *TargetBlock : LoopExitBlocks) {
623 auto *TargetLoop = LI.
getLoopFor(TargetBlock);
626 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
627 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
629 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
630 PendingCandsInTarget);
643enum class ValueProfilingCallType {
661 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
662 if (!Lowerer.lower())
713 if (!isSamplingEnabled())
716 unsigned SampledBurstDuration = SampledInstrBurstDuration.getValue();
717 unsigned SampledPeriod = SampledInstrPeriod.getValue();
718 if (SampledBurstDuration >= SampledPeriod) {
720 "SampledPeriod needs to be greater than SampledBurstDuration");
722 bool UseShort = (SampledPeriod <= USHRT_MAX);
723 bool IsSimpleSampling = (SampledBurstDuration == 1);
726 bool IsFastSampling = (!IsSimpleSampling && SampledPeriod == 65535);
741 M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
742 assert(SamplingVar &&
"SamplingVar not set properly");
746 Value *NewSamplingVarVal;
750 auto *LoadSamplingVar = CondBuilder.CreateLoad(SamplingVarTy, SamplingVar);
751 if (IsSimpleSampling) {
755 IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));
756 SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);
759 auto *DurationCond = CondBuilder.CreateICmpULE(
760 LoadSamplingVar, GetConstant(CondBuilder, SampledBurstDuration));
761 BranchWeight = MDB.createBranchWeights(
762 SampledBurstDuration, SampledPeriod + 1 - SampledBurstDuration);
764 DurationCond,
I,
false, BranchWeight);
767 IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));
768 SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);
769 I->moveBefore(ThenTerm);
778 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
779 NewSamplingVarVal, GetConstant(PeriodCondBuilder, SampledPeriod));
780 BranchWeight = MDB.createBranchWeights(1, SampledPeriod);
782 &ElseTerm, BranchWeight);
785 if (IsSimpleSampling)
786 I->moveBefore(ThenTerm);
789 ResetBuilder.CreateStore(GetConstant(ResetBuilder, 0), SamplingVar);
793bool InstrLowerer::lowerIntrinsics(
Function *
F) {
794 bool MadeChange =
false;
795 PromotionCandidates.clear();
803 if (
auto *IP = dyn_cast<InstrProfInstBase>(&Instr))
808 for (
auto *Instr : InstrProfInsts) {
810 if (
auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Instr)) {
811 lowerIncrement(IPIS);
813 }
else if (
auto *IPI = dyn_cast<InstrProfIncrementInst>(Instr)) {
816 }
else if (
auto *IPC = dyn_cast<InstrProfTimestampInst>(Instr)) {
819 }
else if (
auto *IPC = dyn_cast<InstrProfCoverInst>(Instr)) {
822 }
else if (
auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Instr)) {
823 lowerValueProfileInst(IPVP);
825 }
else if (
auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Instr)) {
826 IPMP->eraseFromParent();
828 }
else if (
auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Instr)) {
829 lowerMCDCTestVectorBitmapUpdate(IPBU);
837 promoteCounterLoadStores(
F);
841bool InstrLowerer::isRuntimeCounterRelocationEnabled()
const {
843 if (
TT.isOSBinFormatMachO())
846 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
847 return RuntimeCounterRelocation;
850 return TT.isOSFuchsia();
853bool InstrLowerer::isSamplingEnabled()
const {
854 if (SampledInstr.getNumOccurrences() > 0)
859bool InstrLowerer::isCounterPromotionEnabled()
const {
860 if (DoCounterPromotion.getNumOccurrences() > 0)
861 return DoCounterPromotion;
863 return Options.DoCounterPromotion;
866void InstrLowerer::promoteCounterLoadStores(
Function *
F) {
867 if (!isCounterPromotionEnabled())
874 std::unique_ptr<BlockFrequencyInfo>
BFI;
875 if (
Options.UseBFIInPromotion) {
876 std::unique_ptr<BranchProbabilityInfo> BPI;
881 for (
const auto &LoadStore : PromotionCandidates) {
888 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
896 PGOCounterPromoter Promoter(LoopPromotionCandidates, *
Loop, LI,
BFI.get());
897 Promoter.run(&TotalCountersPromoted);
903 if (TT.isOSFuchsia())
911 auto containsIntrinsic = [&](
int ID) {
913 return !
F->use_empty();
916 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
917 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
918 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
919 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
920 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
923bool InstrLowerer::lower() {
924 bool MadeChange =
false;
926 if (NeedsRuntimeHook)
927 MadeChange = emitRuntimeHook();
929 if (!IsCS && isSamplingEnabled())
936 if (!ContainsProfiling && !CoverageNamesVar)
945 for (
auto I = BB.
begin(), E = BB.
end();
I != E;
I++) {
946 if (
auto *Ind = dyn_cast<InstrProfValueProfileInst>(
I))
947 computeNumValueSiteCounts(Ind);
949 if (FirstProfInst ==
nullptr &&
950 (isa<InstrProfIncrementInst>(
I) || isa<InstrProfCoverInst>(
I)))
951 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(
I);
953 if (
const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(
I))
954 static_cast<void>(getOrCreateRegionBitmaps(Params));
961 if (FirstProfInst !=
nullptr) {
962 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
969 if (GV.hasMetadata(LLVMContext::MD_type))
970 getOrCreateVTableProfData(&GV);
975 if (CoverageNamesVar) {
976 lowerCoverageData(CoverageNamesVar);
991 if (!NeedsRuntimeHook && ContainsProfiling)
996 emitInitialization();
1002 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
1007 if (
auto AK = TLI.getExtAttrForI32Param(
false))
1008 AL = AL.addParamAttribute(M.getContext(), 2, AK);
1010 assert((CallType == ValueProfilingCallType::Default ||
1011 CallType == ValueProfilingCallType::MemOp) &&
1012 "Must be Default or MemOp");
1013 Type *ParamTypes[] = {
1014#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
1017 auto *ValueProfilingCallTy =
1019 StringRef FuncName = CallType == ValueProfilingCallType::Default
1022 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
1029 auto &
PD = ProfileDataMap[
Name];
1040 "Value profiling is not yet supported with lightweight instrumentation");
1042 auto It = ProfileDataMap.find(
Name);
1043 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1044 "value profiling detected in function with no counter incerement");
1050 Index += It->second.NumValueSites[Kind];
1054 llvm::InstrProfValueKind::IPVK_MemOPSize);
1074 if (
auto AK = TLI->getExtAttrForI32Param(
false))
1075 Call->addParamAttr(2, AK);
1097 if (
TT.supportsCOMDAT())
1098 Bias->
setComdat(
M.getOrInsertComdat(VarName));
1104 auto *
Counters = getOrCreateRegionCounters(
I);
1107 if (isa<InstrProfTimestampInst>(
I))
1113 if (!isRuntimeCounterRelocationEnabled())
1117 Function *Fn =
I->getParent()->getParent();
1118 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
1122 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias,
"profc_bias");
1124 BiasLI->
setMetadata(LLVMContext::MD_invariant_load,
1133Function *InstrLowerer::createRMWOrFunc() {
1134 auto &Ctx =
M.getContext();
1141 auto *ArgAddr = Fn->
getArg(0);
1142 auto *ArgVal = Fn->
getArg(1);
1147 auto *Bitmap = Builder.
CreateLoad(Int8Ty, ArgAddr,
"mcdc.bits");
1149 if (
Options.Atomic || AtomicCounterUpdateAll) {
1159 Builder.
CreateCondBr(ShouldStore, ThenTerm, ElseTerm, Unlikely);
1164 ThenBuilder.CreateRetVoid();
1167 ElseBuilder.CreateRetVoid();
1188 RMWOrFunc = createRMWOrFunc();
1194 auto *Bitmaps = getOrCreateRegionBitmaps(
I);
1195 if (!isRuntimeCounterRelocationEnabled())
1203 auto *BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias,
"profbm_bias");
1205 BiasLI->
setMetadata(LLVMContext::MD_invariant_load,
1210 return Builder.
CreatePtrAdd(Bitmaps, BiasLI,
"profbm_addr");
1214 auto *
Addr = getCounterAddress(CoverInstruction);
1221void InstrLowerer::lowerTimestamp(
1224 "timestamp probes are always the first probe for a function");
1225 auto &Ctx =
M.getContext();
1226 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
1230 auto Callee =
M.getOrInsertFunction(
1231 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
1237 auto *
Addr = getCounterAddress(Inc);
1240 if (
Options.Atomic || AtomicCounterUpdateAll ||
1249 if (isCounterPromotionEnabled())
1250 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
1255void InstrLowerer::lowerCoverageData(
GlobalVariable *CoverageNamesVar) {
1260 Value *
V =
NC->stripPointerCasts();
1261 assert(isa<GlobalVariable>(V) &&
"Missing reference to function name");
1265 ReferencedNames.push_back(
Name);
1266 if (isa<ConstantExpr>(
NC))
1267 NC->dropAllReferences();
1272void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1278 auto *BitmapAddr = getBitmapAddress(Update);
1283 Builder.
CreateLoad(Int32Ty, MCDCCondBitmapAddr,
"mcdc.temp"),
1288 auto *BitmapByteOffset = Builder.
CreateLShr(Temp, 0x3);
1292 auto *BitmapByteAddr =
1304 Builder.
Insert(getRMWOrCall(BitmapByteAddr, ShiftedVal));
1318 return (Prefix +
Name).str();
1324 return (Prefix +
Name).str();
1325 return (Prefix +
Name +
"." +
Twine(FuncHash)).str();
1333 if (!profDataReferencedByCode(*
F->getParent()))
1337 bool HasAvailableExternallyLinkage =
F->hasAvailableExternallyLinkage();
1338 if (!
F->hasLinkOnceLinkage() && !
F->hasLocalLinkage() &&
1339 !HasAvailableExternallyLinkage)
1345 if (HasAvailableExternallyLinkage &&
1346 F->hasFnAttribute(Attribute::AlwaysInline))
1352 if (
F->hasLocalLinkage() &&
F->hasComdat())
1362 return F->hasAddressTaken() ||
F->hasLinkOnceLinkage();
1409 Fn->
getName() +
".local", Fn);
1434 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1435 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
1448 bool UseComdat = (NeedComdat ||
TT.isOSBinFormatELF());
1463 StringRef GroupName =
TT.isOSBinFormatCOFF() && DataReferencedByCode
1466 Comdat *
C =
M.getOrInsertComdat(GroupName);
1486 if (!profDataReferencedByCode(*GV->
getParent()))
1513void InstrLowerer::getOrCreateVTableProfData(
GlobalVariable *GV) {
1515 "Value profiling is not supported with lightweight instrumentation");
1526 auto It = VTableDataMap.find(GV);
1527 if (It != VTableDataMap.end() && It->second)
1535 if (
TT.isOSBinFormatXCOFF()) {
1541 Type *DataTypes[] = {
1542#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1544#undef INSTR_PROF_VTABLE_DATA
1551 const std::string PGOVTableName =
getPGOName(*GV);
1558#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1560#undef INSTR_PROF_VTABLE_DATA
1568 Data->setVisibility(Visibility);
1572 maybeSetComdat(
Data, GV,
Data->getName());
1574 VTableDataMap[GV] =
Data;
1576 ReferencedVTables.push_back(GV);
1580 UsedVars.push_back(
Data);
1604 if (
TT.isOSBinFormatXCOFF()) {
1613 if (IPSK == IPSK_cnts) {
1617 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1618 }
else if (IPSK == IPSK_bitmap) {
1622 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1623 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1628 Ptr->setVisibility(Visibility);
1632 Ptr->setLinkage(Linkage);
1633 maybeSetComdat(
Ptr, Fn, VarName);
1652 auto &
PD = ProfileDataMap[NamePtr];
1653 if (
PD.RegionBitmaps)
1654 return PD.RegionBitmaps;
1658 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1659 PD.RegionBitmaps = BitmapPtr;
1661 return PD.RegionBitmaps;
1668 auto &Ctx =
M.getContext();
1670 if (isa<InstrProfCoverInst>(Inc)) {
1674 std::vector<Constant *> InitialValues(NumCounters,
1692 auto &
PD = ProfileDataMap[NamePtr];
1693 if (
PD.RegionCounters)
1694 return PD.RegionCounters;
1698 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1699 PD.RegionCounters = CounterPtr;
1707 Metadata *FunctionNameAnnotation[] = {
1715 Metadata *NumCountersAnnotation[] = {
1724 auto *DICounter =
DB.createGlobalVariableExpression(
1725 SP, CounterPtr->getName(),
StringRef(), SP->getFile(),
1726 0,
DB.createUnspecifiedType(
"Profile Data Type"),
1727 CounterPtr->hasLocalLinkage(),
true,
nullptr,
1728 nullptr,
nullptr, 0,
1730 CounterPtr->addDebugInfo(DICounter);
1735 CompilerUsedVars.push_back(
PD.RegionCounters);
1739 createDataVariable(Inc);
1741 return PD.RegionCounters;
1751 auto &
PD = ProfileDataMap[NamePtr];
1768 if (
TT.isOSBinFormatXCOFF()) {
1777 std::string CntsVarName =
1779 std::string DataVarName =
1788 NS +=
PD.NumValueSites[Kind];
1789 if (NS > 0 && ValueProfileStaticAlloc &&
1795 ValuesVar->setVisibility(Visibility);
1797 ValuesVar->setSection(
1799 ValuesVar->setAlignment(
Align(8));
1800 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1801 ValuesPtrExpr = ValuesVar;
1805 auto *CounterPtr =
PD.RegionCounters;
1810 auto *IntPtrTy =
M.getDataLayout().getIntPtrType(
M.getContext());
1813 Type *DataTypes[] = {
1814#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1821 Constant *Int16ArrayVals[IPVK_Last + 1];
1823 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty,
PD.NumValueSites[Kind]);
1836 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1837 (
TT.isOSBinFormatELF() ||
1838 (!DataReferencedByCode &&
TT.isOSBinFormatCOFF()))) {
1843 new GlobalVariable(M, DataTy,
false, Linkage,
nullptr, DataVarName);
1846 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
1851 DataSectionKind = IPSK_covdata;
1853 if (BitmapPtr !=
nullptr)
1858 DataSectionKind = IPSK_data;
1859 RelativeCounterPtr =
1862 if (BitmapPtr !=
nullptr)
1869#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1874 Data->setVisibility(Visibility);
1877 Data->setAlignment(
Align(INSTR_PROF_DATA_ALIGNMENT));
1878 maybeSetComdat(
Data, Fn, CntsVarName);
1883 CompilerUsedVars.push_back(
Data);
1889 ReferencedNames.push_back(NamePtr);
1892void InstrLowerer::emitVNodes() {
1893 if (!ValueProfileStaticAlloc)
1903 for (
auto &PD : ProfileDataMap) {
1905 TotalNS +=
PD.second.NumValueSites[Kind];
1911 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1919#define INSTR_PROF_MIN_VAL_COUNTS 10
1923 auto &Ctx =
M.getContext();
1924 Type *VNodeTypes[] = {
1925#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1935 VNodesVar->setSection(
1937 VNodesVar->setAlignment(
M.getDataLayout().getABITypeAlign(VNodesTy));
1940 UsedVars.push_back(VNodesVar);
1943void InstrLowerer::emitNameData() {
1944 std::string UncompressedData;
1946 if (ReferencedNames.empty())
1949 std::string CompressedNameStr;
1955 auto &Ctx =
M.getContext();
1961 NamesSize = CompressedNameStr.size();
1963 NamesVar->setSection(
1970 NamesVar->setAlignment(
Align(1));
1973 UsedVars.push_back(NamesVar);
1975 for (
auto *NamePtr : ReferencedNames)
1979void InstrLowerer::emitVTableNames() {
1984 std::string CompressedVTableNames;
1990 auto &Ctx =
M.getContext();
1992 Ctx,
StringRef(CompressedVTableNames),
false );
2001 UsedVars.push_back(VTableNamesVar);
2004void InstrLowerer::emitRegistration() {
2017 RegisterF->addFnAttr(Attribute::NoRedZone);
2020 auto *RuntimeRegisterF =
2026 if (!isa<Function>(
Data))
2027 IRB.CreateCall(RuntimeRegisterF,
Data);
2029 if (
Data != NamesVar && !isa<Function>(
Data))
2030 IRB.CreateCall(RuntimeRegisterF,
Data);
2033 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
2034 auto *NamesRegisterTy =
2036 auto *NamesRegisterF =
2039 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
2042 IRB.CreateRetVoid();
2045bool InstrLowerer::emitRuntimeHook() {
2048 if (
TT.isOSLinux() ||
TT.isOSAIX())
2062 if (
TT.isOSBinFormatELF() && !
TT.isPS()) {
2064 CompilerUsedVars.push_back(Var);
2070 User->addFnAttr(Attribute::NoInline);
2072 User->addFnAttr(Attribute::NoRedZone);
2074 if (
TT.supportsCOMDAT())
2078 auto *
Load = IRB.CreateLoad(Int32Ty, Var);
2079 IRB.CreateRet(Load);
2082 CompilerUsedVars.push_back(
User);
2087void InstrLowerer::emitUses() {
2097 if (
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO() ||
2098 (
TT.isOSBinFormatCOFF() && !DataReferencedByCode))
2109void InstrLowerer::emitInitialization() {
2126 F->addFnAttr(Attribute::NoInline);
2128 F->addFnAttr(Attribute::NoRedZone);
2132 IRB.CreateCall(RegisterF, {});
2133 IRB.CreateRetVoid();
2141 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
2144 if (SampledInstrPeriod.getValue() <= USHRT_MAX) {
2154 SamplingVar->setThreadLocal(
true);
2155 Triple TT(M.getTargetTriple());
2156 if (TT.supportsCOMDAT()) {
2158 SamplingVar->setComdat(M.getOrInsertComdat(VarName));
This file contains the simple types necessary to represent the attributes associated with functions a...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
static bool lowerIntrinsics(Module &M)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool shouldRecordVTableAddr(GlobalVariable *GV)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static Constant * getVTableAddrForProfData(GlobalVariable *GV)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
Module.h This file contains the declarations for the Module class.
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
Class for arbitrary precision integers.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Annotations lets you mark points and ranges inside source code, for tests:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
const Function * getParent() const
Return the enclosing method, or null if none.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
@ NoDeduplicate
No deduplication is performed.
ConstantArray - Constant Array Declarations.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
bool isZeroValue() const
Return true if the value is negative zero or null value.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Lightweight error class with error context and mandatory checking.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
void setComdat(Comdat *C)
void setSection(StringRef S)
Change the section for this global.
bool hasLinkOnceLinkage() const
VisibilityTypes getVisibility() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LinkageTypes getLinkage() const
bool hasLocalLinkage() const
bool hasPrivateLinkage() const
void setLinkage(LinkageTypes LT)
bool isDeclarationForLinker() const
Module * getParent()
Get the module that this global value is contained inside of...
VisibilityTypes
An enumeration for the kinds of visibility of global values.
@ DefaultVisibility
The GV is visible.
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
bool hasAvailableExternallyLinkage() const
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Value * CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt16(uint16_t C)
Get a constant 16-bit value.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
auto getNumBitmapBytes() const
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
Value * getMCDCCondBitmapAddr() const
ConstantInt * getBitmapIndex() const
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
Value * getTargetValue() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Class to represent integer types.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Helper class for SSA formation on a set of values defined in multiple blocks.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
size - Get the string size.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
Linkage
Describes symbol linkage. This can be used to resolve definition clashes.
This is an optimization pass for GlobalISel generic memory operations.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
void createProfileSamplingVar(Module &M)
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
cl::opt< bool > DoInstrProfNameCompression
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
StringRef getInstrProfVTableNamesVarName()
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
cl::opt< bool > DebugInfoCorrelate
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
StringRef getInstrProfCounterBiasVarName()
auto reverse(ContainerTy &&C)
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfBitmapBiasVarName()
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
void setGlobalVariableLargeSection(const Triple &TargetTriple, GlobalVariable &GV)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
StringRef getInstrProfVTableVarPrefix()
Return the name prefix of variables containing virtual table profile data.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Options for the frontend instrumentation based profiling pass.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.