124#include <unordered_map>
132#define DEBUG_TYPE "pgo-instrumentation"
134STATISTIC(NumOfPGOInstrument,
"Number of edges instrumented.");
135STATISTIC(NumOfPGOSelectInsts,
"Number of select instruction instrumented.");
136STATISTIC(NumOfPGOMemIntrinsics,
"Number of mem intrinsics instrumented.");
139STATISTIC(NumOfPGOSplit,
"Number of critical edge splits.");
140STATISTIC(NumOfPGOFunc,
"Number of functions having valid profile counts.");
141STATISTIC(NumOfPGOMismatch,
"Number of functions having mismatch profile.");
142STATISTIC(NumOfPGOMissing,
"Number of functions without profile.");
143STATISTIC(NumOfPGOICall,
"Number of indirect call value instrumentations.");
144STATISTIC(NumOfCSPGOInstrument,
"Number of edges instrumented in CSPGO.");
146 "Number of select instruction instrumented in CSPGO.");
148 "Number of mem intrinsics instrumented in CSPGO.");
150STATISTIC(NumOfCSPGOBB,
"Number of basic-blocks in CSPGO.");
151STATISTIC(NumOfCSPGOSplit,
"Number of critical edge splits in CSPGO.");
153 "Number of functions having valid profile counts in CSPGO.");
155 "Number of functions having mismatch profile in CSPGO.");
156STATISTIC(NumOfCSPGOMissing,
"Number of functions without profile in CSPGO.");
157STATISTIC(NumCoveredBlocks,
"Number of basic blocks that were executed");
164 cl::desc(
"Specify the path of profile data file. This is"
165 "mainly for test purpose."));
169 cl::desc(
"Specify the path of profile remapping file. This is mainly for "
176 cl::desc(
"Disable Value Profiling"));
182 cl::desc(
"Max number of annotations for a single indirect "
189 cl::desc(
"Max number of preicise value annotations for a single memop"
196 cl::desc(
"Append function hash to the name of COMDAT function to avoid "
197 "function hash mismatch due to the preinliner"));
204 cl::desc(
"Use this option to turn on/off "
205 "warnings about missing profile data for "
212 cl::desc(
"Use this option to turn off/on "
213 "warnings about profile cfg mismatch."));
220 cl::desc(
"The option is used to turn on/off "
221 "warnings about hash mismatch for comdat "
222 "or weak functions."));
228 cl::desc(
"Use this option to turn on/off SELECT "
229 "instruction instrumentation. "));
234 cl::desc(
"A boolean option to show CFG dag or text "
235 "with raw profile counts from "
236 "profile data. See also option "
237 "-pgo-view-counts. To limit graph "
238 "display to only one function, use "
239 "filtering option -view-bfi-func-name."),
247 cl::desc(
"Use this option to turn on/off "
248 "memory intrinsic size profiling."));
253 cl::desc(
"When this option is on, the annotated "
254 "branch probability will be emitted as "
255 "optimization remarks: -{Rpass|"
256 "pass-remarks}=pgo-instrumentation"));
260 cl::desc(
"Force to instrument function entry basicblock."));
265 "Use this option to enable function entry coverage instrumentation."));
268 "pgo-block-coverage",
269 cl::desc(
"Use this option to enable basic block coverage instrumentation"));
273 cl::desc(
"Create a dot file of CFGs with block "
274 "coverage inference information"));
277 "pgo-temporal-instrumentation",
278 cl::desc(
"Use this option to enable temporal instrumentation"));
282 cl::desc(
"Fix function entry count in profile use."));
286 cl::desc(
"Print out the non-match BFI count if a hot raw profile count "
287 "becomes non-hot, or a cold raw profile count becomes hot. "
288 "The print is enabled under -Rpass-analysis=pgo, or "
289 "internal option -pass-remakrs-analysis=pgo."));
293 cl::desc(
"Print out mismatched BFI counts after setting profile metadata "
294 "The print is enabled under -Rpass-analysis=pgo, or "
295 "internal option -pass-remakrs-analysis=pgo."));
299 cl::desc(
"Set the threshold for pgo-verify-bfi: only print out "
300 "mismatched BFI if the difference percentage is greater than "
301 "this value (in percentage)."));
305 cl::desc(
"Set the threshold for pgo-verify-bfi: skip the counts whose "
306 "profile count value is below."));
311 cl::desc(
"Trace the hash of the function with this name."));
315 cl::desc(
"Do not instrument functions smaller than this threshold."));
319 cl::desc(
"Do not instrument functions with the number of critical edges "
320 " greater than this threshold."));
341class FunctionInstrumenter final {
345 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
355 bool isValueProfilingDisabled()
const {
357 InstrumentationType == PGOInstrumentationType::CTXPROF;
360 bool shouldInstrumentEntryBB()
const {
362 InstrumentationType == PGOInstrumentationType::CTXPROF;
366 FunctionInstrumenter(
368 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
371 : M(M),
F(
F), TLI(TLI), ComdatMembers(ComdatMembers), BPI(BPI), BFI(BFI),
372 InstrumentationType(InstrumentationType) {}
383 return std::string();
388 return std::string();
400 else if (CV->
isOne())
412#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,
421 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
423 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
424 if (InstrumentationType == PGOInstrumentationType::CSFDO)
425 ProfileVersion |= VARIANT_MASK_CSIR_PROF;
427 InstrumentationType == PGOInstrumentationType::CTXPROF)
428 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
430 ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
433 VARIANT_MASK_BYTE_COVERAGE | VARIANT_MASK_FUNCTION_ENTRY_ONLY;
435 ProfileVersion |= VARIANT_MASK_BYTE_COVERAGE;
437 ProfileVersion |= VARIANT_MASK_TEMPORAL_PROF;
442 Triple TT(M.getTargetTriple());
443 if (TT.supportsCOMDAT()) {
445 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
447 return IRLevelVersionVariable;
457enum VisitMode { VM_counting, VM_instrument, VM_annotate };
461struct SelectInstVisitor :
public InstVisitor<SelectInstVisitor> {
464 VisitMode
Mode = VM_counting;
465 unsigned *CurCtrIdx =
nullptr;
466 unsigned TotalNumCtrs = 0;
469 PGOUseFunc *UseFunc =
nullptr;
470 bool HasSingleByteCoverage;
472 SelectInstVisitor(
Function &Func,
bool HasSingleByteCoverage)
473 :
F(
Func), HasSingleByteCoverage(HasSingleByteCoverage) {}
475 void countSelects() {
485 void instrumentSelects(
unsigned *Ind,
unsigned TotalNC,
GlobalVariable *FNV,
487 Mode = VM_instrument;
489 TotalNumCtrs = TotalNC;
496 void annotateSelects(PGOUseFunc *UF,
unsigned *Ind) {
511 unsigned getNumOfSelectInsts()
const {
return NSIs; }
523 bool Removed =
false;
524 bool IsCritical =
false;
527 : SrcBB(Src), DestBB(Dest), Weight(
W) {}
530 std::string infoString()
const {
531 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
532 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight))
543 PGOBBInfo(
unsigned IX) : Group(this),
Index(IX) {}
546 std::string infoString()
const {
552template <
class Edge,
class BBInfo>
class FuncPGOInstrumentation {
560 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
564 void computeCFGHash();
565 void renameComdatFunction();
569 std::vector<std::vector<VPCandidateInfo>> ValueSites;
570 SelectInstVisitor SIVisitor;
571 std::string FuncName;
572 std::string DeprecatedFuncName;
581 const std::optional<BlockCoverageInference> BCI;
583 static std::optional<BlockCoverageInference>
584 constructBCI(
Function &Func,
bool HasSingleByteCoverage,
585 bool InstrumentFuncEntry) {
586 if (HasSingleByteCoverage)
593 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);
606 void dumpInfo(
StringRef Str =
"")
const {
608 " Hash: " +
Twine(FunctionHash) +
"\t" + Str);
611 FuncPGOInstrumentation(
613 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
616 bool InstrumentFuncEntry =
true,
bool HasSingleByteCoverage =
false)
617 :
F(
Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(
Func, TLI),
618 TLI(TLI), ValueSites(IPVK_Last + 1),
619 SIVisitor(
Func, HasSingleByteCoverage),
620 MST(
F, InstrumentFuncEntry, BPI,
BFI),
621 BCI(constructBCI(
Func, HasSingleByteCoverage, InstrumentFuncEntry)) {
623 BCI->viewBlockCoverageGraph();
625 SIVisitor.countSelects();
626 ValueSites[IPVK_MemOPSize] = VPC.
get(IPVK_MemOPSize);
628 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
629 NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
631 ValueSites[IPVK_IndirectCallTarget] = VPC.
get(IPVK_IndirectCallTarget);
633 ValueSites[IPVK_VTableTarget] = VPC.
get(IPVK_VTableTarget);
635 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
636 NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
643 if (!ComdatMembers.empty())
644 renameComdatFunction();
647 for (
const auto &E : MST.
allEdges()) {
650 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
652 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
665template <
class Edge,
class BBInfo>
666void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
667 std::vector<uint8_t> Indexes;
671 auto BI = findBBInfo(Succ);
675 for (
int J = 0; J < 4; J++)
676 Indexes.push_back((uint8_t)(
Index >> (J * 8)));
683 auto updateJCH = [&JCH](
uint64_t Num) {
688 updateJCH((
uint64_t)SIVisitor.getNumOfSelectInsts());
689 updateJCH((
uint64_t)ValueSites[IPVK_IndirectCallTarget].
size());
692 updateJCH(BCI->getInstrumentedBlocksHash());
702 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
705 LLVM_DEBUG(
dbgs() <<
"Function Hash Computation for " <<
F.getName() <<
":\n"
706 <<
" CRC = " << JC.
getCRC()
707 <<
", Selects = " << SIVisitor.getNumOfSelectInsts()
708 <<
", Edges = " << MST.
numEdges() <<
", ICSites = "
709 << ValueSites[IPVK_IndirectCallTarget].size()
710 <<
", Memops = " << ValueSites[IPVK_MemOPSize].size()
711 <<
", High32 CRC = " << JCH.
getCRC()
712 <<
", Hash = " << FunctionHash <<
"\n";);
715 dbgs() <<
"Funcname=" <<
F.getName() <<
", Hash=" << FunctionHash
716 <<
" in building " <<
F.getParent()->getSourceFileName() <<
"\n";
722 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
734 for (
auto &&CM :
make_range(ComdatMembers.equal_range(
C))) {
735 assert(!isa<GlobalAlias>(CM.second));
736 Function *FM = dyn_cast<Function>(CM.second);
744template <
class Edge,
class BBInfo>
745void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
748 std::string OrigName =
F.getName().str();
749 std::string NewFuncName =
751 F.setName(
Twine(NewFuncName));
753 FuncName =
Twine(FuncName +
"." +
Twine(FunctionHash)).
str();
759 if (!
F.hasComdat()) {
761 NewComdat =
M->getOrInsertComdat(
StringRef(NewFuncName));
763 F.setComdat(NewComdat);
768 Comdat *OrigComdat =
F.getComdat();
769 std::string NewComdatName =
771 NewComdat =
M->getOrInsertComdat(
StringRef(NewComdatName));
774 for (
auto &&CM :
make_range(ComdatMembers.equal_range(OrigComdat))) {
776 cast<Function>(CM.second)->setComdat(NewComdat);
782template <
class Edge,
class BBInfo>
783void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
784 std::vector<BasicBlock *> &InstrumentBBs) {
787 if (BCI->shouldInstrumentBlock(BB))
788 InstrumentBBs.push_back(&BB);
793 std::vector<Edge *> EdgeList;
795 for (
const auto &E : MST.
allEdges())
796 EdgeList.push_back(E.get());
798 for (
auto &E : EdgeList) {
801 InstrumentBBs.push_back(InstrBB);
807template <
class Edge,
class BBInfo>
808BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
809 if (E->InMST || E->Removed)
815 if (SrcBB ==
nullptr)
817 if (DestBB ==
nullptr)
832 return canInstrument(SrcBB);
834 return canInstrument(DestBB);
843 dbgs() <<
"Fail to split critical edge: not instrument this edge.\n");
848 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
850 <<
" --> " << getBBInfo(DestBB).
Index <<
"\n");
852 MST.
addEdge(SrcBB, InstrBB, 0);
854 Edge &NewEdge1 = MST.
addEdge(InstrBB, DestBB, 0);
855 NewEdge1.InMST =
true;
858 return canInstrument(InstrBB);
874 if (!isa<IntrinsicInst>(OrigCall)) {
877 std::optional<OperandBundleUse> ParentFunclet =
885 if (!BlockColors.
empty()) {
886 const ColorVector &CV = BlockColors.
find(OrigCall->getParent())->second;
887 assert(CV.
size() == 1 &&
"non-unique color for block!");
897void FunctionInstrumenter::instrument() {
904 FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
905 F, TLI, ComdatMembers,
true, BPI, BFI,
906 InstrumentationType == PGOInstrumentationType::CSFDO,
909 auto Name = FuncInfo.FuncNameVar;
913 auto &EntryBB =
F.getEntryBlock();
914 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
919 {
Name, CFGHash, Builder.getInt32(1), Builder.getInt32(0)});
923 std::vector<BasicBlock *> InstrumentBBs;
924 FuncInfo.getInstrumentBBs(InstrumentBBs);
925 unsigned NumCounters =
926 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
928 if (InstrumentationType == PGOInstrumentationType::CTXPROF) {
941 for (
auto &Instr : BB)
942 if (
auto *CS = dyn_cast<CallBase>(&Instr)) {
943 if ((CS->getCalledFunction() &&
944 CS->getCalledFunction()->isIntrinsic()) ||
945 dyn_cast<InlineAsm>(CS->getCalledOperand()))
952 Visit([&TotalNrCallsites](
auto *) { ++TotalNrCallsites; });
956 Visit([&](
auto *CB) {
958 Builder.CreateCall(CSIntrinsic,
959 {
Name, CFGHash, Builder.getInt32(TotalNrCallsites),
960 Builder.getInt32(CallsiteIndex++),
961 CB->getCalledOperand()});
968 auto &EntryBB =
F.getEntryBlock();
969 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
974 {
Name, CFGHash, Builder.getInt32(NumCounters), Builder.getInt32(
I)});
978 for (
auto *InstrBB : InstrumentBBs) {
980 assert(Builder.GetInsertPoint() != InstrBB->
end() &&
981 "Cannot get the Instrumentation point");
986 ? Intrinsic::instrprof_cover
987 : Intrinsic::instrprof_increment),
988 {
Name, CFGHash, Builder.getInt32(NumCounters), Builder.getInt32(
I++)});
992 FuncInfo.SIVisitor.instrumentSelects(&
I, NumCounters, FuncInfo.FuncNameVar,
993 FuncInfo.FunctionHash);
996 if (isValueProfilingDisabled())
999 NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();
1006 if (
F.hasPersonalityFn() &&
1012 unsigned SiteIndex = 0;
1018 <<
" site: CallSite Index = " << SiteIndex <<
"\n");
1021 assert(Builder.GetInsertPoint() != Cand.InsertPt->getParent()->end() &&
1022 "Cannot get the Instrumentation point");
1024 Value *ToProfile =
nullptr;
1025 if (Cand.V->getType()->isIntegerTy())
1026 ToProfile = Builder.CreateZExtOrTrunc(Cand.V, Builder.getInt64Ty());
1027 else if (Cand.V->getType()->isPointerTy())
1028 ToProfile = Builder.CreatePtrToInt(Cand.V, Builder.getInt64Ty());
1029 assert(ToProfile &&
"value profiling Value is of unexpected type");
1035 {FuncInfo.FuncNameVar, Builder.getInt64(FuncInfo.FunctionHash),
1036 ToProfile, Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)},
1045struct PGOUseEdge :
public PGOEdge {
1046 using PGOEdge::PGOEdge;
1048 std::optional<uint64_t> Count;
1054 std::string infoString()
const {
1056 return PGOEdge::infoString();
1057 return (
Twine(PGOEdge::infoString()) +
" Count=" +
Twine(*Count)).str();
1064struct PGOUseBBInfo :
public PGOBBInfo {
1065 std::optional<uint64_t> Count;
1066 int32_t UnknownCountInEdge = 0;
1067 int32_t UnknownCountOutEdge = 0;
1068 DirectEdges InEdges;
1069 DirectEdges OutEdges;
1071 PGOUseBBInfo(
unsigned IX) : PGOBBInfo(IX) {}
1077 std::string infoString()
const {
1079 return PGOBBInfo::infoString();
1080 return (
Twine(PGOBBInfo::infoString()) +
" Count=" +
Twine(*Count)).str();
1084 void addOutEdge(PGOUseEdge *E) {
1085 OutEdges.push_back(E);
1086 UnknownCountOutEdge++;
1090 void addInEdge(PGOUseEdge *E) {
1091 InEdges.push_back(E);
1092 UnknownCountInEdge++;
1101 for (
const auto &E : Edges) {
1115 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
1118 bool HasSingleByteCoverage)
1119 :
F(
Func),
M(Modu),
BFI(BFIin), PSI(PSI),
1120 FuncInfo(
Func, TLI, ComdatMembers,
false, BPI, BFIin, IsCS,
1121 InstrumentFuncEntry, HasSingleByteCoverage),
1122 FreqAttr(FFA_Normal), IsCS(IsCS), VPC(
Func, TLI) {}
1124 void handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum);
1131 void populateCounters();
1140 void annotateValueSites();
1143 void annotateValueSites(
uint32_t Kind);
1146 void annotateIrrLoopHeaderWeights();
1149 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
1152 FuncFreqAttr getFuncFreqAttr()
const {
return FreqAttr; }
1161 PGOUseBBInfo &getBBInfo(
const BasicBlock *BB)
const {
1162 return FuncInfo.getBBInfo(BB);
1166 PGOUseBBInfo *findBBInfo(
const BasicBlock *BB)
const {
1167 return FuncInfo.findBBInfo(BB);
1172 void dumpInfo(
StringRef Str =
"")
const { FuncInfo.dumpInfo(Str); }
1174 uint64_t getProgramMaxCount()
const {
return ProgramMaxCount; }
1183 FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> FuncInfo;
1199 FuncFreqAttr FreqAttr;
1207 bool setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile);
1220 FreqAttr = FFA_Cold;
1228 const FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> &FuncInfo) {
1232 for (
const auto &E : FuncInfo.MST.allEdges()) {
1237 PGOUseBBInfo &SrcInfo = FuncInfo.getBBInfo(SrcBB);
1238 PGOUseBBInfo &DestInfo = FuncInfo.getBBInfo(DestBB);
1239 SrcInfo.addOutEdge(E.get());
1240 DestInfo.addInEdge(E.get());
1246bool PGOUseFunc::setInstrumentedCounts(
1247 const std::vector<uint64_t> &CountFromProfile) {
1249 std::vector<BasicBlock *> InstrumentBBs;
1250 FuncInfo.getInstrumentBBs(InstrumentBBs);
1254 unsigned NumCounters =
1255 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
1258 if (NumCounters != CountFromProfile.size()) {
1261 auto *FuncEntry = &*
F.begin();
1266 uint64_t CountValue = CountFromProfile[
I++];
1267 PGOUseBBInfo &
Info = getBBInfo(InstrBB);
1271 if (InstrBB == FuncEntry && CountValue == 0)
1273 Info.setBBInfoCount(CountValue);
1275 ProfileCountSize = CountFromProfile.size();
1279 auto setEdgeCount = [
this](PGOUseEdge *E,
uint64_t Value) ->
void {
1280 E->setEdgeCount(
Value);
1281 this->getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1282 this->getBBInfo(E->DestBB).UnknownCountInEdge--;
1288 for (
const auto &E : FuncInfo.MST.allEdges()) {
1289 if (E->Removed || E->InMST)
1292 PGOUseBBInfo &SrcInfo = getBBInfo(SrcBB);
1296 if (SrcInfo.Count && SrcInfo.OutEdges.size() == 1)
1297 setEdgeCount(E.get(), *SrcInfo.Count);
1300 PGOUseBBInfo &DestInfo = getBBInfo(DestBB);
1303 if (DestInfo.Count && DestInfo.InEdges.size() == 1)
1304 setEdgeCount(E.get(), *DestInfo.Count);
1310 setEdgeCount(E.get(), 0);
1317void PGOUseFunc::setEdgeCount(DirectEdges &Edges,
uint64_t Value) {
1318 for (
auto &E : Edges) {
1321 E->setEdgeCount(
Value);
1323 getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1324 getBBInfo(E->DestBB).UnknownCountInEdge--;
1332 const char MetadataName[] =
"instr_prof_hash_mismatch";
1335 auto *Existing =
F.getMetadata(LLVMContext::MD_annotation);
1337 MDTuple *Tuple = cast<MDTuple>(Existing);
1338 for (
const auto &
N : Tuple->
operands()) {
1339 if (
N.equalsStr(MetadataName))
1348 F.setMetadata(LLVMContext::MD_annotation, MD);
1351void PGOUseFunc::handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum) {
1353 auto &Ctx =
M->getContext();
1354 auto Err = IPE.
get();
1355 bool SkipWarning =
false;
1357 << FuncInfo.FuncName <<
": ");
1358 if (Err == instrprof_error::unknown_function) {
1359 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
1362 }
else if (Err == instrprof_error::hash_mismatch ||
1363 Err == instrprof_error::malformed) {
1364 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
1370 LLVM_DEBUG(
dbgs() <<
"hash mismatch (hash= " << FuncInfo.FunctionHash
1371 <<
" skip=" << SkipWarning <<
")");
1381 IPE.
message() + std::string(
" ") +
F.getName().str() +
1382 std::string(
" Hash = ") + std::to_string(FuncInfo.FunctionHash) +
1383 std::string(
" up to ") + std::to_string(MismatchedFuncSum) +
1384 std::string(
" count discarded");
1396 auto &Ctx =
M->getContext();
1399 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1400 &MismatchedFuncSum);
1402 handleInstrProfError(std::move(E), MismatchedFuncSum);
1405 ProfileRecord = std::move(
Result.get());
1410 std::vector<uint64_t> &CountFromProfile = ProfileRecord.
Counts;
1412 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1416 for (
unsigned I = 0, S = CountFromProfile.size();
I < S;
I++) {
1418 ValueSum += CountFromProfile[
I];
1420 AllZeros = (ValueSum == 0);
1424 getBBInfo(
nullptr).UnknownCountOutEdge = 2;
1425 getBBInfo(
nullptr).UnknownCountInEdge = 2;
1427 if (!setInstrumentedCounts(CountFromProfile)) {
1429 dbgs() <<
"Inconsistent number of counts, skipping this function");
1431 M->getName().data(),
1432 Twine(
"Inconsistent number of counts in ") +
F.getName().str() +
1433 Twine(
": the profile may be stale or there is a function name "
1445 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1446 &MismatchedFuncSum);
1447 if (
auto Err =
Result.takeError()) {
1448 handleInstrProfError(std::move(Err), MismatchedFuncSum);
1451 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1453 std::vector<uint64_t> &CountsFromProfile =
Result.get().Counts;
1457 if (FuncInfo.BCI->shouldInstrumentBlock(BB))
1463 InverseDependencies;
1464 for (
auto &BB :
F) {
1465 for (
auto *Dep : FuncInfo.BCI->getDependencies(BB)) {
1467 InverseDependencies[Dep].
insert(&BB);
1472 std::stack<const BasicBlock *> CoveredBlocksToProcess;
1473 for (
auto &[BB, IsCovered] : Coverage)
1475 CoveredBlocksToProcess.push(BB);
1477 while (!CoveredBlocksToProcess.empty()) {
1478 auto *CoveredBlock = CoveredBlocksToProcess.top();
1479 assert(Coverage[CoveredBlock]);
1480 CoveredBlocksToProcess.pop();
1481 for (
auto *BB : InverseDependencies[CoveredBlock]) {
1486 CoveredBlocksToProcess.push(BB);
1494 F.setEntryCount(Coverage[&
F.getEntryBlock()] ? 10000 : 0);
1495 for (
auto &BB :
F) {
1504 Weights.
push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);
1505 if (Weights.
size() >= 2)
1510 unsigned NumCorruptCoverage = 0;
1515 auto IsBlockDead = [&](
const BasicBlock &BB) -> std::optional<bool> {
1516 if (
auto C =
BFI.getBlockProfileCount(&BB))
1520 LLVM_DEBUG(
dbgs() <<
"Block Coverage: (Instrumented=*, Covered=X)\n");
1521 for (
auto &BB :
F) {
1522 LLVM_DEBUG(
dbgs() << (FuncInfo.BCI->shouldInstrumentBlock(BB) ?
"* " :
" ")
1523 << (Coverage[&BB] ?
"X " :
" ") <<
" " << BB.getName()
1529 if (Coverage[&BB] == IsBlockDead(BB).value_or(
false)) {
1531 dbgs() <<
"Found inconsistent block covearge for " << BB.getName()
1532 <<
": BCI=" << (Coverage[&BB] ?
"Covered" :
"Dead") <<
" BFI="
1533 << (IsBlockDead(BB).
value() ?
"Dead" :
"Covered") <<
"\n");
1534 ++NumCorruptCoverage;
1540 auto &Ctx =
M->getContext();
1542 M->getName().data(),
1543 Twine(
"Found inconsistent block coverage for function ") +
F.getName() +
1544 " in " +
Twine(NumCorruptCoverage) +
" blocks.",
1548 FuncInfo.BCI->viewBlockCoverageGraph(&Coverage);
1553void PGOUseFunc::populateCounters() {
1554 bool Changes =
true;
1555 unsigned NumPasses = 0;
1563 PGOUseBBInfo *UseBBInfo = findBBInfo(&BB);
1564 if (UseBBInfo ==
nullptr)
1566 if (!UseBBInfo->Count) {
1567 if (UseBBInfo->UnknownCountOutEdge == 0) {
1570 }
else if (UseBBInfo->UnknownCountInEdge == 0) {
1575 if (UseBBInfo->Count) {
1576 if (UseBBInfo->UnknownCountOutEdge == 1) {
1582 if (*UseBBInfo->Count > OutSum)
1583 Total = *UseBBInfo->Count - OutSum;
1584 setEdgeCount(UseBBInfo->OutEdges,
Total);
1587 if (UseBBInfo->UnknownCountInEdge == 1) {
1590 if (*UseBBInfo->Count > InSum)
1591 Total = *UseBBInfo->Count - InSum;
1592 setEdgeCount(UseBBInfo->InEdges,
Total);
1599 LLVM_DEBUG(
dbgs() <<
"Populate counts in " << NumPasses <<
" passes.\n");
1603 for (
auto &BB :
F) {
1604 auto BI = findBBInfo(&BB);
1607 assert(BI->Count &&
"BB count is not valid");
1612 for (
auto &BB :
F) {
1613 auto BI = findBBInfo(&BB);
1616 FuncMaxCount = std::max(FuncMaxCount, *BI->Count);
1626 FuncInfo.SIVisitor.annotateSelects(
this, &CountPosition);
1627 assert(CountPosition == ProfileCountSize);
1629 LLVM_DEBUG(FuncInfo.dumpInfo(
"after reading profile."));
1633void PGOUseFunc::setBranchWeights() {
1635 LLVM_DEBUG(
dbgs() <<
"\nSetting branch weights for func " <<
F.getName()
1636 <<
" IsCS=" << IsCS <<
"\n");
1637 for (
auto &BB :
F) {
1641 if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
1642 isa<IndirectBrInst>(TI) || isa<InvokeInst>(TI) ||
1643 isa<CallBrInst>(TI)))
1646 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1647 if (!*BBCountInfo.Count)
1651 unsigned Size = BBCountInfo.OutEdges.size();
1654 for (
unsigned s = 0; s <
Size; s++) {
1655 const PGOUseEdge *E = BBCountInfo.OutEdges[s];
1658 if (DestBB ==
nullptr)
1662 if (EdgeCount > MaxCount)
1663 MaxCount = EdgeCount;
1664 EdgeCounts[SuccNum] = EdgeCount;
1673 auto &Ctx =
M->getContext();
1675 M->getName().data(),
1676 Twine(
"Profile in ") +
F.getName().str() +
1677 Twine(
" partially ignored") +
1678 Twine(
", possibly due to the lack of a return path."),
1686 if (isa<IndirectBrInst>(Pred->getTerminator()))
1692void PGOUseFunc::annotateIrrLoopHeaderWeights() {
1693 LLVM_DEBUG(
dbgs() <<
"\nAnnotating irreducible loop header weights.\n");
1695 for (
auto &BB :
F) {
1701 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1707void SelectInstVisitor::instrumentOneSelectInst(
SelectInst &SI) {
1711 auto *Step = Builder.CreateZExt(
SI.getCondition(), Int64Ty);
1714 {FuncNameVar, Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs),
1715 Builder.getInt32(*CurCtrIdx), Step});
1719void SelectInstVisitor::annotateOneSelectInst(
SelectInst &SI) {
1720 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1721 assert(*CurCtrIdx < CountFromProfile.size() &&
1722 "Out of bound access of counters");
1724 SCounts[0] = CountFromProfile[*CurCtrIdx];
1727 auto BI = UseFunc->findBBInfo(
SI.getParent());
1729 TotalCount = *BI->Count;
1731 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1732 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1737void SelectInstVisitor::visitSelectInst(
SelectInst &SI) {
1741 if (
SI.getCondition()->getType()->isVectorTy())
1749 instrumentOneSelectInst(SI);
1752 annotateOneSelectInst(SI);
1760 if (ValueProfKind == IPVK_MemOPSize)
1762 if (ValueProfKind == llvm::IPVK_VTableTarget)
1768void PGOUseFunc::annotateValueSites() {
1776 annotateValueSites(Kind);
1780void PGOUseFunc::annotateValueSites(
uint32_t Kind) {
1781 assert(Kind <= IPVK_Last);
1782 unsigned ValueSiteIndex = 0;
1795 if (NumValueSites > 0 && Kind == IPVK_VTableTarget &&
1796 NumValueSites != FuncInfo.ValueSites[IPVK_VTableTarget].size() &&
1798 FuncInfo.ValueSites[IPVK_VTableTarget] = VPC.
get(IPVK_VTableTarget);
1799 auto &ValueSites = FuncInfo.ValueSites[
Kind];
1800 if (NumValueSites != ValueSites.size()) {
1801 auto &Ctx =
M->getContext();
1803 M->getName().data(),
1804 Twine(
"Inconsistent number of value sites for ") +
1807 Twine(
"\", possibly due to the use of a stale profile."),
1813 LLVM_DEBUG(
dbgs() <<
"Read one value site profile (kind = " << Kind
1814 <<
"): Index = " << ValueSiteIndex <<
" out of "
1815 << NumValueSites <<
"\n");
1817 *M, *
I.AnnotatedInst, ProfileRecord,
1828 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1833 ComdatMembers.insert(std::make_pair(
C, &
F));
1835 if (
Comdat *
C = GV.getComdat())
1836 ComdatMembers.insert(std::make_pair(
C, &GV));
1838 if (
Comdat *
C = GA.getComdat())
1839 ComdatMembers.insert(std::make_pair(
C, &GA));
1844 if (
F.isDeclaration())
1849 unsigned NumCriticalEdges = 0;
1850 for (
auto &BB :
F) {
1859 <<
", NumCriticalEdges=" << NumCriticalEdges
1860 <<
" exceed the threshold. Skip PGO.\n");
1870 if (
F.hasFnAttribute(llvm::Attribute::Naked))
1872 if (
F.hasFnAttribute(llvm::Attribute::NoProfile))
1874 if (
F.hasFnAttribute(llvm::Attribute::SkipProfile))
1888 if (InstrumentationType == PGOInstrumentationType::FDO)
1891 Triple TT(M.getTargetTriple());
1896 Twine(
"VTable value profiling is presently not "
1897 "supported for non-ELF object formats"),
1899 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1905 auto &TLI = LookupTLI(
F);
1906 auto *BPI = LookupBPI(
F);
1907 auto *BFI = LookupBFI(
F);
1908 FunctionInstrumenter FI(M,
F, TLI, ComdatMembers, BPI, BFI,
1909 InstrumentationType);
1922 if (ProfileSampling)
1944 InstrumentationType))
1957 auto BFIEntryCount =
F.getEntryCount();
1958 assert(BFIEntryCount && (BFIEntryCount->getCount() > 0) &&
1959 "Invalid BFI Entrycount");
1963 for (
auto &BBI :
F) {
1966 if (!Func.findBBInfo(&BBI))
1969 CountValue = *Func.getBBInfo(&BBI).Count;
1970 BFICountValue = *BFICount;
1974 if (SumCount.isZero())
1978 "Incorrect sum of BFI counts");
1981 double Scale = (SumCount / SumBFICount).convertToDouble();
1982 if (Scale < 1.001 && Scale > 0.999)
1987 if (NewEntryCount == 0)
1993 << NewEntryCount <<
"\n");
2010 unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
2011 for (
auto &BBI :
F) {
2015 CountValue = Func.getBBInfo(&BBI).Count.value_or(CountValue);
2022 BFICountValue = *BFICount;
2025 bool rawIsHot = CountValue >= HotCountThreshold;
2026 bool BFIIsHot = BFICountValue >= HotCountThreshold;
2028 bool ShowCount =
false;
2029 if (rawIsHot && !BFIIsHot) {
2030 Msg =
"raw-Hot to BFI-nonHot";
2032 }
else if (rawIsCold && BFIIsHot) {
2033 Msg =
"raw-Cold to BFI-Hot";
2042 uint64_t Diff = (BFICountValue >= CountValue)
2043 ? BFICountValue - CountValue
2044 : CountValue - BFICountValue;
2052 F.getSubprogram(), &BBI);
2054 <<
" Count=" <<
ore::NV(
"Count", CountValue)
2055 <<
" BFI_Count=" <<
ore::NV(
"Count", BFICountValue);
2057 Remark <<
" (" << Msg <<
")";
2064 F.getSubprogram(), &
F.getEntryBlock())
2065 <<
"In Func " <<
ore::NV(
"Function",
F.getName())
2066 <<
": Num_of_BB=" <<
ore::NV(
"Count", BBNum)
2067 <<
", Num_of_non_zerovalue_BB=" <<
ore::NV(
"Count", NonZeroBBNum)
2068 <<
", Num_of_mis_matching_BB=" <<
ore::NV(
"Count", BBMisMatchNum);
2080 auto &Ctx = M.getContext();
2083 ProfileRemappingFileName);
2084 if (
Error E = ReaderOrErr.takeError()) {
2092 std::unique_ptr<IndexedInstrProfReader> PGOReader =
2093 std::move(ReaderOrErr.get());
2099 if (!PGOReader->hasCSIRLevelProfile() && IsCS)
2103 if (!PGOReader->isIRLevelProfile()) {
2105 ProfileFileName.
data(),
"Not an IR level instrumentation profile"));
2108 if (PGOReader->functionEntryOnly()) {
2110 ProfileFileName.
data(),
2111 "Function entry profiles are not yet supported for optimization"));
2117 if (!
G.hasName() || !
G.hasMetadata(LLVMContext::MD_type))
2128 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
2133 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
2135 std::vector<Function *> HotFunctions;
2136 std::vector<Function *> ColdFunctions;
2140 bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
2144 bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
2148 auto &TLI = LookupTLI(
F);
2149 auto *BPI = LookupBPI(
F);
2150 auto *BFI = LookupBFI(
F);
2151 if (!HasSingleByteCoverage) {
2157 PGOUseFunc Func(
F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
2158 InstrumentFuncEntry, HasSingleByteCoverage);
2159 if (HasSingleByteCoverage) {
2160 Func.populateCoverage(PGOReader.get());
2168 bool AllZeros =
false;
2169 if (!Func.readCounters(PGOReader.get(), AllZeros, PseudoKind))
2173 if (Func.getProgramMaxCount() != 0)
2174 ColdFunctions.push_back(&
F);
2179 if (
F.hasFnAttribute(Attribute::Cold))
2180 F.removeFnAttr(Attribute::Cold);
2183 F.addFnAttr(Attribute::Hot);
2186 Func.populateCounters();
2187 Func.setBranchWeights();
2188 Func.annotateValueSites();
2189 Func.annotateIrrLoopHeaderWeights();
2190 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
2191 if (FreqAttr == PGOUseFunc::FFA_Cold)
2192 ColdFunctions.push_back(&
F);
2193 else if (FreqAttr == PGOUseFunc::FFA_Hot)
2194 HotFunctions.push_back(&
F);
2199 std::unique_ptr<BranchProbabilityInfo> NewBPI =
2200 std::make_unique<BranchProbabilityInfo>(
F, LI);
2201 std::unique_ptr<BlockFrequencyInfo> NewBFI =
2202 std::make_unique<BlockFrequencyInfo>(
F, *NewBPI, LI);
2206 dbgs() <<
"pgo-view-counts: " << Func.getFunc().getName() <<
"\n";
2207 NewBFI->print(
dbgs());
2217 ViewGraph(&Func,
Twine(
"PGORawCounts_") + Func.getFunc().getName());
2219 dbgs() <<
"pgo-view-raw-counts: " << Func.getFunc().getName() <<
"\n";
2246 for (
auto &
F : HotFunctions) {
2247 F->addFnAttr(Attribute::InlineHint);
2248 LLVM_DEBUG(
dbgs() <<
"Set inline attribute to function: " <<
F->getName()
2251 for (
auto &
F : ColdFunctions) {
2254 if (
F->hasFnAttribute(Attribute::Hot)) {
2255 auto &Ctx = M.getContext();
2256 std::string Msg = std::string(
"Function ") +
F->getName().str() +
2257 std::string(
" is annotated as a hot function but"
2258 " the profile is cold");
2263 F->addFnAttr(Attribute::Cold);
2264 LLVM_DEBUG(
dbgs() <<
"Set cold attribute to function: " <<
F->getName()
2271 std::string Filename, std::string RemappingFilename,
bool IsCS,
2273 : ProfileFileName(
std::
move(Filename)),
2274 ProfileRemappingFileName(
std::
move(RemappingFilename)), IsCS(IsCS),
2300 LookupTLI, LookupBPI, LookupBFI, PSI, IsCS))
2307 if (!
Node->getName().empty())
2308 return Node->getName().str();
2310 std::string SimpleNodeName;
2313 return SimpleNodeName;
2318 assert(MaxCount > 0 &&
"Bad max count");
2321 for (
const auto &ECI : EdgeCounts)
2334 if (BrCondStr.empty())
2346 std::string BranchProbStr;
2349 OS <<
" (total count : " << TotalCount <<
")";
2355 << BrCondStr <<
" is true with probability : " << BranchProbStr;
2374 return &
G->getFunc().front();
2397 return std::string(
G->getFunc().getName());
2405 PGOUseBBInfo *BI = Graph->findBBInfo(Node);
2407 if (BI && BI->Count)
2408 OS << *BI->Count <<
"\\l";
2416 if (!isa<SelectInst>(&
I))
2419 OS <<
"SELECT : { T = ";
2423 OS <<
"Unknown, F = Unknown }\\l";
2425 OS << TC <<
", F = " << FC <<
" }\\l";
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
This file finds the minimum set of blocks on a CFG that must be instrumented to infer execution cover...
Analysis containing CSE Info
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Given that RA is a live value
post inline ee instrument
static BasicBlock * getInstrBB(CFGMST< Edge, BBInfo > &MST, Edge &E, const DenseSet< const BasicBlock * > &ExecBlocks)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
Module.h This file contains the declarations for the Module class.
static cl::opt< bool > PGOInstrumentEntry("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))
static GlobalVariable * createIRLevelProfileFlagVar(Module &M, PGOInstrumentationType InstrumentationType)
static cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static cl::opt< bool > PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))
static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)
static cl::opt< bool > PGOVerifyHotBFI("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx)
cl::opt< unsigned > MaxNumVTableAnnotations
static cl::opt< bool > PGOTemporalInstrumentation("pgo-temporal-instrumentation", cl::desc("Use this option to enable temporal instrumentation"))
static cl::opt< unsigned > PGOFunctionSizeThreshold("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))
static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static bool skipPGOGen(const Function &F)
static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< unsigned > PGOVerifyBFICutoff("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))
static cl::opt< std::string > PGOTraceFuncHash("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))
static void populateEHOperandBundle(VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)
static bool InstrumentAllFunctions(Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, PGOInstrumentationType InstrumentationType)
static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))
static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)
static cl::opt< unsigned > PGOVerifyBFIRatio("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))
static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static cl::opt< unsigned > PGOFunctionCriticalEdgeThreshold("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))
static void setupBBInfoEdges(const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo)
Set up InEdges/OutEdges for all BBs in the MST.
static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
static bool skipPGOUse(const Function &F)
static bool canRenameComdat(Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< bool > PGOVerifyBFI("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static cl::opt< bool > PGOBlockCoverage("pgo-block-coverage", cl::desc("Use this option to enable basic block coverage instrumentation"))
static uint64_t sumEdgeCount(const ArrayRef< PGOUseEdge * > Edges)
static cl::opt< bool > PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
static uint32_t getMaxNumAnnotations(InstrProfValueKind ValueProfKind)
Function::ProfileCount ProfileCount
static cl::opt< bool > EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
static cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static std::string getSimpleNodeName(const BasicBlock *Node)
static cl::opt< bool > PGOViewBlockCoverageGraph("pgo-view-block-coverage-graph", cl::desc("Create a dot file of CFGs with block " "coverage inference information"))
static bool isIndirectBrTarget(BasicBlock *BB)
static std::string getBranchCondString(Instruction *TI)
static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, ProfileSummaryInfo *PSI, bool IsCS)
static cl::opt< PGOViewCountsType > PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static const char * ValueProfKindDescr[]
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Defines the virtual file system interface vfs::FileSystem.
void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const
Returns the estimated profile count of BB.
Conditional or Unconditional Branch instruction.
bool isConditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
An union-find based Minimum Spanning Tree for CFG.
Edge & addEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W)
const std::vector< std::unique_ptr< Edge > > & allEdges() const
size_t bbInfoSize() const
BBInfo * findBBInfo(const BasicBlock *BB) const
BBInfo & getBBInfo(const BasicBlock *BB) const
void dumpEdges(raw_ostream &OS, const Twine &Message) const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Predicate getPredicate() const
Return the predicate for this instruction.
StringRef getName() const
void setSelectionKind(SelectionKind Val)
SelectionKind getSelectionKind() const
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Base class for error info classes.
virtual std::string message() const
Return the error message as a string.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
Class to represent profile counts.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
@ HiddenVisibility
The GV is hidden.
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
This instruction compares its operands according to the predicate given to the constructor.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Reader for the indexed binary instrprof format.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
uint64_t getMaximumFunctionCount(bool UseCS)
Return the maximum of all known function counts.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
RetTy visitSelectInst(SelectInst &I)
instrprof_error get() const
std::string message() const override
Return the error message as a string.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
void update(ArrayRef< uint8_t > Data)
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
MDString * createString(StringRef Str)
Return the given string as metadata.
MDNode * createIrrLoopHeaderWeight(uint64_t Weight)
Return metadata containing an irreducible loop header weight.
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PGOInstrumentationUse(std::string Filename="", std::string RemappingFilename="", bool IsCS=false, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
uint64_t getOrCompColdCountThreshold() const
Returns ColdCountThreshold if set.
void refresh()
If no summary is present, attempt to refresh.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
uint64_t getOrCompHotCountThreshold() const
Returns HotCountThreshold if set.
This class represents the LLVM 'select' instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const
Print the current type.
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Utility analysis that determines what values are worth profiling.
std::vector< CandidateInfo > get(InstrProfValueKind Kind) const
returns a list of value profiling candidates of the given kind
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
A raw_ostream that writes to an std::string.
The virtual file system interface.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
This file contains the declaration of the Comdat class, which represents a single COMDAT in LLVM.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
uint64_t getFuncHash(const FuncRecordTy *Record)
Return the structural hash associated with the function.
void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< FuncNode * > Func
void write64le(void *P, uint64_t V)
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Function::ProfileCount ProfileCount
auto successors(const MachineBasicBlock *BB)
void createProfileSamplingVar(Module &M)
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
DenseMap< BasicBlock *, ColorVector > colorEHFunclets(Function &F)
If an EH funclet personality is in use (see isFuncletEHPersonality), this will recompute which blocks...
void createPGONameMetadata(GlobalObject &GO, StringRef PGOName)
Create the PGOName metadata if a global object's PGO name is different from its mangled name.
cl::opt< bool > PGOWarnMissing
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr)
cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
cl::opt< bool > DebugInfoCorrelate
OperandBundleDefT< Value * > OperandBundleDef
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
auto reverse(ContainerTy &&C)
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
cl::opt< bool > NoPGOWarnMismatch
RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)
cl::opt< PGOViewCountsType > PGOViewCounts("pgo-view-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text with " "block profile counts and branch probabilities " "right after PGO profile annotation step. The " "profile counts are computed using branch " "probabilities from the runtime profile data and " "block frequency propagation algorithm. To view " "the raw counts from the profile, use option " "-pgo-view-raw-counts instead. To limit graph " "display to only one function, use filtering option " "-view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)
Return true if the specified edge is a critical edge.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
auto predecessors(const MachineBasicBlock *BB)
void setProfMetadata(Module *M, Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
SuccIterator< const Instruction, const BasicBlock > const_succ_iterator
cl::opt< bool > NoPGOWarnMismatchComdatWeak
Implement std::hash so that hash_code can be used in STL containers.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
DOTGraphTraits(bool isSimple=false)
static std::string getGraphName(const PGOUseFunc *G)
std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph)
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
static ChildIteratorType child_end(const NodeRef N)
static NodeRef getEntryNode(const PGOUseFunc *G)
static ChildIteratorType child_begin(const NodeRef N)
static nodes_iterator nodes_end(const PGOUseFunc *G)
static nodes_iterator nodes_begin(const PGOUseFunc *G)
Profiling information for a single function.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
static void setCSFlagInHash(uint64_t &FuncHash)
Instruction * AnnotatedInst