123#include <unordered_map>
131#define DEBUG_TYPE "pgo-instrumentation"
133STATISTIC(NumOfPGOInstrument,
"Number of edges instrumented.");
134STATISTIC(NumOfPGOSelectInsts,
"Number of select instruction instrumented.");
135STATISTIC(NumOfPGOMemIntrinsics,
"Number of mem intrinsics instrumented.");
138STATISTIC(NumOfPGOSplit,
"Number of critical edge splits.");
139STATISTIC(NumOfPGOFunc,
"Number of functions having valid profile counts.");
140STATISTIC(NumOfPGOMismatch,
"Number of functions having mismatch profile.");
141STATISTIC(NumOfPGOMissing,
"Number of functions without profile.");
142STATISTIC(NumOfPGOICall,
"Number of indirect call value instrumentations.");
143STATISTIC(NumOfCSPGOInstrument,
"Number of edges instrumented in CSPGO.");
145 "Number of select instruction instrumented in CSPGO.");
147 "Number of mem intrinsics instrumented in CSPGO.");
149STATISTIC(NumOfCSPGOBB,
"Number of basic-blocks in CSPGO.");
150STATISTIC(NumOfCSPGOSplit,
"Number of critical edge splits in CSPGO.");
152 "Number of functions having valid profile counts in CSPGO.");
154 "Number of functions having mismatch profile in CSPGO.");
155STATISTIC(NumOfCSPGOMissing,
"Number of functions without profile in CSPGO.");
156STATISTIC(NumCoveredBlocks,
"Number of basic blocks that were executed");
163 cl::desc(
"Specify the path of profile data file. This is"
164 "mainly for test purpose."));
168 cl::desc(
"Specify the path of profile remapping file. This is mainly for "
175 cl::desc(
"Disable Value Profiling"));
181 cl::desc(
"Max number of annotations for a single indirect "
188 cl::desc(
"Max number of preicise value annotations for a single memop"
195 cl::desc(
"Append function hash to the name of COMDAT function to avoid "
196 "function hash mismatch due to the preinliner"));
203 cl::desc(
"Use this option to turn on/off "
204 "warnings about missing profile data for "
211 cl::desc(
"Use this option to turn off/on "
212 "warnings about profile cfg mismatch."));
219 cl::desc(
"The option is used to turn on/off "
220 "warnings about hash mismatch for comdat "
221 "or weak functions."));
227 cl::desc(
"Use this option to turn on/off SELECT "
228 "instruction instrumentation. "));
233 cl::desc(
"A boolean option to show CFG dag or text "
234 "with raw profile counts from "
235 "profile data. See also option "
236 "-pgo-view-counts. To limit graph "
237 "display to only one function, use "
238 "filtering option -view-bfi-func-name."),
246 cl::desc(
"Use this option to turn on/off "
247 "memory intrinsic size profiling."));
252 cl::desc(
"When this option is on, the annotated "
253 "branch probability will be emitted as "
254 "optimization remarks: -{Rpass|"
255 "pass-remarks}=pgo-instrumentation"));
259 cl::desc(
"Force to instrument function entry basicblock."));
264 "Use this option to enable function entry coverage instrumentation."));
267 "pgo-block-coverage",
268 cl::desc(
"Use this option to enable basic block coverage instrumentation"));
272 cl::desc(
"Create a dot file of CFGs with block "
273 "coverage inference information"));
276 "pgo-temporal-instrumentation",
277 cl::desc(
"Use this option to enable temporal instrumentation"));
281 cl::desc(
"Fix function entry count in profile use."));
285 cl::desc(
"Print out the non-match BFI count if a hot raw profile count "
286 "becomes non-hot, or a cold raw profile count becomes hot. "
287 "The print is enabled under -Rpass-analysis=pgo, or "
288 "internal option -pass-remakrs-analysis=pgo."));
292 cl::desc(
"Print out mismatched BFI counts after setting profile metadata "
293 "The print is enabled under -Rpass-analysis=pgo, or "
294 "internal option -pass-remakrs-analysis=pgo."));
298 cl::desc(
"Set the threshold for pgo-verify-bfi: only print out "
299 "mismatched BFI if the difference percentage is greater than "
300 "this value (in percentage)."));
304 cl::desc(
"Set the threshold for pgo-verify-bfi: skip the counts whose "
305 "profile count value is below."));
310 cl::desc(
"Trace the hash of the function with this name."));
314 cl::desc(
"Do not instrument functions smaller than this threshold."));
318 cl::desc(
"Do not instrument functions with the number of critical edges "
319 " greater than this threshold."));
341 return std::string();
346 return std::string();
358 else if (CV->
isOne())
370#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,
377 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
379 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
381 ProfileVersion |= VARIANT_MASK_CSIR_PROF;
383 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
385 ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
388 VARIANT_MASK_BYTE_COVERAGE | VARIANT_MASK_FUNCTION_ENTRY_ONLY;
390 ProfileVersion |= VARIANT_MASK_BYTE_COVERAGE;
392 ProfileVersion |= VARIANT_MASK_TEMPORAL_PROF;
397 Triple TT(M.getTargetTriple());
398 if (TT.supportsCOMDAT()) {
400 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
402 return IRLevelVersionVariable;
412enum VisitMode { VM_counting, VM_instrument, VM_annotate };
416struct SelectInstVisitor :
public InstVisitor<SelectInstVisitor> {
419 VisitMode
Mode = VM_counting;
420 unsigned *CurCtrIdx =
nullptr;
421 unsigned TotalNumCtrs = 0;
424 PGOUseFunc *UseFunc =
nullptr;
425 bool HasSingleByteCoverage;
427 SelectInstVisitor(
Function &Func,
bool HasSingleByteCoverage)
428 :
F(
Func), HasSingleByteCoverage(HasSingleByteCoverage) {}
430 void countSelects() {
440 void instrumentSelects(
unsigned *Ind,
unsigned TotalNC,
GlobalVariable *FNV,
442 Mode = VM_instrument;
444 TotalNumCtrs = TotalNC;
451 void annotateSelects(PGOUseFunc *UF,
unsigned *Ind) {
466 unsigned getNumOfSelectInsts()
const {
return NSIs; }
478 bool Removed =
false;
479 bool IsCritical =
false;
482 : SrcBB(Src), DestBB(Dest), Weight(
W) {}
485 std::string infoString()
const {
486 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
487 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight))
498 PGOBBInfo(
unsigned IX) : Group(this),
Index(IX) {}
501 std::string infoString()
const {
507template <
class Edge,
class BBInfo>
class FuncPGOInstrumentation {
515 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
519 void computeCFGHash();
520 void renameComdatFunction();
524 std::vector<std::vector<VPCandidateInfo>> ValueSites;
525 SelectInstVisitor SIVisitor;
526 std::string FuncName;
527 std::string DeprecatedFuncName;
536 const std::optional<BlockCoverageInference> BCI;
538 static std::optional<BlockCoverageInference>
539 constructBCI(
Function &Func,
bool HasSingleByteCoverage,
540 bool InstrumentFuncEntry) {
541 if (HasSingleByteCoverage)
548 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);
561 void dumpInfo(
StringRef Str =
"")
const {
563 " Hash: " +
Twine(FunctionHash) +
"\t" + Str);
566 FuncPGOInstrumentation(
568 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
571 bool InstrumentFuncEntry =
true,
bool HasSingleByteCoverage =
false)
572 :
F(
Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(
Func, TLI),
573 TLI(TLI), ValueSites(IPVK_Last + 1),
574 SIVisitor(
Func, HasSingleByteCoverage),
575 MST(
F, InstrumentFuncEntry, BPI,
BFI),
576 BCI(constructBCI(
Func, HasSingleByteCoverage, InstrumentFuncEntry)) {
578 BCI->viewBlockCoverageGraph();
580 SIVisitor.countSelects();
581 ValueSites[IPVK_MemOPSize] = VPC.
get(IPVK_MemOPSize);
583 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
584 NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
586 ValueSites[IPVK_IndirectCallTarget] = VPC.
get(IPVK_IndirectCallTarget);
588 ValueSites[IPVK_VTableTarget] = VPC.
get(IPVK_VTableTarget);
590 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
591 NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
598 if (!ComdatMembers.empty())
599 renameComdatFunction();
602 for (
const auto &E : MST.
allEdges()) {
605 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
607 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
620template <
class Edge,
class BBInfo>
621void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
622 std::vector<uint8_t> Indexes;
626 auto BI = findBBInfo(Succ);
630 for (
int J = 0; J < 4; J++)
631 Indexes.push_back((uint8_t)(
Index >> (J * 8)));
638 auto updateJCH = [&JCH](
uint64_t Num) {
643 updateJCH((
uint64_t)SIVisitor.getNumOfSelectInsts());
644 updateJCH((
uint64_t)ValueSites[IPVK_IndirectCallTarget].
size());
647 updateJCH(BCI->getInstrumentedBlocksHash());
657 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
660 LLVM_DEBUG(
dbgs() <<
"Function Hash Computation for " <<
F.getName() <<
":\n"
661 <<
" CRC = " << JC.
getCRC()
662 <<
", Selects = " << SIVisitor.getNumOfSelectInsts()
663 <<
", Edges = " << MST.
numEdges() <<
", ICSites = "
664 << ValueSites[IPVK_IndirectCallTarget].size()
665 <<
", Memops = " << ValueSites[IPVK_MemOPSize].size()
666 <<
", High32 CRC = " << JCH.
getCRC()
667 <<
", Hash = " << FunctionHash <<
"\n";);
670 dbgs() <<
"Funcname=" <<
F.getName() <<
", Hash=" << FunctionHash
671 <<
" in building " <<
F.getParent()->getSourceFileName() <<
"\n";
677 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
689 for (
auto &&CM :
make_range(ComdatMembers.equal_range(
C))) {
690 assert(!isa<GlobalAlias>(CM.second));
691 Function *FM = dyn_cast<Function>(CM.second);
699template <
class Edge,
class BBInfo>
700void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
703 std::string OrigName =
F.getName().str();
704 std::string NewFuncName =
706 F.setName(
Twine(NewFuncName));
708 FuncName =
Twine(FuncName +
"." +
Twine(FunctionHash)).
str();
714 if (!
F.hasComdat()) {
716 NewComdat =
M->getOrInsertComdat(
StringRef(NewFuncName));
718 F.setComdat(NewComdat);
723 Comdat *OrigComdat =
F.getComdat();
724 std::string NewComdatName =
726 NewComdat =
M->getOrInsertComdat(
StringRef(NewComdatName));
729 for (
auto &&CM :
make_range(ComdatMembers.equal_range(OrigComdat))) {
731 cast<Function>(CM.second)->setComdat(NewComdat);
737template <
class Edge,
class BBInfo>
738void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
739 std::vector<BasicBlock *> &InstrumentBBs) {
742 if (BCI->shouldInstrumentBlock(BB))
743 InstrumentBBs.push_back(&BB);
748 std::vector<Edge *> EdgeList;
750 for (
const auto &E : MST.
allEdges())
751 EdgeList.push_back(E.get());
753 for (
auto &E : EdgeList) {
756 InstrumentBBs.push_back(InstrBB);
762template <
class Edge,
class BBInfo>
763BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
764 if (E->InMST || E->Removed)
770 if (SrcBB ==
nullptr)
772 if (DestBB ==
nullptr)
787 return canInstrument(SrcBB);
789 return canInstrument(DestBB);
798 dbgs() <<
"Fail to split critical edge: not instrument this edge.\n");
803 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
805 <<
" --> " << getBBInfo(DestBB).
Index <<
"\n");
807 MST.
addEdge(SrcBB, InstrBB, 0);
809 Edge &NewEdge1 = MST.
addEdge(InstrBB, DestBB, 0);
810 NewEdge1.InMST =
true;
813 return canInstrument(InstrBB);
829 if (!isa<IntrinsicInst>(OrigCall)) {
832 std::optional<OperandBundleUse> ParentFunclet =
840 if (!BlockColors.
empty()) {
841 const ColorVector &CV = BlockColors.
find(OrigCall->getParent())->second;
842 assert(CV.
size() == 1 &&
"non-unique color for block!");
855 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
863 FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
867 auto Name = FuncInfo.FuncNameVar;
869 FuncInfo.FunctionHash);
871 auto &EntryBB =
F.getEntryBlock();
872 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
881 std::vector<BasicBlock *> InstrumentBBs;
882 FuncInfo.getInstrumentBBs(InstrumentBBs);
883 unsigned NumCounters =
884 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
889 auto &EntryBB =
F.getEntryBlock();
890 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
899 for (
auto *InstrBB : InstrumentBBs) {
902 "Cannot get the Instrumentation point");
907 ? Intrinsic::instrprof_cover
908 : Intrinsic::instrprof_increment),
913 FuncInfo.SIVisitor.instrumentSelects(&
I, NumCounters, FuncInfo.FuncNameVar,
914 FuncInfo.FunctionHash);
920 NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();
927 if (
F.hasPersonalityFn() &&
932 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
933 unsigned SiteIndex = 0;
939 <<
" site: CallSite Index = " << SiteIndex <<
"\n");
943 "Cannot get the Instrumentation point");
945 Value *ToProfile =
nullptr;
946 if (Cand.V->getType()->isIntegerTy())
948 else if (Cand.V->getType()->isPointerTy())
950 assert(ToProfile &&
"value profiling Value is of unexpected type");
956 {FuncInfo.FuncNameVar, Builder.
getInt64(FuncInfo.FunctionHash),
966struct PGOUseEdge :
public PGOEdge {
967 using PGOEdge::PGOEdge;
969 std::optional<uint64_t> Count;
975 std::string infoString()
const {
977 return PGOEdge::infoString();
978 return (
Twine(PGOEdge::infoString()) +
" Count=" +
Twine(*Count)).str();
985struct PGOUseBBInfo :
public PGOBBInfo {
986 std::optional<uint64_t> Count;
987 int32_t UnknownCountInEdge = 0;
988 int32_t UnknownCountOutEdge = 0;
990 DirectEdges OutEdges;
992 PGOUseBBInfo(
unsigned IX) : PGOBBInfo(IX) {}
998 std::string infoString()
const {
1000 return PGOBBInfo::infoString();
1001 return (
Twine(PGOBBInfo::infoString()) +
" Count=" +
Twine(*Count)).str();
1005 void addOutEdge(PGOUseEdge *E) {
1006 OutEdges.push_back(E);
1007 UnknownCountOutEdge++;
1011 void addInEdge(PGOUseEdge *E) {
1012 InEdges.push_back(E);
1013 UnknownCountInEdge++;
1022 for (
const auto &E : Edges) {
1036 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
1039 bool HasSingleByteCoverage)
1040 :
F(
Func),
M(Modu),
BFI(BFIin), PSI(PSI),
1041 FuncInfo(
Func, TLI, ComdatMembers,
false, BPI, BFIin, IsCS,
1042 InstrumentFuncEntry, HasSingleByteCoverage),
1043 FreqAttr(FFA_Normal), IsCS(IsCS) {}
1045 void handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum);
1052 void populateCounters();
1061 void annotateValueSites();
1064 void annotateValueSites(
uint32_t Kind);
1067 void annotateIrrLoopHeaderWeights();
1070 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
1073 FuncFreqAttr getFuncFreqAttr()
const {
return FreqAttr; }
1082 PGOUseBBInfo &getBBInfo(
const BasicBlock *BB)
const {
1083 return FuncInfo.getBBInfo(BB);
1087 PGOUseBBInfo *findBBInfo(
const BasicBlock *BB)
const {
1088 return FuncInfo.findBBInfo(BB);
1093 void dumpInfo(
StringRef Str =
"")
const { FuncInfo.dumpInfo(Str); }
1095 uint64_t getProgramMaxCount()
const {
return ProgramMaxCount; }
1104 FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> FuncInfo;
1120 FuncFreqAttr FreqAttr;
1126 bool setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile);
1139 FreqAttr = FFA_Cold;
1147 const FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> &FuncInfo) {
1151 for (
const auto &E : FuncInfo.MST.allEdges()) {
1156 PGOUseBBInfo &SrcInfo = FuncInfo.getBBInfo(SrcBB);
1157 PGOUseBBInfo &DestInfo = FuncInfo.getBBInfo(DestBB);
1158 SrcInfo.addOutEdge(E.get());
1159 DestInfo.addInEdge(E.get());
1165bool PGOUseFunc::setInstrumentedCounts(
1166 const std::vector<uint64_t> &CountFromProfile) {
1168 std::vector<BasicBlock *> InstrumentBBs;
1169 FuncInfo.getInstrumentBBs(InstrumentBBs);
1173 unsigned NumCounters =
1174 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
1177 if (NumCounters != CountFromProfile.size()) {
1180 auto *FuncEntry = &*
F.begin();
1185 uint64_t CountValue = CountFromProfile[
I++];
1186 PGOUseBBInfo &
Info = getBBInfo(InstrBB);
1190 if (InstrBB == FuncEntry && CountValue == 0)
1192 Info.setBBInfoCount(CountValue);
1194 ProfileCountSize = CountFromProfile.size();
1198 auto setEdgeCount = [
this](PGOUseEdge *E,
uint64_t Value) ->
void {
1199 E->setEdgeCount(
Value);
1200 this->getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1201 this->getBBInfo(E->DestBB).UnknownCountInEdge--;
1207 for (
const auto &E : FuncInfo.MST.allEdges()) {
1208 if (E->Removed || E->InMST)
1211 PGOUseBBInfo &SrcInfo = getBBInfo(SrcBB);
1215 if (SrcInfo.Count && SrcInfo.OutEdges.size() == 1)
1216 setEdgeCount(E.get(), *SrcInfo.Count);
1219 PGOUseBBInfo &DestInfo = getBBInfo(DestBB);
1222 if (DestInfo.Count && DestInfo.InEdges.size() == 1)
1223 setEdgeCount(E.get(), *DestInfo.Count);
1229 setEdgeCount(E.get(), 0);
1236void PGOUseFunc::setEdgeCount(DirectEdges &Edges,
uint64_t Value) {
1237 for (
auto &E : Edges) {
1240 E->setEdgeCount(
Value);
1242 getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1243 getBBInfo(E->DestBB).UnknownCountInEdge--;
1251 const char MetadataName[] =
"instr_prof_hash_mismatch";
1254 auto *Existing =
F.getMetadata(LLVMContext::MD_annotation);
1256 MDTuple *Tuple = cast<MDTuple>(Existing);
1257 for (
const auto &
N : Tuple->
operands()) {
1258 if (
N.equalsStr(MetadataName))
1267 F.setMetadata(LLVMContext::MD_annotation, MD);
1270void PGOUseFunc::handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum) {
1272 auto &Ctx =
M->getContext();
1273 auto Err = IPE.
get();
1274 bool SkipWarning =
false;
1276 << FuncInfo.FuncName <<
": ");
1277 if (Err == instrprof_error::unknown_function) {
1278 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
1281 }
else if (Err == instrprof_error::hash_mismatch ||
1282 Err == instrprof_error::malformed) {
1283 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
1289 LLVM_DEBUG(
dbgs() <<
"hash mismatch (hash= " << FuncInfo.FunctionHash
1290 <<
" skip=" << SkipWarning <<
")");
1300 IPE.
message() + std::string(
" ") +
F.getName().str() +
1301 std::string(
" Hash = ") + std::to_string(FuncInfo.FunctionHash) +
1302 std::string(
" up to ") + std::to_string(MismatchedFuncSum) +
1303 std::string(
" count discarded");
1315 auto &Ctx =
M->getContext();
1318 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1319 &MismatchedFuncSum);
1321 handleInstrProfError(std::move(E), MismatchedFuncSum);
1324 ProfileRecord = std::move(
Result.get());
1329 std::vector<uint64_t> &CountFromProfile = ProfileRecord.
Counts;
1331 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1335 for (
unsigned I = 0, S = CountFromProfile.size();
I < S;
I++) {
1337 ValueSum += CountFromProfile[
I];
1339 AllZeros = (ValueSum == 0);
1343 getBBInfo(
nullptr).UnknownCountOutEdge = 2;
1344 getBBInfo(
nullptr).UnknownCountInEdge = 2;
1346 if (!setInstrumentedCounts(CountFromProfile)) {
1348 dbgs() <<
"Inconsistent number of counts, skipping this function");
1350 M->getName().data(),
1351 Twine(
"Inconsistent number of counts in ") +
F.getName().str() +
1352 Twine(
": the profile may be stale or there is a function name "
1364 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1365 &MismatchedFuncSum);
1366 if (
auto Err =
Result.takeError()) {
1367 handleInstrProfError(std::move(Err), MismatchedFuncSum);
1370 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1372 std::vector<uint64_t> &CountsFromProfile =
Result.get().Counts;
1376 if (FuncInfo.BCI->shouldInstrumentBlock(BB))
1382 InverseDependencies;
1383 for (
auto &BB :
F) {
1384 for (
auto *Dep : FuncInfo.BCI->getDependencies(BB)) {
1386 InverseDependencies[Dep].
insert(&BB);
1391 std::stack<const BasicBlock *> CoveredBlocksToProcess;
1392 for (
auto &[BB, IsCovered] : Coverage)
1394 CoveredBlocksToProcess.push(BB);
1396 while (!CoveredBlocksToProcess.empty()) {
1397 auto *CoveredBlock = CoveredBlocksToProcess.top();
1398 assert(Coverage[CoveredBlock]);
1399 CoveredBlocksToProcess.pop();
1400 for (
auto *BB : InverseDependencies[CoveredBlock]) {
1405 CoveredBlocksToProcess.push(BB);
1413 F.setEntryCount(Coverage[&
F.getEntryBlock()] ? 10000 : 0);
1414 for (
auto &BB :
F) {
1423 Weights.
push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);
1424 if (Weights.
size() >= 2)
1428 unsigned NumCorruptCoverage = 0;
1433 auto IsBlockDead = [&](
const BasicBlock &BB) -> std::optional<bool> {
1434 if (
auto C =
BFI.getBlockProfileCount(&BB))
1438 LLVM_DEBUG(
dbgs() <<
"Block Coverage: (Instrumented=*, Covered=X)\n");
1439 for (
auto &BB :
F) {
1440 LLVM_DEBUG(
dbgs() << (FuncInfo.BCI->shouldInstrumentBlock(BB) ?
"* " :
" ")
1441 << (Coverage[&BB] ?
"X " :
" ") <<
" " << BB.getName()
1447 if (Coverage[&BB] == IsBlockDead(BB).value_or(
false)) {
1449 dbgs() <<
"Found inconsistent block covearge for " << BB.getName()
1450 <<
": BCI=" << (Coverage[&BB] ?
"Covered" :
"Dead") <<
" BFI="
1451 << (IsBlockDead(BB).
value() ?
"Dead" :
"Covered") <<
"\n");
1452 ++NumCorruptCoverage;
1458 auto &Ctx =
M->getContext();
1460 M->getName().data(),
1461 Twine(
"Found inconsistent block coverage for function ") +
F.getName() +
1462 " in " +
Twine(NumCorruptCoverage) +
" blocks.",
1466 FuncInfo.BCI->viewBlockCoverageGraph(&Coverage);
1471void PGOUseFunc::populateCounters() {
1472 bool Changes =
true;
1473 unsigned NumPasses = 0;
1481 PGOUseBBInfo *UseBBInfo = findBBInfo(&BB);
1482 if (UseBBInfo ==
nullptr)
1484 if (!UseBBInfo->Count) {
1485 if (UseBBInfo->UnknownCountOutEdge == 0) {
1488 }
else if (UseBBInfo->UnknownCountInEdge == 0) {
1493 if (UseBBInfo->Count) {
1494 if (UseBBInfo->UnknownCountOutEdge == 1) {
1500 if (*UseBBInfo->Count > OutSum)
1501 Total = *UseBBInfo->Count - OutSum;
1502 setEdgeCount(UseBBInfo->OutEdges,
Total);
1505 if (UseBBInfo->UnknownCountInEdge == 1) {
1508 if (*UseBBInfo->Count > InSum)
1509 Total = *UseBBInfo->Count - InSum;
1510 setEdgeCount(UseBBInfo->InEdges,
Total);
1517 LLVM_DEBUG(
dbgs() <<
"Populate counts in " << NumPasses <<
" passes.\n");
1521 for (
auto &BB :
F) {
1522 auto BI = findBBInfo(&BB);
1525 assert(BI->Count &&
"BB count is not valid");
1530 for (
auto &BB :
F) {
1531 auto BI = findBBInfo(&BB);
1534 FuncMaxCount = std::max(FuncMaxCount, *BI->Count);
1544 FuncInfo.SIVisitor.annotateSelects(
this, &CountPosition);
1545 assert(CountPosition == ProfileCountSize);
1547 LLVM_DEBUG(FuncInfo.dumpInfo(
"after reading profile."));
1551void PGOUseFunc::setBranchWeights() {
1553 LLVM_DEBUG(
dbgs() <<
"\nSetting branch weights for func " <<
F.getName()
1554 <<
" IsCS=" << IsCS <<
"\n");
1555 for (
auto &BB :
F) {
1559 if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
1560 isa<IndirectBrInst>(TI) || isa<InvokeInst>(TI) ||
1561 isa<CallBrInst>(TI)))
1564 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1565 if (!*BBCountInfo.Count)
1569 unsigned Size = BBCountInfo.OutEdges.size();
1572 for (
unsigned s = 0; s <
Size; s++) {
1573 const PGOUseEdge *E = BBCountInfo.OutEdges[s];
1576 if (DestBB ==
nullptr)
1580 if (EdgeCount > MaxCount)
1581 MaxCount = EdgeCount;
1582 EdgeCounts[SuccNum] = EdgeCount;
1591 auto &Ctx =
M->getContext();
1593 M->getName().data(),
1594 Twine(
"Profile in ") +
F.getName().str() +
1595 Twine(
" partially ignored") +
1596 Twine(
", possibly due to the lack of a return path."),
1604 if (isa<IndirectBrInst>(Pred->getTerminator()))
1610void PGOUseFunc::annotateIrrLoopHeaderWeights() {
1611 LLVM_DEBUG(
dbgs() <<
"\nAnnotating irreducible loop header weights.\n");
1613 for (
auto &BB :
F) {
1619 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1625void SelectInstVisitor::instrumentOneSelectInst(
SelectInst &SI) {
1629 auto *Step = Builder.CreateZExt(
SI.getCondition(), Int64Ty);
1632 {FuncNameVar, Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs),
1633 Builder.getInt32(*CurCtrIdx), Step});
1637void SelectInstVisitor::annotateOneSelectInst(
SelectInst &SI) {
1638 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1639 assert(*CurCtrIdx < CountFromProfile.size() &&
1640 "Out of bound access of counters");
1642 SCounts[0] = CountFromProfile[*CurCtrIdx];
1645 auto BI = UseFunc->findBBInfo(
SI.getParent());
1647 TotalCount = *BI->Count;
1649 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1650 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1655void SelectInstVisitor::visitSelectInst(
SelectInst &SI) {
1659 if (
SI.getCondition()->getType()->isVectorTy())
1667 instrumentOneSelectInst(SI);
1670 annotateOneSelectInst(SI);
1678void PGOUseFunc::annotateValueSites() {
1686 annotateValueSites(Kind);
1690void PGOUseFunc::annotateValueSites(
uint32_t Kind) {
1691 assert(Kind <= IPVK_Last);
1692 unsigned ValueSiteIndex = 0;
1693 auto &ValueSites = FuncInfo.ValueSites[
Kind];
1695 if (NumValueSites != ValueSites.size()) {
1696 auto &Ctx =
M->getContext();
1698 M->getName().data(),
1699 Twine(
"Inconsistent number of value sites for ") +
1702 Twine(
"\", possibly due to the use of a stale profile."),
1708 LLVM_DEBUG(
dbgs() <<
"Read one value site profile (kind = " << Kind
1709 <<
"): Index = " << ValueSiteIndex <<
" out of "
1710 << NumValueSites <<
"\n");
1723 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1728 ComdatMembers.insert(std::make_pair(
C, &
F));
1730 if (
Comdat *
C = GV.getComdat())
1731 ComdatMembers.insert(std::make_pair(
C, &GV));
1733 if (
Comdat *
C = GA.getComdat())
1734 ComdatMembers.insert(std::make_pair(
C, &GA));
1739 if (
F.isDeclaration())
1744 unsigned NumCriticalEdges = 0;
1745 for (
auto &BB :
F) {
1754 <<
", NumCriticalEdges=" << NumCriticalEdges
1755 <<
" exceed the threshold. Skip PGO.\n");
1765 if (
F.hasFnAttribute(llvm::Attribute::Naked))
1767 if (
F.hasFnAttribute(llvm::Attribute::NoProfile))
1769 if (
F.hasFnAttribute(llvm::Attribute::SkipProfile))
1785 Triple TT(M.getTargetTriple());
1790 Twine(
"VTable value profiling is presently not "
1791 "supported for non-ELF object formats"),
1793 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1799 auto &TLI = LookupTLI(
F);
1800 auto *BPI = LookupBPI(
F);
1801 auto *BFI = LookupBFI(
F);
1845 auto BFIEntryCount =
F.getEntryCount();
1846 assert(BFIEntryCount && (BFIEntryCount->getCount() > 0) &&
1847 "Invalid BFI Entrycount");
1851 for (
auto &BBI :
F) {
1854 if (!Func.findBBInfo(&BBI))
1857 CountValue = *Func.getBBInfo(&BBI).Count;
1858 BFICountValue = *BFICount;
1862 if (SumCount.isZero())
1866 "Incorrect sum of BFI counts");
1869 double Scale = (SumCount / SumBFICount).convertToDouble();
1870 if (Scale < 1.001 && Scale > 0.999)
1875 if (NewEntryCount == 0)
1881 << NewEntryCount <<
"\n");
1898 unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
1899 for (
auto &BBI :
F) {
1903 CountValue = Func.getBBInfo(&BBI).Count.value_or(CountValue);
1910 BFICountValue = *BFICount;
1913 bool rawIsHot = CountValue >= HotCountThreshold;
1914 bool BFIIsHot = BFICountValue >= HotCountThreshold;
1916 bool ShowCount =
false;
1917 if (rawIsHot && !BFIIsHot) {
1918 Msg =
"raw-Hot to BFI-nonHot";
1920 }
else if (rawIsCold && BFIIsHot) {
1921 Msg =
"raw-Cold to BFI-Hot";
1930 uint64_t Diff = (BFICountValue >= CountValue)
1931 ? BFICountValue - CountValue
1932 : CountValue - BFICountValue;
1940 F.getSubprogram(), &BBI);
1942 <<
" Count=" <<
ore::NV(
"Count", CountValue)
1943 <<
" BFI_Count=" <<
ore::NV(
"Count", BFICountValue);
1945 Remark <<
" (" << Msg <<
")";
1952 F.getSubprogram(), &
F.getEntryBlock())
1953 <<
"In Func " <<
ore::NV(
"Function",
F.getName())
1954 <<
": Num_of_BB=" <<
ore::NV(
"Count", BBNum)
1955 <<
", Num_of_non_zerovalue_BB=" <<
ore::NV(
"Count", NonZeroBBNum)
1956 <<
", Num_of_mis_matching_BB=" <<
ore::NV(
"Count", BBMisMatchNum);
1968 auto &Ctx = M.getContext();
1971 ProfileRemappingFileName);
1972 if (
Error E = ReaderOrErr.takeError()) {
1980 std::unique_ptr<IndexedInstrProfReader> PGOReader =
1981 std::move(ReaderOrErr.get());
1987 if (!PGOReader->hasCSIRLevelProfile() && IsCS)
1991 if (!PGOReader->isIRLevelProfile()) {
1993 ProfileFileName.
data(),
"Not an IR level instrumentation profile"));
1996 if (PGOReader->functionEntryOnly()) {
1998 ProfileFileName.
data(),
1999 "Function entry profiles are not yet supported for optimization"));
2006 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
2011 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
2013 std::vector<Function *> HotFunctions;
2014 std::vector<Function *> ColdFunctions;
2018 bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
2021 bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
2025 auto &TLI = LookupTLI(
F);
2026 auto *BPI = LookupBPI(
F);
2027 auto *BFI = LookupBFI(
F);
2028 if (!HasSingleByteCoverage) {
2034 PGOUseFunc Func(
F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
2035 InstrumentFuncEntry, HasSingleByteCoverage);
2036 if (HasSingleByteCoverage) {
2037 Func.populateCoverage(PGOReader.get());
2045 bool AllZeros =
false;
2046 if (!Func.readCounters(PGOReader.get(), AllZeros, PseudoKind))
2050 if (Func.getProgramMaxCount() != 0)
2051 ColdFunctions.push_back(&
F);
2056 if (
F.hasFnAttribute(Attribute::Cold))
2057 F.removeFnAttr(Attribute::Cold);
2060 F.addFnAttr(Attribute::Hot);
2063 Func.populateCounters();
2064 Func.setBranchWeights();
2065 Func.annotateValueSites();
2066 Func.annotateIrrLoopHeaderWeights();
2067 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
2068 if (FreqAttr == PGOUseFunc::FFA_Cold)
2069 ColdFunctions.push_back(&
F);
2070 else if (FreqAttr == PGOUseFunc::FFA_Hot)
2071 HotFunctions.push_back(&
F);
2076 std::unique_ptr<BranchProbabilityInfo> NewBPI =
2077 std::make_unique<BranchProbabilityInfo>(
F, LI);
2078 std::unique_ptr<BlockFrequencyInfo> NewBFI =
2079 std::make_unique<BlockFrequencyInfo>(
F, *NewBPI, LI);
2083 dbgs() <<
"pgo-view-counts: " << Func.getFunc().getName() <<
"\n";
2084 NewBFI->print(
dbgs());
2094 ViewGraph(&Func,
Twine(
"PGORawCounts_") + Func.getFunc().getName());
2096 dbgs() <<
"pgo-view-raw-counts: " << Func.getFunc().getName() <<
"\n";
2123 for (
auto &
F : HotFunctions) {
2124 F->addFnAttr(Attribute::InlineHint);
2125 LLVM_DEBUG(
dbgs() <<
"Set inline attribute to function: " <<
F->getName()
2128 for (
auto &
F : ColdFunctions) {
2131 if (
F->hasFnAttribute(Attribute::Hot)) {
2132 auto &Ctx = M.getContext();
2133 std::string Msg = std::string(
"Function ") +
F->getName().str() +
2134 std::string(
" is annotated as a hot function but"
2135 " the profile is cold");
2140 F->addFnAttr(Attribute::Cold);
2141 LLVM_DEBUG(
dbgs() <<
"Set cold attribute to function: " <<
F->getName()
2148 std::string Filename, std::string RemappingFilename,
bool IsCS,
2150 : ProfileFileName(
std::
move(Filename)),
2151 ProfileRemappingFileName(
std::
move(RemappingFilename)), IsCS(IsCS),
2178 LookupTLI, LookupBPI, LookupBFI, PSI, IsCS))
2185 if (!
Node->getName().empty())
2186 return Node->getName().str();
2188 std::string SimpleNodeName;
2196 assert(MaxCount > 0 &&
"Bad max count");
2199 for (
const auto &ECI : EdgeCounts)
2212 if (BrCondStr.empty())
2224 std::string BranchProbStr;
2227 OS <<
" (total count : " << TotalCount <<
")";
2233 << BrCondStr <<
" is true with probability : " << BranchProbStr;
2252 return &
G->getFunc().front();
2275 return std::string(
G->getFunc().getName());
2283 PGOUseBBInfo *BI = Graph->findBBInfo(Node);
2285 if (BI && BI->Count)
2286 OS << *BI->Count <<
"\\l";
2294 if (!isa<SelectInst>(&
I))
2297 OS <<
"SELECT : { T = ";
2301 OS <<
"Unknown, F = Unknown }\\l";
2303 OS << TC <<
", F = " << FC <<
" }\\l";
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
This file finds the minimum set of blocks on a CFG that must be instrumented to infer execution cover...
Analysis containing CSE Info
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Given that RA is a live value
static BasicBlock * getInstrBB(CFGMST< Edge, BBInfo > &MST, Edge &E, const DenseSet< const BasicBlock * > &ExecBlocks)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
Module.h This file contains the declarations for the Module class.
static cl::opt< bool > PGOInstrumentEntry("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))
static cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static cl::opt< bool > PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))
static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)
static cl::opt< bool > PGOVerifyHotBFI("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx)
static cl::opt< bool > PGOTemporalInstrumentation("pgo-temporal-instrumentation", cl::desc("Use this option to enable temporal instrumentation"))
static cl::opt< unsigned > PGOFunctionSizeThreshold("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))
static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static bool skipPGOGen(const Function &F)
static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< unsigned > PGOVerifyBFICutoff("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))
static cl::opt< std::string > PGOTraceFuncHash("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))
static void instrumentOneFunc(Function &F, Module *M, TargetLibraryInfo &TLI, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers, bool IsCS)
static void populateEHOperandBundle(VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)
static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))
static bool InstrumentAllFunctions(Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS)
static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)
static cl::opt< unsigned > PGOVerifyBFIRatio("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))
static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static cl::opt< unsigned > PGOFunctionCriticalEdgeThreshold("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))
static void setupBBInfoEdges(const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo)
Set up InEdges/OutEdges for all BBs in the MST.
static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
static bool skipPGOUse(const Function &F)
static bool canRenameComdat(Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< bool > PGOVerifyBFI("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static cl::opt< bool > PGOBlockCoverage("pgo-block-coverage", cl::desc("Use this option to enable basic block coverage instrumentation"))
static uint64_t sumEdgeCount(const ArrayRef< PGOUseEdge * > Edges)
static cl::opt< bool > PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
Function::ProfileCount ProfileCount
static cl::opt< bool > EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
static cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static std::string getSimpleNodeName(const BasicBlock *Node)
static cl::opt< bool > PGOViewBlockCoverageGraph("pgo-view-block-coverage-graph", cl::desc("Create a dot file of CFGs with block " "coverage inference information"))
static GlobalVariable * createIRLevelProfileFlagVar(Module &M, bool IsCS)
static bool isIndirectBrTarget(BasicBlock *BB)
static std::string getBranchCondString(Instruction *TI)
static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, ProfileSummaryInfo *PSI, bool IsCS)
static cl::opt< PGOViewCountsType > PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static const char * ValueProfKindDescr[]
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Defines the virtual file system interface vfs::FileSystem.
void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const
Returns the estimated profile count of BB.
Conditional or Unconditional Branch instruction.
bool isConditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
An union-find based Minimum Spanning Tree for CFG.
Edge & addEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W)
const std::vector< std::unique_ptr< Edge > > & allEdges() const
size_t bbInfoSize() const
BBInfo * findBBInfo(const BasicBlock *BB) const
BBInfo & getBBInfo(const BasicBlock *BB) const
void dumpEdges(raw_ostream &OS, const Twine &Message) const
Predicate getPredicate() const
Return the predicate for this instruction.
StringRef getName() const
void setSelectionKind(SelectionKind Val)
SelectionKind getSelectionKind() const
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Base class for error info classes.
virtual std::string message() const
Return the error message as a string.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
Class to represent profile counts.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
@ HiddenVisibility
The GV is hidden.
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
This instruction compares its operands according to the predicate given to the constructor.
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
BasicBlock::iterator GetInsertPoint() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Reader for the indexed binary instrprof format.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
uint64_t getMaximumFunctionCount(bool UseCS)
Return the maximum of all known function counts.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
RetTy visitSelectInst(SelectInst &I)
instrprof_error get() const
std::string message() const override
Return the error message as a string.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
const BasicBlock * getParent() const
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
void update(ArrayRef< uint8_t > Data)
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
MDString * createString(StringRef Str)
Return the given string as metadata.
MDNode * createIrrLoopHeaderWeight(uint64_t Weight)
Return metadata containing an irreducible loop header weight.
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PGOInstrumentationUse(std::string Filename="", std::string RemappingFilename="", bool IsCS=false, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
uint64_t getOrCompColdCountThreshold() const
Returns ColdCountThreshold if set.
void refresh()
If no summary is present, attempt to refresh.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
uint64_t getOrCompHotCountThreshold() const
Returns HotCountThreshold if set.
This class represents the LLVM 'select' instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const
Print the current type.
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Utility analysis that determines what values are worth profiling.
std::vector< CandidateInfo > get(InstrProfValueKind Kind) const
returns a list of value profiling candidates of the given kind
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to an std::string.
The virtual file system interface.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
This file contains the declaration of the Comdat class, which represents a single COMDAT in LLVM.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
uint64_t getFuncHash(const FuncRecordTy *Record)
Return the structural hash associated with the function.
void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< FuncNode * > Func
void write64le(void *P, uint64_t V)
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Function::ProfileCount ProfileCount
auto successors(const MachineBasicBlock *BB)
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
DenseMap< BasicBlock *, ColorVector > colorEHFunclets(Function &F)
If an EH funclet personality is in use (see isFuncletEHPersonality), this will recompute which blocks...
cl::opt< bool > PGOWarnMissing
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr)
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
cl::opt< bool > DebugInfoCorrelate
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
OperandBundleDefT< Value * > OperandBundleDef
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
cl::opt< bool > NoPGOWarnMismatch
RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)
cl::opt< PGOViewCountsType > PGOViewCounts("pgo-view-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text with " "block profile counts and branch probabilities " "right after PGO profile annotation step. The " "profile counts are computed using branch " "probabilities from the runtime profile data and " "block frequency propagation algorithm. To view " "the raw counts from the profile, use option " "-pgo-view-raw-counts instead. To limit graph " "display to only one function, use filtering option " "-view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)
Return true if the specified edge is a critical edge.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
auto predecessors(const MachineBasicBlock *BB)
void setProfMetadata(Module *M, Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
SuccIterator< const Instruction, const BasicBlock > const_succ_iterator
cl::opt< bool > NoPGOWarnMismatchComdatWeak
Implement std::hash so that hash_code can be used in STL containers.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
DOTGraphTraits(bool isSimple=false)
static std::string getGraphName(const PGOUseFunc *G)
std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph)
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
static ChildIteratorType child_end(const NodeRef N)
static NodeRef getEntryNode(const PGOUseFunc *G)
static ChildIteratorType child_begin(const NodeRef N)
static nodes_iterator nodes_end(const PGOUseFunc *G)
static nodes_iterator nodes_begin(const PGOUseFunc *G)
Profiling information for a single function.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
static void setCSFlagInHash(uint64_t &FuncHash)
Instruction * AnnotatedInst