128#include <unordered_map>
137#define DEBUG_TYPE "pgo-instrumentation"
139STATISTIC(NumOfPGOInstrument,
"Number of edges instrumented.");
140STATISTIC(NumOfPGOSelectInsts,
"Number of select instruction instrumented.");
141STATISTIC(NumOfPGOMemIntrinsics,
"Number of mem intrinsics instrumented.");
144STATISTIC(NumOfPGOSplit,
"Number of critical edge splits.");
145STATISTIC(NumOfPGOFunc,
"Number of functions having valid profile counts.");
146STATISTIC(NumOfPGOMismatch,
"Number of functions having mismatch profile.");
147STATISTIC(NumOfPGOMissing,
"Number of functions without profile.");
148STATISTIC(NumOfMemProfMissing,
"Number of functions without memory profile.");
149STATISTIC(NumOfPGOICall,
"Number of indirect call value instrumentations.");
150STATISTIC(NumOfCSPGOInstrument,
"Number of edges instrumented in CSPGO.");
152 "Number of select instruction instrumented in CSPGO.");
154 "Number of mem intrinsics instrumented in CSPGO.");
156STATISTIC(NumOfCSPGOBB,
"Number of basic-blocks in CSPGO.");
157STATISTIC(NumOfCSPGOSplit,
"Number of critical edge splits in CSPGO.");
159 "Number of functions having valid profile counts in CSPGO.");
161 "Number of functions having mismatch profile in CSPGO.");
162STATISTIC(NumOfCSPGOMissing,
"Number of functions without profile in CSPGO.");
169 cl::desc(
"Specify the path of profile data file. This is"
170 "mainly for test purpose."));
174 cl::desc(
"Specify the path of profile remapping file. This is mainly for "
181 cl::desc(
"Disable Value Profiling"));
187 cl::desc(
"Max number of annotations for a single indirect "
194 cl::desc(
"Max number of preicise value annotations for a single memop"
201 cl::desc(
"Append function hash to the name of COMDAT function to avoid "
202 "function hash mismatch due to the preinliner"));
208 cl::desc(
"Use this option to turn on/off "
209 "warnings about missing profile data for "
217 cl::desc(
"Use this option to turn off/on "
218 "warnings about profile cfg mismatch."));
226 cl::desc(
"The option is used to turn on/off "
227 "warnings about hash mismatch for comdat "
228 "or weak functions."));
233 cl::desc(
"Use this option to turn on/off SELECT "
234 "instruction instrumentation. "));
239 cl::desc(
"A boolean option to show CFG dag or text "
240 "with raw profile counts from "
241 "profile data. See also option "
242 "-pgo-view-counts. To limit graph "
243 "display to only one function, use "
244 "filtering option -view-bfi-func-name."),
252 cl::desc(
"Use this option to turn on/off "
253 "memory intrinsic size profiling."));
258 cl::desc(
"When this option is on, the annotated "
259 "branch probability will be emitted as "
260 "optimization remarks: -{Rpass|"
261 "pass-remarks}=pgo-instrumentation"));
265 cl::desc(
"Force to instrument function entry basicblock."));
270 "Use this option to enable function entry coverage instrumentation."));
274 cl::desc(
"Fix function entry count in profile use."));
278 cl::desc(
"Print out the non-match BFI count if a hot raw profile count "
279 "becomes non-hot, or a cold raw profile count becomes hot. "
280 "The print is enabled under -Rpass-analysis=pgo, or "
281 "internal option -pass-remakrs-analysis=pgo."));
285 cl::desc(
"Print out mismatched BFI counts after setting profile metadata "
286 "The print is enabled under -Rpass-analysis=pgo, or "
287 "internal option -pass-remakrs-analysis=pgo."));
291 cl::desc(
"Set the threshold for pgo-verify-bfi: only print out "
292 "mismatched BFI if the difference percentage is greater than "
293 "this value (in percentage)."));
297 cl::desc(
"Set the threshold for pgo-verify-bfi: skip the counts whose "
298 "profile count value is below."));
303 cl::desc(
"Trace the hash of the function with this name."));
307 cl::desc(
"Do not instrument functions smaller than this threshold."));
311 cl::desc(
"Perform matching and annotation of memprof profiles."));
315 cl::desc(
"Do not instrument functions with the number of critical edges "
316 " greater than this threshold."));
332 cl::desc(
"Use the old CFG function hashing"));
339 return std::string();
344 return std::string();
356 else if (CV->
isOne())
368#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,
375 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
377 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
379 ProfileVersion |= VARIANT_MASK_CSIR_PROF;
381 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
383 ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
386 VARIANT_MASK_BYTE_COVERAGE | VARIANT_MASK_FUNCTION_ENTRY_ONLY;
391 Triple TT(M.getTargetTriple());
392 if (TT.supportsCOMDAT()) {
394 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
396 return IRLevelVersionVariable;
406enum VisitMode { VM_counting, VM_instrument, VM_annotate };
410struct SelectInstVisitor :
public InstVisitor<SelectInstVisitor> {
413 VisitMode
Mode = VM_counting;
414 unsigned *CurCtrIdx =
nullptr;
415 unsigned TotalNumCtrs = 0;
418 PGOUseFunc *UseFunc =
nullptr;
432 void instrumentSelects(
Function &Func,
unsigned *Ind,
unsigned TotalNC,
434 Mode = VM_instrument;
436 TotalNumCtrs = TotalNC;
443 void annotateSelects(
Function &Func, PGOUseFunc *UF,
unsigned *Ind) {
458 unsigned getNumOfSelectInsts()
const {
return NSIs; }
476 bool Removed =
false;
477 bool IsCritical =
false;
480 : SrcBB(Src), DestBB(Dest), Weight(
W) {}
483 std::string infoString()
const {
484 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
485 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight)).str();
495 BBInfo(
unsigned IX) : Group(this),
Index(IX) {}
498 std::string infoString()
const {
510template <
class Edge,
class BBInfo>
class FuncPGOInstrumentation {
518 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
522 void computeCFGHash();
523 void renameComdatFunction();
527 std::vector<std::vector<VPCandidateInfo>> ValueSites;
528 SelectInstVisitor SIVisitor;
529 std::string FuncName;
540 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);
553 void dumpInfo(std::string Str =
"")
const {
555 Twine(FunctionHash) +
"\t" + Str);
558 FuncPGOInstrumentation(
560 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
563 bool InstrumentFuncEntry =
true)
564 :
F(
Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(
Func, TLI),
565 TLI(TLI), ValueSites(IPVK_Last + 1), SIVisitor(
Func),
566 MST(
F, InstrumentFuncEntry, BPI,
BFI) {
568 SIVisitor.countSelects(Func);
569 ValueSites[IPVK_MemOPSize] = VPC.
get(IPVK_MemOPSize);
571 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
572 NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
573 NumOfPGOBB += MST.
BBInfos.size();
574 ValueSites[IPVK_IndirectCallTarget] = VPC.
get(IPVK_IndirectCallTarget);
576 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
577 NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
578 NumOfCSPGOBB += MST.
BBInfos.size();
583 if (!ComdatMembers.empty())
584 renameComdatFunction();
590 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
592 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
605template <
class Edge,
class BBInfo>
606void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
607 std::vector<uint8_t> Indexes;
613 auto BI = findBBInfo(Succ);
617 for (
int J = 0; J < 4; J++)
618 Indexes.push_back((uint8_t)(
Index >> (J * 8)));
627 FunctionHash = (
uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
628 (
uint64_t)ValueSites[IPVK_IndirectCallTarget].
size() << 48 |
633 auto updateJCH = [&JCH](
uint64_t Num) {
638 updateJCH((
uint64_t)SIVisitor.getNumOfSelectInsts());
639 updateJCH((
uint64_t)ValueSites[IPVK_IndirectCallTarget].
size());
649 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
652 LLVM_DEBUG(
dbgs() <<
"Function Hash Computation for " <<
F.getName() <<
":\n"
653 <<
" CRC = " << JC.
getCRC()
654 <<
", Selects = " << SIVisitor.getNumOfSelectInsts()
655 <<
", Edges = " << MST.
AllEdges.size() <<
", ICSites = "
656 << ValueSites[IPVK_IndirectCallTarget].size());
659 <<
", High32 CRC = " << JCH.
getCRC());
664 dbgs() <<
"Funcname=" <<
F.getName() <<
", Hash=" << FunctionHash
665 <<
" in building " <<
F.getParent()->getSourceFileName() <<
"\n";
671 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
683 for (
auto &&CM :
make_range(ComdatMembers.equal_range(
C))) {
684 assert(!isa<GlobalAlias>(CM.second));
685 Function *FM = dyn_cast<Function>(CM.second);
693template <
class Edge,
class BBInfo>
694void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
697 std::string OrigName =
F.getName().str();
698 std::string NewFuncName =
700 F.setName(
Twine(NewFuncName));
702 FuncName =
Twine(FuncName +
"." +
Twine(FunctionHash)).
str();
708 if (!
F.hasComdat()) {
710 NewComdat =
M->getOrInsertComdat(
StringRef(NewFuncName));
712 F.setComdat(NewComdat);
717 Comdat *OrigComdat =
F.getComdat();
718 std::string NewComdatName =
720 NewComdat =
M->getOrInsertComdat(
StringRef(NewComdatName));
723 for (
auto &&CM :
make_range(ComdatMembers.equal_range(OrigComdat))) {
725 cast<Function>(CM.second)->setComdat(NewComdat);
731template <
class Edge,
class BBInfo>
732void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
733 std::vector<BasicBlock *> &InstrumentBBs) {
735 std::vector<Edge *> EdgeList;
736 EdgeList.reserve(MST.
AllEdges.size());
738 EdgeList.push_back(
E.get());
740 for (
auto &
E : EdgeList) {
743 InstrumentBBs.push_back(InstrBB);
752 BBInfo &SrcInfo = getBBInfo(SrcBB);
753 BBInfo &DestInfo = getBBInfo(DestBB);
754 SrcInfo.addOutEdge(
E.get());
755 DestInfo.addInEdge(
E.get());
761template <
class Edge,
class BBInfo>
762BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *
E) {
763 if (
E->InMST ||
E->Removed)
769 if (SrcBB ==
nullptr)
771 if (DestBB ==
nullptr)
786 return canInstrument(SrcBB);
788 return canInstrument(DestBB);
797 dbgs() <<
"Fail to split critical edge: not instrument this edge.\n");
802 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
804 <<
" --> " << getBBInfo(DestBB).
Index <<
"\n");
806 MST.
addEdge(SrcBB, InstrBB, 0);
808 Edge &NewEdge1 = MST.
addEdge(InstrBB, DestBB, 0);
809 NewEdge1.InMST =
true;
812 return canInstrument(InstrBB);
828 if (!isa<IntrinsicInst>(OrigCall)) {
831 std::optional<OperandBundleUse> ParentFunclet =
839 if (!BlockColors.
empty()) {
840 const ColorVector &CV = BlockColors.
find(OrigCall->getParent())->second;
841 assert(CV.
size() == 1 &&
"non-unique color for block!");
854 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
860 FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(
866 FuncInfo.FunctionHash);
868 auto &EntryBB =
F.getEntryBlock();
878 std::vector<BasicBlock *> InstrumentBBs;
879 FuncInfo.getInstrumentBBs(InstrumentBBs);
880 unsigned NumCounters =
881 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
884 for (
auto *InstrBB : InstrumentBBs) {
887 "Cannot get the Instrumentation point");
896 FuncInfo.SIVisitor.instrumentSelects(
F, &
I, NumCounters, FuncInfo.FuncNameVar,
897 FuncInfo.FunctionHash);
903 NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();
910 if (
F.hasPersonalityFn() &&
915 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
916 unsigned SiteIndex = 0;
922 <<
" site: CallSite Index = " << SiteIndex <<
"\n");
925 assert(
Builder.GetInsertPoint() != Cand.InsertPt->getParent()->end() &&
926 "Cannot get the Instrumentation point");
928 Value *ToProfile =
nullptr;
929 if (Cand.V->getType()->isIntegerTy())
930 ToProfile =
Builder.CreateZExtOrTrunc(Cand.V,
Builder.getInt64Ty());
931 else if (Cand.V->getType()->isPointerTy())
933 assert(ToProfile &&
"value profiling Value is of unexpected type");
940 Builder.getInt64(FuncInfo.FunctionHash), ToProfile,
950struct PGOUseEdge :
public PGOEdge {
951 bool CountValid =
false;
955 : PGOEdge(Src, Dest,
W) {}
964 std::string infoString()
const {
966 return PGOEdge::infoString();
967 return (
Twine(PGOEdge::infoString()) +
" Count=" +
Twine(CountValue))
975struct UseBBInfo :
public BBInfo {
978 int32_t UnknownCountInEdge = 0;
979 int32_t UnknownCountOutEdge = 0;
981 DirectEdges OutEdges;
983 UseBBInfo(
unsigned IX) : BBInfo(IX), CountValid(
false) {}
986 : BBInfo(IX), CountValue(
C), CountValid(
true) {}
995 std::string infoString()
const {
997 return BBInfo::infoString();
998 return (
Twine(BBInfo::infoString()) +
" Count=" +
Twine(CountValue)).str();
1002 void addOutEdge(PGOUseEdge *
E) {
1003 OutEdges.push_back(
E);
1004 UnknownCountOutEdge++;
1008 void addInEdge(PGOUseEdge *
E) {
1009 InEdges.push_back(
E);
1010 UnknownCountInEdge++;
1019 for (
const auto &
E : Edges) {
1032 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
1035 :
F(
Func),
M(Modu),
BFI(BFIin), PSI(PSI),
1036 FuncInfo(
Func, TLI, ComdatMembers,
false, BPI, BFIin, IsCS,
1037 InstrumentFuncEntry),
1038 FreqAttr(FFA_Normal), IsCS(IsCS) {}
1048 void populateCounters();
1054 void annotateValueSites();
1057 void annotateValueSites(
uint32_t Kind);
1060 void annotateIrrLoopHeaderWeights();
1063 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
1066 FuncFreqAttr getFuncFreqAttr()
const {
return FreqAttr; }
1075 UseBBInfo &getBBInfo(
const BasicBlock *BB)
const {
1076 return FuncInfo.getBBInfo(BB);
1080 UseBBInfo *findBBInfo(
const BasicBlock *BB)
const {
1081 return FuncInfo.findBBInfo(BB);
1086 void dumpInfo(std::string Str =
"")
const {
1087 FuncInfo.dumpInfo(Str);
1090 uint64_t getProgramMaxCount()
const {
return ProgramMaxCount; }
1098 FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
1114 FuncFreqAttr FreqAttr;
1120 bool setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile);
1127 std::string getFuncName()
const {
return FuncInfo.FuncName; }
1136 FreqAttr = FFA_Cold;
1144bool PGOUseFunc::setInstrumentedCounts(
1145 const std::vector<uint64_t> &CountFromProfile) {
1147 std::vector<BasicBlock *> InstrumentBBs;
1148 FuncInfo.getInstrumentBBs(InstrumentBBs);
1149 unsigned NumCounters =
1150 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
1153 if (NumCounters != CountFromProfile.size()) {
1156 auto *FuncEntry = &*
F.begin();
1161 uint64_t CountValue = CountFromProfile[
I++];
1162 UseBBInfo &
Info = getBBInfo(InstrBB);
1166 if (InstrBB == FuncEntry && CountValue == 0)
1168 Info.setBBInfoCount(CountValue);
1170 ProfileCountSize = CountFromProfile.size();
1174 auto setEdgeCount = [
this](PGOUseEdge *
E,
uint64_t Value) ->
void {
1176 this->getBBInfo(
E->SrcBB).UnknownCountOutEdge--;
1177 this->getBBInfo(
E->DestBB).UnknownCountInEdge--;
1183 for (
auto &
E : FuncInfo.MST.AllEdges) {
1184 if (
E->Removed ||
E->InMST)
1187 UseBBInfo &SrcInfo = getBBInfo(SrcBB);
1191 if (SrcInfo.CountValid && SrcInfo.OutEdges.size() == 1)
1192 setEdgeCount(
E.get(), SrcInfo.CountValue);
1195 UseBBInfo &DestInfo = getBBInfo(DestBB);
1198 if (DestInfo.CountValid && DestInfo.InEdges.size() == 1)
1199 setEdgeCount(
E.get(), DestInfo.CountValue);
1205 setEdgeCount(
E.get(), 0);
1212void PGOUseFunc::setEdgeCount(DirectEdges &Edges,
uint64_t Value) {
1213 for (
auto &
E : Edges) {
1218 getBBInfo(
E->SrcBB).UnknownCountOutEdge--;
1219 getBBInfo(
E->DestBB).UnknownCountInEdge--;
1228 const char MetadataName[] =
"instr_prof_hash_mismatch";
1231 auto *Existing =
F.getMetadata(LLVMContext::MD_annotation);
1233 MDTuple *Tuple = cast<MDTuple>(Existing);
1234 for (
const auto &
N : Tuple->
operands()) {
1235 if (cast<MDString>(
N.get())->getString() == MetadataName)
1237 Names.push_back(
N.get());
1244 F.setMetadata(LLVMContext::MD_annotation, MD);
1248 std::vector<uint64_t> &InlinedCallStack,
1250 I.setMetadata(LLVMContext::MD_callsite,
1261 std::memcpy(&Id, Hash.data(),
sizeof(Hash));
1272 for (
auto StackFrame :
AllocInfo->CallStack)
1287 unsigned StartIndex = 0) {
1288 auto StackFrame = ProfileCallStack.
begin() + StartIndex;
1289 auto InlCallStackIter = InlinedCallStack.
begin();
1290 for (; StackFrame != ProfileCallStack.
end() &&
1291 InlCallStackIter != InlinedCallStack.
end();
1292 ++StackFrame, ++InlCallStackIter) {
1294 if (StackId != *InlCallStackIter)
1299 return InlCallStackIter == InlinedCallStack.
end();
1306 auto &Ctx =
M->getContext();
1308 auto FuncGUID = Function::getGUID(FuncInfo.FuncName);
1313 auto Err = IPE.
get();
1314 bool SkipWarning =
false;
1316 << FuncInfo.FuncName <<
": ");
1317 if (Err == instrprof_error::unknown_function) {
1318 NumOfMemProfMissing++;
1321 }
else if (Err == instrprof_error::hash_mismatch) {
1327 LLVM_DEBUG(
dbgs() <<
"hash mismatch (skip=" << SkipWarning <<
")");
1335 std::to_string(FuncInfo.FunctionHash))
1346 std::map<uint64_t, std::set<const AllocationInfo *>> LocHashToAllocInfo;
1349 std::map<uint64_t, std::set<std::pair<const SmallVector<Frame> *,
unsigned>>>
1351 const auto MemProfRec = std::move(MemProfResult.
get());
1352 for (
auto &AI : MemProfRec.AllocSites) {
1357 LocHashToAllocInfo[StackId].insert(&AI);
1359 for (
auto &CS : MemProfRec.CallSites) {
1363 for (
auto &StackFrame : CS) {
1365 LocHashToCallSites[StackId].insert(std::make_pair(&CS,
Idx++));
1367 if (StackFrame.Function == FuncGUID)
1370 assert(
Idx <= CS.size() && CS[
Idx - 1].Function == FuncGUID);
1374 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &
1380 for (
auto &BB :
F) {
1381 for (
auto &
I : BB) {
1382 if (
I.isDebugOrPseudoInst())
1386 auto *CI = dyn_cast<CallBase>(&
I);
1389 auto *CalledFunction = CI->getCalledFunction();
1390 if (CalledFunction && CalledFunction->isIntrinsic())
1394 std::vector<uint64_t> InlinedCallStack;
1396 bool LeafFound =
false;
1402 std::map<uint64_t, std::set<const AllocationInfo *>>::iterator
1404 std::map<uint64_t, std::set<std::pair<const SmallVector<Frame> *,
1405 unsigned>>>::iterator CallSitesIter;
1406 for (
const DILocation *DIL =
I.getDebugLoc(); DIL !=
nullptr;
1407 DIL = DIL->getInlinedAt()) {
1410 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
1412 Name = DIL->getScope()->getSubprogram()->getName();
1413 auto CalleeGUID = Function::getGUID(
Name);
1419 AllocInfoIter = LocHashToAllocInfo.find(StackId);
1420 CallSitesIter = LocHashToCallSites.find(StackId);
1423 if (AllocInfoIter == LocHashToAllocInfo.end() &&
1424 CallSitesIter == LocHashToCallSites.end())
1428 InlinedCallStack.push_back(StackId);
1438 if (AllocInfoIter != LocHashToAllocInfo.end()) {
1447 for (
auto *
AllocInfo : AllocInfoIter->second) {
1458 if (!AllocTrie.
empty()) {
1462 assert(MemprofMDAttached ==
I.hasMetadata(LLVMContext::MD_memprof));
1463 if (MemprofMDAttached) {
1480 assert(CallSitesIter != LocHashToCallSites.end());
1481 for (
auto CallStackIdx : CallSitesIter->second) {
1485 *CallStackIdx.first, InlinedCallStack, CallStackIdx.second)) {
1503 auto &Ctx =
M->getContext();
1506 FuncInfo.FuncName, FuncInfo.FunctionHash, &MismatchedFuncSum);
1509 auto Err = IPE.
get();
1510 bool SkipWarning =
false;
1512 << FuncInfo.FuncName <<
": ");
1513 if (Err == instrprof_error::unknown_function) {
1514 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
1517 }
else if (Err == instrprof_error::hash_mismatch ||
1518 Err == instrprof_error::malformed) {
1519 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
1525 LLVM_DEBUG(
dbgs() <<
"hash mismatch (hash= " << FuncInfo.FunctionHash
1526 <<
" skip=" << SkipWarning <<
")");
1536 IPE.
message() + std::string(
" ") +
F.getName().str() +
1537 std::string(
" Hash = ") + std::to_string(FuncInfo.FunctionHash) +
1538 std::string(
" up to ") + std::to_string(MismatchedFuncSum) +
1539 std::string(
" count discarded");
1546 ProfileRecord = std::move(
Result.get());
1551 std::vector<uint64_t> &CountFromProfile = ProfileRecord.
Counts;
1553 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1557 for (
unsigned I = 0, S = CountFromProfile.size();
I < S;
I++) {
1559 ValueSum += CountFromProfile[
I];
1561 AllZeros = (ValueSum == 0);
1565 getBBInfo(
nullptr).UnknownCountOutEdge = 2;
1566 getBBInfo(
nullptr).UnknownCountInEdge = 2;
1568 if (!setInstrumentedCounts(CountFromProfile)) {
1570 dbgs() <<
"Inconsistent number of counts, skipping this function");
1572 M->getName().data(),
1573 Twine(
"Inconsistent number of counts in ") +
F.getName().str()
1574 +
Twine(
": the profile may be stale or there is a function name collision."),
1584void PGOUseFunc::populateCounters() {
1585 bool Changes =
true;
1586 unsigned NumPasses = 0;
1594 UseBBInfo *Count = findBBInfo(&BB);
1595 if (Count ==
nullptr)
1597 if (!Count->CountValid) {
1598 if (Count->UnknownCountOutEdge == 0) {
1600 Count->CountValid =
true;
1602 }
else if (Count->UnknownCountInEdge == 0) {
1604 Count->CountValid =
true;
1608 if (Count->CountValid) {
1609 if (Count->UnknownCountOutEdge == 1) {
1615 if (Count->CountValue > OutSum)
1616 Total = Count->CountValue - OutSum;
1617 setEdgeCount(Count->OutEdges,
Total);
1620 if (Count->UnknownCountInEdge == 1) {
1623 if (Count->CountValue > InSum)
1624 Total = Count->CountValue - InSum;
1625 setEdgeCount(Count->InEdges,
Total);
1632 LLVM_DEBUG(
dbgs() <<
"Populate counts in " << NumPasses <<
" passes.\n");
1636 for (
auto &BB :
F) {
1637 auto BI = findBBInfo(&BB);
1640 assert(BI->CountValid &&
"BB count is not valid");
1643 uint64_t FuncEntryCount = getBBInfo(&*
F.begin()).CountValue;
1644 uint64_t FuncMaxCount = FuncEntryCount;
1645 for (
auto &BB :
F) {
1646 auto BI = findBBInfo(&BB);
1649 FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
1653 if (FuncMaxCount > 0 && FuncEntryCount == 0)
1656 markFunctionAttributes(FuncEntryCount, FuncMaxCount);
1659 FuncInfo.SIVisitor.annotateSelects(
F,
this, &CountPosition);
1660 assert(CountPosition == ProfileCountSize);
1662 LLVM_DEBUG(FuncInfo.dumpInfo(
"after reading profile."));
1666void PGOUseFunc::setBranchWeights() {
1668 LLVM_DEBUG(
dbgs() <<
"\nSetting branch weights for func " <<
F.getName()
1669 <<
" IsCS=" << IsCS <<
"\n");
1670 for (
auto &BB :
F) {
1674 if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
1675 isa<IndirectBrInst>(TI) || isa<InvokeInst>(TI) ||
1676 isa<CallBrInst>(TI)))
1679 if (getBBInfo(&BB).CountValue == 0)
1683 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
1684 unsigned Size = BBCountInfo.OutEdges.size();
1687 for (
unsigned s = 0; s <
Size; s++) {
1688 const PGOUseEdge *
E = BBCountInfo.OutEdges[s];
1691 if (DestBB ==
nullptr)
1695 if (EdgeCount > MaxCount)
1696 MaxCount = EdgeCount;
1697 EdgeCounts[SuccNum] = EdgeCount;
1706 auto &Ctx =
M->getContext();
1708 M->getName().data(),
1709 Twine(
"Profile in ") +
F.getName().str() +
1710 Twine(
" partially ignored") +
1711 Twine(
", possibly due to the lack of a return path."),
1719 if (isa<IndirectBrInst>(Pred->getTerminator()))
1725void PGOUseFunc::annotateIrrLoopHeaderWeights() {
1726 LLVM_DEBUG(
dbgs() <<
"\nAnnotating irreducible loop header weights.\n");
1728 for (
auto &BB :
F) {
1734 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
1740void SelectInstVisitor::instrumentOneSelectInst(
SelectInst &SI) {
1747 auto *Step =
Builder.CreateZExt(
SI.getCondition(), Int64Ty);
1752 Builder.getInt32(*CurCtrIdx), Step});
1756void SelectInstVisitor::annotateOneSelectInst(
SelectInst &SI) {
1757 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1758 assert(*CurCtrIdx < CountFromProfile.size() &&
1759 "Out of bound access of counters");
1761 SCounts[0] = CountFromProfile[*CurCtrIdx];
1764 auto BI = UseFunc->findBBInfo(
SI.getParent());
1766 TotalCount = BI->CountValue;
1768 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1769 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1774void SelectInstVisitor::visitSelectInst(
SelectInst &SI) {
1778 if (
SI.getCondition()->getType()->isVectorTy())
1786 instrumentOneSelectInst(SI);
1789 annotateOneSelectInst(SI);
1797void PGOUseFunc::annotateValueSites() {
1805 annotateValueSites(Kind);
1809void PGOUseFunc::annotateValueSites(
uint32_t Kind) {
1810 assert(Kind <= IPVK_Last);
1811 unsigned ValueSiteIndex = 0;
1812 auto &ValueSites = FuncInfo.ValueSites[
Kind];
1814 if (NumValueSites != ValueSites.size()) {
1815 auto &Ctx =
M->getContext();
1817 M->getName().data(),
1818 Twine(
"Inconsistent number of value sites for ") +
1820 Twine(
" profiling in \"") +
F.getName().str() +
1821 Twine(
"\", possibly due to the use of a stale profile."),
1827 LLVM_DEBUG(
dbgs() <<
"Read one value site profile (kind = " << Kind
1828 <<
"): Index = " << ValueSiteIndex <<
" out of "
1829 << NumValueSites <<
"\n");
1842 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1847 ComdatMembers.insert(std::make_pair(
C, &
F));
1849 if (
Comdat *
C = GV.getComdat())
1850 ComdatMembers.insert(std::make_pair(
C, &GV));
1852 if (
Comdat *
C = GA.getComdat())
1853 ComdatMembers.insert(std::make_pair(
C, &GA));
1858 if (
F.isDeclaration())
1860 if (
F.hasFnAttribute(llvm::Attribute::NoProfile))
1862 if (
F.hasFnAttribute(llvm::Attribute::SkipProfile))
1870 unsigned NumCriticalEdges = 0;
1871 for (
auto &BB :
F) {
1880 <<
", NumCriticalEdges=" << NumCriticalEdges
1881 <<
" exceed the threshold. Skip PGO.\n");
1896 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1902 auto &TLI = LookupTLI(
F);
1903 auto *BPI = LookupBPI(
F);
1904 auto *BFI = LookupBFI(
F);
1945 auto BFIEntryCount =
F.getEntryCount();
1946 assert(BFIEntryCount && (BFIEntryCount->getCount() > 0) &&
1947 "Invalid BFI Entrycount");
1951 for (
auto &BBI :
F) {
1954 if (!Func.findBBInfo(&BBI))
1957 CountValue = Func.getBBInfo(&BBI).CountValue;
1958 BFICountValue = *BFICount;
1962 if (SumCount.isZero())
1966 "Incorrect sum of BFI counts");
1969 double Scale = (SumCount / SumBFICount).convertToDouble();
1970 if (Scale < 1.001 && Scale > 0.999)
1973 uint64_t FuncEntryCount = Func.getBBInfo(&*
F.begin()).CountValue;
1974 uint64_t NewEntryCount = 0.5 + FuncEntryCount * Scale;
1975 if (NewEntryCount == 0)
1977 if (NewEntryCount != FuncEntryCount) {
1980 <<
", entry_count " << FuncEntryCount <<
" --> "
1981 << NewEntryCount <<
"\n");
1998 unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
1999 for (
auto &BBI :
F) {
2003 if (Func.getBBInfo(&BBI).CountValid)
2004 CountValue = Func.getBBInfo(&BBI).CountValue;
2011 BFICountValue = *BFICount;
2014 bool rawIsHot = CountValue >= HotCountThreshold;
2015 bool BFIIsHot = BFICountValue >= HotCountThreshold;
2017 bool ShowCount =
false;
2018 if (rawIsHot && !BFIIsHot) {
2019 Msg =
"raw-Hot to BFI-nonHot";
2021 }
else if (rawIsCold && BFIIsHot) {
2022 Msg =
"raw-Cold to BFI-Hot";
2031 uint64_t Diff = (BFICountValue >= CountValue)
2032 ? BFICountValue - CountValue
2033 : CountValue - BFICountValue;
2041 F.getSubprogram(), &BBI);
2043 <<
" Count=" <<
ore::NV(
"Count", CountValue)
2044 <<
" BFI_Count=" <<
ore::NV(
"Count", BFICountValue);
2046 Remark <<
" (" << Msg <<
")";
2053 F.getSubprogram(), &
F.getEntryBlock())
2054 <<
"In Func " <<
ore::NV(
"Function",
F.getName())
2055 <<
": Num_of_BB=" <<
ore::NV(
"Count", BBNum)
2056 <<
", Num_of_non_zerovalue_BB=" <<
ore::NV(
"Count", NonZeroBBNum)
2057 <<
", Num_of_mis_matching_BB=" <<
ore::NV(
"Count", BBMisMatchNum);
2069 auto &Ctx = M.getContext();
2072 ProfileRemappingFileName);
2073 if (
Error E = ReaderOrErr.takeError()) {
2081 std::unique_ptr<IndexedInstrProfReader> PGOReader =
2082 std::move(ReaderOrErr.get());
2088 if (!PGOReader->hasCSIRLevelProfile() && IsCS)
2092 if (!PGOReader->isIRLevelProfile() && !PGOReader->hasMemoryProfile()) {
2094 ProfileFileName.
data(),
"Not an IR level instrumentation profile"));
2097 if (PGOReader->hasSingleByteCoverage()) {
2099 ProfileFileName.
data(),
2100 "Cannot use coverage profiles for optimization"));
2103 if (PGOReader->functionEntryOnly()) {
2105 ProfileFileName.
data(),
2106 "Function entry profiles are not yet supported for optimization"));
2113 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
2118 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
2120 std::vector<Function *> HotFunctions;
2121 std::vector<Function *> ColdFunctions;
2125 bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
2131 auto &TLI = LookupTLI(
F);
2132 auto *BPI = LookupBPI(
F);
2133 auto *BFI = LookupBFI(
F);
2137 PGOUseFunc Func(
F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
2138 InstrumentFuncEntry);
2141 if (PGOReader->hasMemoryProfile())
2142 Func.readMemprof(PGOReader.get());
2144 if (!PGOReader->isIRLevelProfile())
2152 bool AllZeros =
false;
2153 if (!Func.readCounters(PGOReader.get(), AllZeros, PseudoKind))
2157 if (Func.getProgramMaxCount() != 0)
2158 ColdFunctions.push_back(&
F);
2163 if (
F.hasFnAttribute(Attribute::Cold))
2164 F.removeFnAttr(Attribute::Cold);
2167 F.addFnAttr(Attribute::Hot);
2170 Func.populateCounters();
2171 Func.setBranchWeights();
2172 Func.annotateValueSites();
2173 Func.annotateIrrLoopHeaderWeights();
2174 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
2175 if (FreqAttr == PGOUseFunc::FFA_Cold)
2176 ColdFunctions.push_back(&
F);
2177 else if (FreqAttr == PGOUseFunc::FFA_Hot)
2178 HotFunctions.push_back(&
F);
2183 std::unique_ptr<BranchProbabilityInfo> NewBPI =
2184 std::make_unique<BranchProbabilityInfo>(
F, LI);
2185 std::unique_ptr<BlockFrequencyInfo> NewBFI =
2186 std::make_unique<BlockFrequencyInfo>(
F, *NewBPI, LI);
2190 dbgs() <<
"pgo-view-counts: " << Func.getFunc().getName() <<
"\n";
2191 NewBFI->print(
dbgs());
2201 ViewGraph(&Func,
Twine(
"PGORawCounts_") + Func.getFunc().getName());
2203 dbgs() <<
"pgo-view-raw-counts: " << Func.getFunc().getName() <<
"\n";
2230 for (
auto &
F : HotFunctions) {
2231 F->addFnAttr(Attribute::InlineHint);
2232 LLVM_DEBUG(
dbgs() <<
"Set inline attribute to function: " <<
F->getName()
2235 for (
auto &
F : ColdFunctions) {
2238 if (
F->hasFnAttribute(Attribute::Hot)) {
2239 auto &Ctx = M.getContext();
2240 std::string Msg = std::string(
"Function ") +
F->getName().str() +
2241 std::string(
" is annotated as a hot function but"
2242 " the profile is cold");
2247 F->addFnAttr(Attribute::Cold);
2248 LLVM_DEBUG(
dbgs() <<
"Set cold attribute to function: " <<
F->getName()
2255 std::string Filename, std::string RemappingFilename,
bool IsCS,
2257 : ProfileFileName(
std::
move(Filename)),
2258 ProfileRemappingFileName(
std::
move(RemappingFilename)), IsCS(IsCS),
2285 LookupTLI, LookupBPI, LookupBFI, PSI, IsCS))
2292 if (!
Node->getName().empty())
2293 return std::string(
Node->getName());
2295 std::string SimpleNodeName;
2305 assert(MaxCount > 0 &&
"Bad max count");
2308 for (
const auto &ECI : EdgeCounts)
2321 if (BrCondStr.empty())
2333 std::string BranchProbStr;
2336 OS <<
" (total count : " << TotalCount <<
")";
2342 << BrCondStr <<
" is true with probability : " << BranchProbStr;
2361 return &
G->getFunc().front();
2384 return std::string(
G->getFunc().getName());
2392 UseBBInfo *BI = Graph->findBBInfo(Node);
2394 if (BI && BI->CountValid)
2395 OS << BI->CountValue <<
"\\l";
2403 if (!isa<SelectInst>(&
I))
2406 OS <<
"SELECT : { T = ";
2410 OS <<
"Unknown, F = Unknown }\\l";
2412 OS << TC <<
", F = " << FC <<
" }\\l";
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static BasicBlock * getInstrBB(CFGMST< Edge, BBInfo > &MST, Edge &E, const DenseSet< const BasicBlock * > &ExecBlocks)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
Module.h This file contains the declarations for the Module class.
return ToRemove size() > 0
static cl::opt< bool > PGOInstrumentEntry("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))
static cl::opt< bool > PGOWarnMissing("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions."))
static cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static cl::opt< bool > PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))
static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)
static cl::opt< bool > PGOVerifyHotBFI("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx)
static cl::opt< bool > MatchMemProf("pgo-match-memprof", cl::init(true), cl::Hidden, cl::desc("Perform matching and annotation of memprof profiles."))
static bool skipPGO(const Function &F)
static cl::opt< unsigned > PGOFunctionSizeThreshold("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))
static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static void addCallStack(CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo)
static cl::opt< unsigned > PGOVerifyBFICutoff("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))
static cl::opt< std::string > PGOTraceFuncHash("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))
static void instrumentOneFunc(Function &F, Module *M, TargetLibraryInfo &TLI, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers, bool IsCS)
static void populateEHOperandBundle(VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)
static uint64_t computeStackId(GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)
static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))
static bool InstrumentAllFunctions(Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS)
static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)
static cl::opt< unsigned > PGOVerifyBFIRatio("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))
static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static cl::opt< unsigned > PGOFunctionCriticalEdgeThreshold("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))
static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
static bool canRenameComdat(Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< bool > PGOVerifyBFI("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static cl::opt< bool > PGOOldCFGHashing("pgo-instr-old-cfg-hashing", cl::init(false), cl::Hidden, cl::desc("Use the old CFG function hashing"))
static uint64_t sumEdgeCount(const ArrayRef< PGOUseEdge * > Edges)
static cl::opt< bool > PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
Function::ProfileCount ProfileCount
static void addCallsiteMetadata(Instruction &I, std::vector< uint64_t > &InlinedCallStack, LLVMContext &Ctx)
static cl::opt< bool > EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
static cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static std::string getSimpleNodeName(const BasicBlock *Node)
static cl::opt< bool > NoPGOWarnMismatchComdatWeak("no-pgo-warn-mismatch-comdat-weak", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "or weak functions."))
static bool stackFrameIncludesInlinedCallStack(ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack, unsigned StartIndex=0)
static GlobalVariable * createIRLevelProfileFlagVar(Module &M, bool IsCS)
static bool isIndirectBrTarget(BasicBlock *BB)
static std::string getBranchCondString(Instruction *TI)
static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, ProfileSummaryInfo *PSI, bool IsCS)
static cl::opt< PGOViewCountsType > PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static const char * ValueProfKindDescr[]
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
This file contains the declarations for profiling metadata utility functions.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
static void setBranchWeights(SwitchInst *SI, ArrayRef< uint32_t > Weights)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Defines the virtual file system interface vfs::FileSystem.
void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const
Returns the estimated profile count of BB.
Conditional or Unconditional Branch instruction.
bool isConditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
An union-find based Minimum Spanning Tree for CFG.
Edge & addEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W)
std::vector< std::unique_ptr< Edge > > AllEdges
BBInfo * findBBInfo(const BasicBlock *BB) const
BBInfo & getBBInfo(const BasicBlock *BB) const
DenseMap< const BasicBlock *, std::unique_ptr< BBInfo > > BBInfos
void dumpEdges(raw_ostream &OS, const Twine &Message) const
Predicate getPredicate() const
Return the predicate for this instruction.
StringRef getName() const
void setSelectionKind(SelectionKind Val)
SelectionKind getSelectionKind() const
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
iterator find(const_arg_type_t< KeyT > Val)
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Base class for error info classes.
virtual std::string message() const
Return the error message as a string.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
reference get()
Returns a reference to the stored T value.
Class to represent profile counts.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
@ HiddenVisibility
The GV is hidden.
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
HashResultTy< HasherT_ > final()
Forward to HasherT::final() if available.
Implementation of the HashBuilder interface.
std::enable_if_t< hashbuilder_detail::IsHashableData< T >::value, HashBuilderImpl & > add(T Value)
Implement hashing for hashable data types, e.g. integral or enum values.
This instruction compares its operands according to the predicate given to the constructor.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Reader for the indexed binary instrprof format.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< memprof::MemProfRecord > getMemProfRecord(uint64_t FuncNameHash)
Return the memprof record for the function identified by llvm::md5(Name).
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
uint64_t getMaximumFunctionCount(bool UseCS)
Return the maximum of all known function counts.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
RetTy visitSelectInst(SelectInst &I)
instrprof_error get() const
std::string message() const override
Return the error message as a string.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
const BasicBlock * getParent() const
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
void update(ArrayRef< uint8_t > Data)
This is an important class for using LLVM in a threaded context.
MDString * createString(StringRef Str)
Return the given string as metadata.
MDNode * createIrrLoopHeaderWeight(uint64_t Weight)
Return metadata containing an irreducible loop header weight.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
PGOInstrumentationUse(std::string Filename="", std::string RemappingFilename="", bool IsCS=false, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
uint64_t getOrCompColdCountThreshold() const
Returns ColdCountThreshold if set.
void refresh()
If no summary is present, attempt to refresh.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
uint64_t getOrCompHotCountThreshold() const
Returns HotCountThreshold if set.
This class represents the LLVM 'select' instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const
Print the current type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Utility analysis that determines what values are worth profiling.
std::vector< CandidateInfo > get(InstrProfValueKind Kind) const
returns a list of value profiling candidates of the given kind
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
An efficient, type-erasing, non-owning reference to a callable.
Class to build a trie of call stack contexts for a particular profiled allocation call,...
void addCallStack(AllocationType AllocType, ArrayRef< uint64_t > StackIds)
Add a call stack context with the given allocation type to the Trie.
bool buildAndAttachMIBMetadata(CallBase *CI)
Build and attach the minimal necessary MIB metadata.
A raw_ostream that writes to an std::string.
The virtual file system interface.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
This file contains the declaration of the Comdat class, which represents a single COMDAT in LLVM.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
uint64_t getFuncHash(const FuncRecordTy *Record)
Return the structural hash associated with the function.
MDNode * buildCallstackMetadata(ArrayRef< uint64_t > CallStack, LLVMContext &Ctx)
Build callstack metadata from the provided list of call stack ids.
AllocationType getAllocType(uint64_t MaxAccessCount, uint64_t MinSize, uint64_t MinLifetime)
Return the allocation type for a given set of memory profile values.
void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
DiagnosticInfoOptimizationBase::Argument NV
void write64le(void *P, uint64_t V)
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Interval::succ_iterator succ_end(Interval *I)
void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count)
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
Function::ProfileCount ProfileCount
cl::opt< bool > NoPGOWarnMismatch
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
DenseMap< BasicBlock *, ColorVector > colorEHFunclets(Function &F)
If an EH funclet personality is in use (see isFuncletEHPersonality), this will recompute which blocks...
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr)
OperandBundleDefT< Value * > OperandBundleDef
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
std::array< uint8_t, NumBytes > BLAKE3Result
The constant LLVM_BLAKE3_OUT_LEN provides the default output length, 32 bytes, which is recommended f...
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
HashBuilderImpl< HasherT,(Endianness==support::endianness::native ? support::endian::system_endianness() :Endianness)> HashBuilder
Interface to help hash various types through a hasher type.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
cl::opt< PGOViewCountsType > PGOViewCounts("pgo-view-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text with " "block profile counts and branch probabilities " "right after PGO profile annotation step. The " "profile counts are computed using branch " "probabilities from the runtime profile data and " "block frequency propagation algorithm. To view " "the raw counts from the profile, use option " "-pgo-view-raw-counts instead. To limit graph " "display to only one function, use filtering option " "-view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)
Return true if the specified edge is a critical edge.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
auto predecessors(const MachineBasicBlock *BB)
void setProfMetadata(Module *M, Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)
cl::opt< bool > DebugInfoCorrelate("debug-info-correlate", cl::desc("Use debug info to correlate profiles."), cl::init(false))
bool isNewLikeFn(const Value *V, const TargetLibraryInfo *TLI)
Tests if a value is a call or invoke to a library function that allocates memory via new.
SuccIterator< const Instruction, const BasicBlock > const_succ_iterator
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
Summary of memprof metadata on allocations.
DOTGraphTraits(bool isSimple=false)
static std::string getGraphName(const PGOUseFunc *G)
std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph)
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
static ChildIteratorType child_end(const NodeRef N)
static NodeRef getEntryNode(const PGOUseFunc *G)
static ChildIteratorType child_begin(const NodeRef N)
static nodes_iterator nodes_end(const PGOUseFunc *G)
static nodes_iterator nodes_begin(const PGOUseFunc *G)
Profiling information for a single function.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
static void setCSFlagInHash(uint64_t &FuncHash)
Instruction * AnnotatedInst
GlobalValue::GUID Function