80 #include <unordered_map>
86 #define DEBUG_TYPE "pgo-instrumentation"
88 STATISTIC(NumOfPGOInstrument,
"Number of edges instrumented.");
89 STATISTIC(NumOfPGOSelectInsts,
"Number of select instruction instrumented.");
90 STATISTIC(NumOfPGOEdge,
"Number of edges.");
91 STATISTIC(NumOfPGOBB,
"Number of basic-blocks.");
92 STATISTIC(NumOfPGOSplit,
"Number of critical edge splits.");
93 STATISTIC(NumOfPGOFunc,
"Number of functions having valid profile counts.");
94 STATISTIC(NumOfPGOMismatch,
"Number of functions having mismatch profile.");
95 STATISTIC(NumOfPGOMissing,
"Number of functions without profile.");
96 STATISTIC(NumOfPGOICall,
"Number of indirect call value instrumentations.");
103 cl::desc(
"Specify the path of profile data file. This is"
104 "mainly for test purpose."));
110 cl::desc(
"Disable Value Profiling"));
116 cl::desc(
"Max number of annotations for a single indirect "
123 cl::desc(
"Append function hash to the name of COMDAT function to avoid "
124 "function hash mismatch due to the preinliner"));
153 enum VisitMode { VM_counting, VM_instrument, VM_annotate };
157 struct SelectInstVisitor :
public InstVisitor<SelectInstVisitor> {
161 unsigned *CurCtrIdx =
nullptr;
162 unsigned TotalNumCtrs = 0;
164 uint64_t FuncHash = 0;
165 PGOUseFunc *UseFunc =
nullptr;
177 void instrumentSelects(
Function &
Func,
unsigned *Ind,
unsigned TotalNC,
179 Mode = VM_instrument;
181 TotalNumCtrs = TotalNC;
188 void annotateSelects(
Function &
Func, PGOUseFunc *UF,
unsigned *Ind) {
199 unsigned getNumOfSelectInsts()
const {
return NSIs; }
202 class PGOInstrumentationGenLegacyPass :
public ModulePass {
211 StringRef getPassName()
const override {
return "PGOInstrumentationGenPass"; }
214 bool runOnModule(
Module &M)
override;
221 class PGOInstrumentationUseLegacyPass :
public ModulePass {
226 PGOInstrumentationUseLegacyPass(std::string Filename =
"")
227 :
ModulePass(
ID), ProfileFileName(std::move(Filename)) {
234 StringRef getPassName()
const override {
return "PGOInstrumentationUsePass"; }
237 std::string ProfileFileName;
239 bool runOnModule(
Module &M)
override;
249 "PGO instrumentation.",
false,
false)
256 return new PGOInstrumentationGenLegacyPass();
261 "Read PGO instrumentation profile.",
false,
false)
268 return new PGOInstrumentationUseLegacyPass(Filename.str());
286 : SrcBB(Src), DestBB(Dest), Weight(W), InMST(
false), Removed(
false),
289 const std::string infoString()
const {
290 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
291 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight)).str();
301 BBInfo(
unsigned IX) : Group(this), Index(IX), Rank(0) {}
304 const std::string infoString()
const {
305 return (
Twine(
"Index=") +
Twine(Index)).str();
310 template <
class Edge,
class BBInfo>
class FuncPGOInstrumentation {
313 void computeCFGHash();
314 void renameComdatFunction();
316 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
319 std::vector<Instruction *> IndirectCallSites;
320 SelectInstVisitor SIVisitor;
321 std::string FuncName;
324 uint64_t FunctionHash;
334 BBInfo &getBBInfo(
const BasicBlock *BB)
const {
return MST.getBBInfo(BB); }
337 BBInfo *findBBInfo(
const BasicBlock *BB)
const {
return MST.findBBInfo(BB); }
340 void dumpInfo(std::string Str =
"")
const {
341 MST.dumpEdges(
dbgs(),
Twine(
"Dump Function ") + FuncName +
" Hash: " +
342 Twine(FunctionHash) +
"\t" + Str);
345 FuncPGOInstrumentation(
347 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
350 :
F(Func), ComdatMembers(ComdatMembers), SIVisitor(Func), FunctionHash(0),
354 SIVisitor.countSelects(Func);
355 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
360 if (ComdatMembers.size())
361 renameComdatFunction();
362 DEBUG(dumpInfo(
"after CFGMST"));
364 NumOfPGOBB += MST.BBInfos.size();
365 for (
auto &
E : MST.AllEdges) {
370 NumOfPGOInstrument++;
378 unsigned getNumCounters() {
379 unsigned NumCounters = 0;
380 for (
auto &
E : this->MST.AllEdges) {
381 if (!
E->InMST && !
E->Removed)
384 return NumCounters + SIVisitor.getNumOfSelectInsts();
390 template <
class Edge,
class BBInfo>
391 void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
392 std::vector<char> Indexes;
398 auto BI = findBBInfo(Succ);
402 for (
int J = 0; J < 4; J++)
403 Indexes.push_back((
char)(Index >> (J * 8)));
407 FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
408 (uint64_t)IndirectCallSites.size() << 48 |
409 (uint64_t)MST.AllEdges.size() << 32 | JC.
getCRC();
413 static bool canRenameComdat(
415 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
427 for (
auto &&CM :
make_range(ComdatMembers.equal_range(C))) {
428 if (dyn_cast<GlobalAlias>(CM.second))
438 template <
class Edge,
class BBInfo>
439 void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
440 if (!canRenameComdat(F, ComdatMembers))
443 std::string NewFuncName =
447 FuncName =
Twine(FuncName +
"." +
Twine(FunctionHash)).
str();
463 std::string NewComdatName =
468 for (
auto &&CM :
make_range(ComdatMembers.equal_range(OrigComdat))) {
469 if (
GlobalAlias *GA = dyn_cast<GlobalAlias>(CM.second)) {
471 assert(dyn_cast<Function>(GA->getAliasee()->stripPointerCasts()) == &F);
472 std::string OrigGAName = GA->getName().str();
473 GA->setName(
Twine(GA->getName() +
"." +
Twine(FunctionHash)));
486 template <
class Edge,
class BBInfo>
487 BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *
E) {
488 if (E->InMST || E->Removed)
494 if (SrcBB ==
nullptr)
496 if (DestBB ==
nullptr)
510 DEBUG(
dbgs() <<
"Split critical edge: " << getBBInfo(SrcBB).Index <<
" --> "
511 << getBBInfo(DestBB).Index <<
"\n");
514 assert(InstrBB &&
"Critical edge is not split");
522 static void instrumentOneFunc(
524 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
525 FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, ComdatMembers,
true, BPI,
527 unsigned NumCounters = FuncInfo.getNumCounters();
531 for (
auto &E : FuncInfo.MST.AllEdges) {
532 BasicBlock *InstrBB = FuncInfo.getInstrBB(E.get());
537 assert(Builder.GetInsertPoint() != InstrBB->
end() &&
538 "Cannot get the Instrumentation point");
542 Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters),
543 Builder.getInt32(I++)});
547 FuncInfo.SIVisitor.instrumentSelects(F, &I, NumCounters, FuncInfo.FuncNameVar,
548 FuncInfo.FunctionHash);
554 unsigned NumIndirectCallSites = 0;
555 for (
auto &I : FuncInfo.IndirectCallSites) {
557 Value *Callee =
CS.getCalledValue();
558 DEBUG(
dbgs() <<
"Instrument one indirect call: CallSite Index = "
559 << NumIndirectCallSites <<
"\n");
561 assert(Builder.GetInsertPoint() != I->getParent()->end() &&
562 "Cannot get the Instrumentation point");
566 Builder.getInt64(FuncInfo.FunctionHash),
567 Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()),
568 Builder.getInt32(llvm::InstrProfValueKind::IPVK_IndirectCallTarget),
569 Builder.getInt32(NumIndirectCallSites++)});
571 NumOfPGOICall += NumIndirectCallSites;
575 struct PGOUseEdge :
public PGOEdge {
579 : PGOEdge(Src, Dest, W), CountValid(
false), CountValue(0) {}
582 void setEdgeCount(uint64_t
Value) {
588 const std::string infoString()
const {
590 return PGOEdge::infoString();
591 return (
Twine(PGOEdge::infoString()) +
" Count=" +
Twine(CountValue))
599 struct UseBBInfo :
public BBInfo {
602 int32_t UnknownCountInEdge;
603 int32_t UnknownCountOutEdge;
605 DirectEdges OutEdges;
606 UseBBInfo(
unsigned IX)
607 : BBInfo(IX), CountValue(0), CountValid(
false), UnknownCountInEdge(0),
608 UnknownCountOutEdge(0) {}
609 UseBBInfo(
unsigned IX, uint64_t C)
610 : BBInfo(IX), CountValue(C), CountValid(
true), UnknownCountInEdge(0),
611 UnknownCountOutEdge(0) {}
614 void setBBInfoCount(uint64_t
Value) {
620 const std::string infoString()
const {
622 return BBInfo::infoString();
623 return (
Twine(BBInfo::infoString()) +
" Count=" +
Twine(CountValue)).str();
630 for (
auto &E : Edges) {
633 Total += E->CountValue;
641 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
644 : F(Func), M(Modu), FuncInfo(Func, ComdatMembers,
false, BPI, BFI),
645 CountPosition(0), ProfileCountSize(0), FreqAttr(FFA_Normal) {}
651 void populateCounters();
654 void setBranchWeights();
657 void annotateIndirectCallSites();
660 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
663 FuncFreqAttr getFuncFreqAttr()
const {
return FreqAttr; }
666 uint64_t getFuncHash()
const {
return FuncInfo.FunctionHash; }
671 UseBBInfo &getBBInfo(
const BasicBlock *BB)
const {
672 return FuncInfo.getBBInfo(BB);
676 UseBBInfo *findBBInfo(
const BasicBlock *BB)
const {
677 return FuncInfo.findBBInfo(BB);
684 FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
688 uint64_t ProgramMaxCount;
700 FuncFreqAttr FreqAttr;
703 void setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile);
707 void setEdgeCount(DirectEdges &Edges, uint64_t
Value);
710 const std::string getFuncName()
const {
return FuncInfo.FuncName; }
715 void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {
716 if (ProgramMaxCount == 0)
722 if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount))
724 else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount))
731 void PGOUseFunc::setInstrumentedCounts(
732 const std::vector<uint64_t> &CountFromProfile) {
734 assert(FuncInfo.getNumCounters() == CountFromProfile.size());
736 std::vector<PGOUseEdge *> WorkList;
737 for (
auto &E : FuncInfo.MST.AllEdges)
738 WorkList.push_back(E.get());
741 for (
auto &E : WorkList) {
745 uint64_t CountValue = CountFromProfile[I++];
747 getBBInfo(InstrBB).setBBInfoCount(CountValue);
748 E->setEdgeCount(CountValue);
756 PGOUseEdge &NewEdge = FuncInfo.MST.addEdge(SrcBB, InstrBB, 0);
757 NewEdge.setEdgeCount(CountValue);
759 PGOUseEdge &NewEdge1 = FuncInfo.MST.addEdge(InstrBB, DestBB, 0);
760 NewEdge1.setEdgeCount(CountValue);
761 NewEdge1.InMST =
true;
762 getBBInfo(InstrBB).setBBInfoCount(CountValue);
764 ProfileCountSize = CountFromProfile.size();
770 void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t
Value) {
771 for (
auto &E : Edges) {
774 E->setEdgeCount(Value);
776 getBBInfo(E->SrcBB).UnknownCountOutEdge--;
777 getBBInfo(E->DestBB).UnknownCountInEdge--;
792 auto Err = IPE.
get();
793 bool SkipWarning =
false;
816 ProfileRecord = std::move(Result.
get());
817 std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
820 DEBUG(
dbgs() << CountFromProfile.size() <<
" counts\n");
821 uint64_t ValueSum = 0;
822 for (
unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
823 DEBUG(
dbgs() <<
" " << I <<
": " << CountFromProfile[I] <<
"\n");
824 ValueSum += CountFromProfile[
I];
827 DEBUG(
dbgs() <<
"SUM = " << ValueSum <<
"\n");
829 getBBInfo(
nullptr).UnknownCountOutEdge = 2;
830 getBBInfo(
nullptr).UnknownCountInEdge = 2;
832 setInstrumentedCounts(CountFromProfile);
839 void PGOUseFunc::populateCounters() {
841 for (
auto &E : FuncInfo.MST.AllEdges) {
847 UseBBInfo &SrcInfo = getBBInfo(SrcBB);
848 UseBBInfo &DestInfo = getBBInfo(DestBB);
849 SrcInfo.OutEdges.push_back(E.get());
850 DestInfo.InEdges.push_back(E.get());
851 SrcInfo.UnknownCountOutEdge++;
852 DestInfo.UnknownCountInEdge++;
856 DestInfo.UnknownCountInEdge--;
857 SrcInfo.UnknownCountOutEdge--;
861 unsigned NumPasses = 0;
869 UseBBInfo *Count = findBBInfo(&BB);
870 if (Count ==
nullptr)
872 if (!Count->CountValid) {
873 if (Count->UnknownCountOutEdge == 0) {
874 Count->CountValue = sumEdgeCount(Count->OutEdges);
875 Count->CountValid =
true;
877 }
else if (Count->UnknownCountInEdge == 0) {
878 Count->CountValue = sumEdgeCount(Count->InEdges);
879 Count->CountValid =
true;
883 if (Count->CountValid) {
884 if (Count->UnknownCountOutEdge == 1) {
886 uint64_t OutSum = sumEdgeCount(Count->OutEdges);
890 if (Count->CountValue > OutSum)
891 Total = Count->CountValue - OutSum;
892 setEdgeCount(Count->OutEdges, Total);
895 if (Count->UnknownCountInEdge == 1) {
897 uint64_t InSum = sumEdgeCount(Count->InEdges);
898 if (Count->CountValue > InSum)
899 Total = Count->CountValue - InSum;
900 setEdgeCount(Count->InEdges, Total);
907 DEBUG(
dbgs() <<
"Populate counts in " << NumPasses <<
" passes.\n");
911 auto BI = findBBInfo(&BB);
914 assert(BI->CountValid &&
"BB count is not valid");
917 uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
918 F.setEntryCount(FuncEntryCount);
919 uint64_t FuncMaxCount = FuncEntryCount;
921 auto BI = findBBInfo(&BB);
924 FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
926 markFunctionAttributes(FuncEntryCount, FuncMaxCount);
929 FuncInfo.SIVisitor.annotateSelects(F,
this, &CountPosition);
930 assert(CountPosition == ProfileCountSize);
932 DEBUG(FuncInfo.dumpInfo(
"after reading profile."));
938 assert(MaxCount > 0 &&
"Bad max count");
941 for (
const auto &ECI : EdgeCounts)
945 for (
const auto &W : Weights) {
dbgs() << W <<
" "; }
951 void PGOUseFunc::setBranchWeights() {
953 DEBUG(
dbgs() <<
"\nSetting branch weights.\n");
958 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
960 if (getBBInfo(&BB).CountValue == 0)
964 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
965 unsigned Size = BBCountInfo.OutEdges.size();
967 uint64_t MaxCount = 0;
968 for (
unsigned s = 0; s < Size; s++) {
969 const PGOUseEdge *E = BBCountInfo.OutEdges[s];
972 if (DestBB ==
nullptr)
975 uint64_t EdgeCount = E->CountValue;
976 if (EdgeCount > MaxCount)
977 MaxCount = EdgeCount;
978 EdgeCounts[SuccNum] = EdgeCount;
980 setProfMetadata(M, TI, EdgeCounts, MaxCount);
984 void SelectInstVisitor::instrumentOneSelectInst(
SelectInst &SI) {
985 Module *M = F.getParent();
989 auto *Step = Builder.CreateZExt(SI.
getCondition(), Int64Ty);
993 Builder.getInt64(FuncHash),
994 Builder.getInt32(TotalNumCtrs), Builder.getInt32(*CurCtrIdx), Step});
998 void SelectInstVisitor::annotateOneSelectInst(
SelectInst &SI) {
999 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1000 assert(*CurCtrIdx < CountFromProfile.size() &&
1001 "Out of bound access of counters");
1002 uint64_t SCounts[2];
1003 SCounts[0] = CountFromProfile[*CurCtrIdx];
1005 uint64_t TotalCount = 0;
1006 auto BI = UseFunc->findBBInfo(SI.
getParent());
1008 TotalCount = BI->CountValue;
1010 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1011 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1013 setProfMetadata(F.getParent(), &SI, SCounts, MaxCount);
1016 void SelectInstVisitor::visitSelectInst(
SelectInst &SI) {
1028 instrumentOneSelectInst(SI);
1031 annotateOneSelectInst(SI);
1039 void PGOUseFunc::annotateIndirectCallSites() {
1046 unsigned IndirectCallSiteIndex = 0;
1047 auto &IndirectCallSites = FuncInfo.IndirectCallSites;
1048 unsigned NumValueSites =
1049 ProfileRecord.getNumValueSites(IPVK_IndirectCallTarget);
1050 if (NumValueSites != IndirectCallSites.size()) {
1052 std::string(
"Inconsistent number of indirect call sites: ") +
1060 for (
auto &I : IndirectCallSites) {
1061 DEBUG(
dbgs() <<
"Read one indirect call instrumentation: Index="
1062 << IndirectCallSiteIndex <<
" out of " << NumValueSites
1066 IndirectCallSiteIndex++;
1075 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
1079 INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1082 if (!TT.supportsCOMDAT())
1086 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR))));
1093 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1097 if (
Comdat *C = F.getComdat())
1098 ComdatMembers.insert(std::make_pair(C, &F));
1100 if (
Comdat *C = GV.getComdat())
1101 ComdatMembers.insert(std::make_pair(C, &GV));
1103 if (
Comdat *C = GA.getComdat())
1104 ComdatMembers.insert(std::make_pair(C, &GA));
1111 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1115 if (F.isDeclaration())
1117 auto *BPI = LookupBPI(F);
1118 auto *BFI = LookupBFI(F);
1119 instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers);
1124 bool PGOInstrumentationGenLegacyPass::runOnModule(
Module &M) {
1129 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(
F).getBPI();
1132 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(
F).getBFI();
1159 DEBUG(
dbgs() <<
"Read in profile counters: ");
1163 if (
Error E = ReaderOrErr.takeError()) {
1171 std::unique_ptr<IndexedInstrProfReader> PGOReader =
1172 std::move(ReaderOrErr.get());
1179 if (!PGOReader->isIRLevelProfile()) {
1181 ProfileFileName.
data(),
"Not an IR level instrumentation profile"));
1185 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1187 std::vector<Function *> HotFunctions;
1188 std::vector<Function *> ColdFunctions;
1190 if (F.isDeclaration())
1192 auto *BPI = LookupBPI(F);
1193 auto *BFI = LookupBFI(F);
1194 PGOUseFunc
Func(F, &M, ComdatMembers, BPI, BFI);
1195 if (!Func.readCounters(PGOReader.get()))
1197 Func.populateCounters();
1198 Func.setBranchWeights();
1199 Func.annotateIndirectCallSites();
1200 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
1201 if (FreqAttr == PGOUseFunc::FFA_Cold)
1202 ColdFunctions.push_back(&F);
1203 else if (FreqAttr == PGOUseFunc::FFA_Hot)
1204 HotFunctions.push_back(&F);
1206 M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext()));
1211 for (
auto &F : HotFunctions) {
1212 F->addFnAttr(llvm::Attribute::InlineHint);
1213 DEBUG(
dbgs() <<
"Set inline attribute to function: " << F->getName()
1216 for (
auto &F : ColdFunctions) {
1218 DEBUG(
dbgs() <<
"Set cold attribute to function: " << F->getName() <<
"\n");
1224 : ProfileFileName(std::move(Filename)) {
1247 bool PGOInstrumentationUseLegacyPass::runOnModule(
Module &M) {
1252 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(
F).getBPI();
1255 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(
F).getBFI();
void push_back(const T &Elt)
LinkageTypes getLinkage() const
ModulePass * createPGOInstrumentationGenLegacyPass()
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Base class for instruction visitors.
STATISTIC(NumFunctions,"Total number of functions")
A Module instance is used to store all the information related to an LLVM module. ...
Diagnostic information for the PGO profiler.
Available for inspection, not emission.
static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
An efficient, type-erasing, non-owning reference to a callable.
Externally visible function.
uint64_t getMaximumFunctionCount()
Return the maximum of all known function counts.
static IntegerType * getInt64Ty(LLVMContext &C)
Error takeError()
Take ownership of the stored error.
Base class for error info classes.
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
An union-find based Minimum Spanning Tree for CFG.
StringRef getName() const
Return a constant reference to the value's name.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
std::string str() const
Return the twine contents as a std::string.
static cl::opt< bool > PGOWarnMissing("pgo-warn-missing-function", cl::init(false), cl::Hidden)
This class represents the LLVM 'select' instruction.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
PGOInstrumentationUse(std::string Filename="")
StringRef getName() const
Get a short "name" for the module.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Legacy analysis pass which computes BlockFrequencyInfo.
VisitMode
The select instruction visitor plays three roles specified by the mode.
void setName(const Twine &Name)
Change the name of the value.
Tagged union holding either a T or a Error.
StringRef getName() const
Analysis pass which computes BranchProbabilityInfo.
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
static cl::opt< bool > NoPGOWarnMismatchComdat("no-pgo-warn-mismatch-comdat", cl::init(true), cl::Hidden)
This file provides the interface for IR based instrumentation passes ( (profile-gen, and profile-use).
INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass,"pgo-instr-gen","PGO instrumentation.", false, false) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass
static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static void createIRLevelProfileFlagVariable(Module &M)
static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of annotations for a single indirect ""call callsite"))
void setComdat(Comdat *C)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
void initializePGOInstrumentationGenLegacyPassPass(PassRegistry &)
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
static bool InstrumentAllFunctions(Module &M, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI)
Legacy analysis pass which computes BranchProbabilityInfo.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is""mainly for test purpose."))
std::string message() const override
Return the error message as a string.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
BasicBlock * SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions())
If this edge is a critical edge, insert a new node to split the critical edge.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Same, but only replaced by something equivalent.
initializer< Ty > init(const Ty &Val)
Subclasses of this class are all able to terminate a basic block.
A set of analyses that are preserved following a run of a transformation pass.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
bool isVectorTy() const
True if this is an instance of VectorType.
const Comdat * getComdat() const
const Value * getCondition() const
Represent the analysis usage information of a pass.
ModulePass * createPGOInstrumentationUseLegacyPass(StringRef Filename=StringRef(""))
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
SelectionKind getSelectionKind() const
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
void handleAllErrors(Error E, HandlerTs &&...Handlers)
Behaves the same as handleErrors, except that it requires that all errors be handled by the given han...
void initializePGOInstrumentationUseLegacyPassPass(PassRegistry &)
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Comdat * getOrInsertComdat(StringRef Name)
Return the Comdat in the module with the specified name.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Triple - Helper class for working with autoconf configuration names.
static cl::opt< bool > NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false), cl::Hidden)
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Analysis pass which computes BlockFrequencyInfo.
void setSelectionKind(SelectionKind Val)
instrprof_error get() const
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
pgo instr PGO instrumentation
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
reference get()
Returns a reference to the stored T value.
void setLinkage(LinkageTypes LT)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Class for arbitrary precision integers.
static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden)
pgo instr Read PGO instrumentation profile
Analysis providing branch probability information.
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
Profiling information for a single function.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash)
Return the pointer to InstrProfRecord associated with FuncName and FuncHash.
static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI)
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
static std::vector< Instruction * > findIndirectCallSites(Function &F)
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Keep one copy of named function when linking (weak)
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid ""function hash mismatch due to the preinliner"))
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path)
Factory method to create an indexed reader.
void update(ArrayRef< char > Data)
Lightweight error class with error context and mandatory checking.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
virtual std::string message() const
Return the error message as a string.
StringRef - Represent a constant reference to a string, i.e.
Reader for the indexed binary instrprof format.
A container for analyses that lazily runs them and caches their results.
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getParent() const
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
LLVMContext & getContext() const
Get the global data context.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...