49#define DEBUG_TYPE "pgo-icall-prom"
51STATISTIC(NumOfPGOICallPromotion,
"Number of indirect call promotions.");
52STATISTIC(NumOfPGOICallsites,
"Number of indirect call candidate sites.");
57 cl::desc(
"Disable indirect call promotion"));
65 cl::desc(
"Max number of promotions for this compilation"));
71 cl::desc(
"Skip Callsite up to this number for this compilation"));
77 cl::desc(
"Run indirect-call promotion in LTO "
84 cl::desc(
"Run indirect-call promotion in SamplePGO mode"));
90 cl::desc(
"Run indirect-call promotion for call instructions "
97 cl::desc(
"Run indirect-call promotion for "
98 "invoke instruction only"));
104 cl::desc(
"Dump IR after transformation happens"));
110class IndirectCallPromoter {
118 const bool SamplePGO;
123 struct PromotionCandidate {
135 std::vector<PromotionCandidate> getPromotionCandidatesForCallSite(
142 const std::vector<PromotionCandidate> &Candidates,
148 :
F(
Func), Symtab(Symtab), SamplePGO(SamplePGO), ORE(ORE) {}
149 IndirectCallPromoter(
const IndirectCallPromoter &) =
delete;
150 IndirectCallPromoter &operator=(
const IndirectCallPromoter &) =
delete;
159std::vector<IndirectCallPromoter::PromotionCandidate>
160IndirectCallPromoter::getPromotionCandidatesForCallSite(
163 std::vector<PromotionCandidate>
Ret;
165 LLVM_DEBUG(
dbgs() <<
" \nWork on callsite #" << NumOfPGOICallsites << CB
166 <<
" Num_targets: " << ValueDataRef.
size()
167 <<
" Num_candidates: " << NumCandidates <<
"\n");
168 NumOfPGOICallsites++;
176 assert(Count <= TotalCount);
180 <<
" Target_func: " <<
Target <<
"\n");
186 <<
" Not promote: User options";
194 <<
" Not promote: User options";
202 <<
" Not promote: Cutoff reached";
215 if (TargetFunction ==
nullptr || TargetFunction->
isDeclaration()) {
219 <<
"Cannot promote indirect call: target with md5sum "
225 const char *Reason =
nullptr;
231 <<
"Cannot promote indirect call to "
232 <<
NV(
"TargetFunction", TargetFunction) <<
" with count of "
233 <<
NV(
"Count", Count) <<
": " << Reason;
238 Ret.push_back(PromotionCandidate(TargetFunction, Count));
246 bool AttachProfToDirectCall,
249 uint64_t ElseCount = TotalCount - Count;
250 uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount);
259 if (AttachProfToDirectCall) {
268 <<
"Promote indirect call to " << NV(
"DirectCallee", DirectCallee)
269 <<
" with count " << NV(
"Count", Count) <<
" out of "
270 << NV(
"TotalCount", TotalCount);
276uint32_t IndirectCallPromoter::tryToPromote(
277 CallBase &CB,
const std::vector<PromotionCandidate> &Candidates,
281 for (
const auto &
C : Candidates) {
285 assert(TotalCount >= Count);
287 NumOfPGOICallPromotion++;
296 bool Changed =
false;
302 CB, NumVals, TotalCount, NumCandidates);
303 if (!NumCandidates ||
306 auto PromotionCandidates = getPromotionCandidatesForCallSite(
307 *CB, ICallProfDataRef, TotalCount, NumCandidates);
308 uint32_t NumPromoted = tryToPromote(*CB, PromotionCandidates, TotalCount);
309 if (NumPromoted == 0)
316 if (TotalCount == 0 || NumPromoted == NumVals)
320 TotalCount, IPVK_IndirectCallTarget, NumCandidates);
332 std::string SymtabFailure =
toString(std::move(E));
333 M.getContext().emitError(
"Failed to create symtab: " + SymtabFailure);
336 bool Changed =
false;
338 if (
F.isDeclaration() ||
F.hasOptNone())
345 IndirectCallPromoter CallPromoter(
F, &Symtab, SamplePGO, ORE);
346 bool FuncChanged = CallPromoter.processFunction(PSI);
351 Changed |= FuncChanged;
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file contains the declarations for profiling metadata utility functions.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Lightweight error class with error context and mandatory checking.
const Function & getFunction() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
A Module instance is used to store all the information related to an LLVM module.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
Target - Wrapper for Target specific information.
LLVMContext & getContext() const
All values hold a context through their type.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
DiagnosticInfoOptimizationBase::Argument NV
CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
std::vector< CallBase * > findIndirectCalls(Function &F)
CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)
Promote the given indirect call site to conditionally call Callee.
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.