52 #define DEBUG_TYPE "pgo-icall-prom"
54 STATISTIC(NumOfPGOICallPromotion,
"Number of indirect call promotions.");
55 STATISTIC(NumOfPGOICallsites,
"Number of indirect call candidate sites.");
60 cl::desc(
"Disable indirect call promotion"));
68 cl::desc(
"Max number of promotions for this compilaiton"));
74 cl::desc(
"Skip Callsite up to this number for this compilaiton"));
80 cl::desc(
"Run indirect-call promotion in LTO "
87 cl::desc(
"Run indirect-call promotion for call instructions "
94 cl::desc(
"Run indirect-call promotion for "
95 "invoke instruction only"));
101 cl::desc(
"Dump IR after transformation happens"));
104 class PGOIndirectCallPromotionLegacyPass :
public ModulePass {
108 PGOIndirectCallPromotionLegacyPass(
bool InLTO =
false)
114 StringRef getPassName()
const override {
return "PGOIndirectCallPromotion"; }
117 bool runOnModule(
Module &M)
override;
127 "Use PGO instrumentation profile to promote indirect calls to "
132 return new PGOIndirectCallPromotionLegacyPass(InLTO);
138 class ICallPromotionFunc {
149 NotAvailableInModule,
160 struct PromotionCandidate {
163 PromotionCandidate(
Function *
F, uint64_t
C) : TargetFunction(F), Count(C) {}
171 std::vector<PromotionCandidate> getPromotionCandidatesForCallSite(
173 uint64_t TotalCount,
uint32_t NumCandidates);
186 uint64_t TotalCount);
191 const std::vector<PromotionCandidate> &Candidates,
192 uint64_t &TotalCount);
194 static const char *StatusToString(
const TargetStatus S) {
197 return "OK to promote";
198 case NotAvailableInModule:
199 return "Cannot find the target";
200 case ReturnTypeMismatch:
201 return "Return type mismatch";
202 case NumArgsMismatch:
203 return "The number of arguments mismatch";
204 case ArgTypeMismatch:
205 return "Argument Type mismatch";
211 ICallPromotionFunc(
const ICallPromotionFunc &other) =
delete;
212 ICallPromotionFunc &operator=(
const ICallPromotionFunc &other) =
delete;
216 :
F(Func), M(Modu), Symtab(Symtab) {
219 bool processFunction();
223 ICallPromotionFunc::TargetStatus
226 Function *DirectCallee = Symtab->getFunction(Target);
227 if (DirectCallee ==
nullptr)
228 return NotAvailableInModule;
233 if (FuncRetType != CallRetType &&
235 return ReturnTypeMismatch;
242 unsigned ArgNum =
CS.arg_size();
244 if (ParamNum != ArgNum && !DirectCalleeType->
isVarArg())
245 return NumArgsMismatch;
247 for (
unsigned I = 0;
I < ParamNum; ++
I) {
249 Type *ATy =
CS.getArgument(
I)->getType();
253 return ArgTypeMismatch;
256 DEBUG(
dbgs() <<
" #" << NumOfPGOICallPromotion <<
" Promote the icall to "
257 << Symtab->getFuncName(Target) <<
"\n");
258 TargetFunction = DirectCallee;
264 std::vector<ICallPromotionFunc::PromotionCandidate>
265 ICallPromotionFunc::getPromotionCandidatesForCallSite(
267 uint64_t TotalCount,
uint32_t NumCandidates) {
268 std::vector<PromotionCandidate>
Ret;
270 DEBUG(
dbgs() <<
" \nWork on callsite #" << NumOfPGOICallsites << *Inst
271 <<
" Num_targets: " << ValueDataRef.
size()
272 <<
" Num_candidates: " << NumCandidates <<
"\n");
273 NumOfPGOICallsites++;
275 DEBUG(
dbgs() <<
" Skip: User options.\n");
280 uint64_t Count = ValueDataRef[
I].Count;
281 assert(Count <= TotalCount);
282 uint64_t Target = ValueDataRef[
I].Value;
283 DEBUG(
dbgs() <<
" Candidate " <<
I <<
" Count=" << Count
284 <<
" Target_func: " << Target <<
"\n");
287 DEBUG(
dbgs() <<
" Not promote: User options.\n");
291 DEBUG(
dbgs() <<
" Not promote: User option.\n");
295 DEBUG(
dbgs() <<
" Not promote: Cutoff reached.\n");
299 TargetStatus
Status = isPromotionLegal(Inst, Target, TargetFunction);
301 StringRef TargetFuncName = Symtab->getFuncName(Target);
302 const char *Reason = StatusToString(Status);
303 DEBUG(
dbgs() <<
" Not promote: " << Reason <<
"\n");
306 Twine(
"Cannot promote indirect call to ") +
308 Twine(
" with count of ") +
Twine(Count) +
": " + Reason);
311 Ret.push_back(PromotionCandidate(TargetFunction, Count));
320 uint64_t Count, uint64_t TotalCount,
335 uint64_t ElseCount = TotalCount - Count;
336 uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount);
339 MDNode *BranchWeights = MDB.createBranchWeights(
345 (*DirectCallBB)->
setName(
"if.true.direct_targ");
347 (*IndirectCallBB)->
setName(
"if.false.orig_indirect");
349 (*MergeBB)->
setName(
"if.end.icp");
368 for (
auto &
I : *BB) {
376 if (dyn_cast<Instruction>(V) == Inst)
392 for (
auto &
I : *BB) {
414 for (
auto &
I : *BB) {
422 if (dyn_cast<Instruction>(V) == Inst) {
436 return DirectCallInst;
440 if (FuncRetType == CallRetType)
441 return DirectCallInst;
444 if (
CallInst *CI = dyn_cast<CallInst>(DirectCallInst))
449 return (
new BitCastInst(DirectCallInst, CallRetType,
"",
466 if (
CallInst *CI = dyn_cast<CallInst>(NewInst)) {
467 CI->setCalledFunction(DirectCallee);
489 for (
unsigned I = 0;
I < ParamNum; ++
I) {
510 if (
InvokeInst *II = dyn_cast<InvokeInst>(CallResult))
511 RetValBB = II->getNormalDest();
536 uint64_t Count, uint64_t TotalCount) {
537 assert(DirectCallee !=
nullptr);
541 DEBUG(
dbgs() <<
"\n\n== Basic Block Before ==\n");
544 BasicBlock *DirectCallBB, *IndirectCallBB, *MergeBB;
546 &IndirectCallBB, &MergeBB);
556 if (
InvokeInst *II = dyn_cast<InvokeInst>(Inst)) {
570 IndirectCallBB, NewInst);
575 DEBUG(
dbgs() <<
"\n== Basic Blocks After ==\n");
576 DEBUG(
dbgs() << *BB << *DirectCallBB << *IndirectCallBB << *MergeBB <<
"\n");
580 Twine(
"Promote indirect call to ") + DirectCallee->
getName() +
581 " with count " +
Twine(Count) +
" out of " +
Twine(TotalCount));
585 uint32_t ICallPromotionFunc::tryToPromote(
586 Instruction *Inst,
const std::vector<PromotionCandidate> &Candidates,
587 uint64_t &TotalCount) {
590 for (
auto &
C : Candidates) {
591 uint64_t Count =
C.Count;
592 promote(Inst,
C.TargetFunction, Count, TotalCount);
593 assert(TotalCount >= Count);
595 NumOfPGOICallPromotion++;
603 bool ICallPromotionFunc::processFunction() {
604 bool Changed =
false;
610 I, NumVals, TotalCount, NumCandidates);
613 auto PromotionCandidates = getPromotionCandidatesForCallSite(
614 I, ICallProfDataRef, TotalCount, NumCandidates);
615 uint32_t NumPromoted = tryToPromote(
I, PromotionCandidates, TotalCount);
616 if (NumPromoted == 0)
623 if (TotalCount == 0 || NumPromoted == NumVals)
627 IPVK_IndirectCallTarget, NumCandidates);
638 bool Changed =
false;
640 if (
F.isDeclaration())
642 if (
F.hasFnAttribute(Attribute::OptimizeNone))
644 ICallPromotionFunc ICallPromotion(
F, &M, &Symtab);
645 bool FuncChanged = ICallPromotion.processFunction();
650 Changed |= FuncChanged;
652 DEBUG(
dbgs() <<
" Stop: Cutoff reached.\n");
659 bool PGOIndirectCallPromotionLegacyPass::runOnModule(
Module &M) {
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore, TerminatorInst **ThenTerm, TerminatorInst **ElseTerm, MDNode *BranchWeights=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
STATISTIC(NumFunctions,"Total number of functions")
void setCalledFunction(Value *Fn)
Set the function called.
A Module instance is used to store all the information related to an LLVM module. ...
This class represents a function call, abstracting a target machine's calling convention.
Type * getReturnType() const
Returns the type of the ret val.
const Function * getParent() const
Return the enclosing method, or null if none.
void emitOptimizationRemark(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg)
Emit an optimization-applied message.
Type * getFunctionParamType(unsigned i) const
StringRef getName() const
Return a constant reference to the value's name.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
static bool castIsValid(Instruction::CastOps op, Value *S, Type *DstTy)
This method can be used to determine if a cast from S to DstTy using Opcode op is valid or not...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void setName(const Twine &Name)
Change the name of the value.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
void setArgument(unsigned ArgNo, Value *newVal)
Class to represent function types.
This file provides the interface for IR based instrumentation passes ( (profile-gen, and profile-use).
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
void emitOptimizationRemarkMissed(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg)
Emit an optimization-missed message.
void setNormalDest(BasicBlock *B)
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
size_t size() const
size - Get the array size.
BasicBlock * getNormalDest() const
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Type * getParamType(unsigned i) const
Parameter type accessors.
initializer< Ty > init(const Ty &Val)
Subclasses of this class are all able to terminate a basic block.
A set of analyses that are preserved following a run of a transformation pass.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
void push_front(pointer val)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
const InstListType & getInstList() const
Return the underlying instruction list container.
ModulePass * createPGOIndirectCallPromotionLegacyPass(bool InLTO=false)
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
LLVMContext & getContext() const
All values hold a context through their type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setIncomingBlock(unsigned i, BasicBlock *BB)
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
ValTy * getArgument(unsigned ArgNo) const
Type * getType() const
All values are typed, get the type of this value.
void mutateFunctionType(FunctionType *FTy)
void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry &)
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Target - Wrapper for Target specific information.
BasicBlock * getSingleSuccessor()
Return the successor of this block if it has a single successor.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
unsigned getFunctionNumParams() const
iterator insert(iterator where, pointer New)
static std::vector< Instruction * > findIndirectCallSites(Function &F)
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
FunctionType * getFunctionType() const
Returns the FunctionType for me.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
StringRef - Represent a constant reference to a string, i.e.
A container for analyses that lazily runs them and caches their results.
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
This header defines various interfaces for pass management in LLVM.
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
const BasicBlock * getParent() const
bool isVoidTy() const
Return true if this is 'void'.