29#define DEBUG_TYPE "call-promotion-utils"
59 int Idx = Phi.getBasicBlockIndex(OrigBlock);
62 Phi.setIncomingBlock(
Idx, MergeBlock);
91 int Idx = Phi.getBasicBlockIndex(OrigBlock);
94 auto *V = Phi.getIncomingValue(
Idx);
95 Phi.setIncomingBlock(
Idx, ThenBlock);
96 Phi.addIncoming(V, ElseBlock);
122 for (
User *U : UsersToUpdate)
123 U->replaceUsesOfWith(OrigInst, Phi);
124 Phi->addIncoming(OrigInst, OrigInst->
getParent());
125 Phi->addIncoming(NewInst, NewInst->
getParent());
177 if (
auto *Invoke = dyn_cast<InvokeInst>(&CB))
189 for (
User *U : UsersToUpdate)
190 U->replaceUsesOfWith(&CB, Cast);
300 ThenBlock->
setName(
"if.true.direct_targ");
305 Value *NewRetVal = NewInst;
307 if (
auto *BitCast = dyn_cast_or_null<BitCastInst>(Next)) {
308 assert(BitCast->getOperand(0) == OrigInst &&
309 "bitcast following musttail call must use the call");
310 auto NewBitCast = BitCast->clone();
311 NewBitCast->replaceUsesOfWith(OrigInst, NewInst);
312 NewBitCast->insertBefore(ThenTerm);
313 NewRetVal = NewBitCast;
314 Next = BitCast->getNextNode();
318 ReturnInst *Ret = dyn_cast_or_null<ReturnInst>(Next);
319 assert(Ret &&
"musttail call must precede a ret with an optional bitcast");
320 auto NewRet = Ret->clone();
321 if (Ret->getReturnValue())
322 NewRet->replaceUsesOfWith(Ret->getReturnValue(), NewRetVal);
323 NewRet->insertBefore(ThenTerm);
342 ThenBlock->
setName(
"if.true.direct_targ");
343 ElseBlock->
setName(
"if.false.orig_indirect");
344 MergeBlock->
setName(
"if.end.icp");
353 if (
auto *OrigInvoke = dyn_cast<InvokeInst>(OrigInst)) {
354 auto *NewInvoke = cast<InvokeInst>(NewInst);
363 Builder.
CreateBr(OrigInvoke->getNormalDest());
371 OrigInvoke->setNormalDest(MergeBlock);
372 NewInvoke->setNormalDest(MergeBlock);
398 const char **FailureReason) {
401 auto &
DL = Callee->getDataLayout();
406 Type *FuncRetTy = Callee->getReturnType();
407 if (CallRetTy != FuncRetTy)
410 *FailureReason =
"Return type mismatch";
415 unsigned NumParams = Callee->getFunctionType()->getNumParams();
422 if (NumArgs != NumParams && !Callee->isVarArg()) {
424 *FailureReason =
"The number of arguments mismatch";
432 for (;
I < NumParams; ++
I) {
435 if (Callee->hasParamAttribute(
I, Attribute::ByVal) !=
438 *FailureReason =
"byval mismatch";
441 if (Callee->hasParamAttribute(
I, Attribute::InAlloca) !=
444 *FailureReason =
"inalloca mismatch";
448 Type *FormalTy = Callee->getFunctionType()->getFunctionParamType(
I);
450 if (FormalTy == ActualTy)
454 *FailureReason =
"Argument type mismatch";
465 *FailureReason =
"Musttail call Argument type mismatch";
470 for (;
I < NumArgs;
I++) {
472 assert(Callee->isVarArg());
475 *FailureReason =
"SRet arg to vararg function";
504 Type *CalleeRetTy = Callee->getReturnType();
512 auto CalleeType = Callee->getFunctionType();
513 auto CalleeParamNum = CalleeType->getNumParams();
519 bool AttributeChanged =
false;
521 for (
unsigned ArgNo = 0; ArgNo < CalleeParamNum; ++ArgNo) {
523 Type *FormalTy = CalleeType->getParamType(ArgNo);
524 Type *ActualTy = Arg->getType();
525 if (FormalTy != ActualTy) {
537 ArgAttrs.
addByValAttr(Callee->getParamByValType(ArgNo));
542 AttributeChanged =
true;
551 if (!CallSiteRetTy->
isVoidTy() && CallSiteRetTy != CalleeRetTy) {
555 AttributeChanged =
true;
559 if (AttributeChanged)
588 const uint64_t CSIndex = CSInstr->getIndex()->getZExtValue();
592 CSInstr->moveBefore(&CB);
594 auto *NewCSInstr = cast<InstrProfCallsite>(CSInstr->clone());
595 NewCSInstr->setIndex(NewCSID);
596 NewCSInstr->setCallee(&Callee);
602 "The ICP direct BB is new, it shouldn't have instrumentation");
604 "The ICP indirect BB is new, it shouldn't have instrumentation");
611 auto *DirectBBIns = cast<InstrProfCntrInstBase>(EntryBBIns->clone());
612 DirectBBIns->setIndex(DirectID);
613 DirectBBIns->insertInto(&DirectBB, DirectBB.getFirstInsertionPt());
615 auto *IndirectBBIns = cast<InstrProfCntrInstBase>(EntryBBIns->clone());
616 IndirectBBIns->setIndex(IndirectID);
617 IndirectBBIns->insertInto(&IndirectBB, IndirectBB.getFirstInsertionPt());
620 const uint32_t NewCountersSize = IndirectID + 1;
624 assert(NewCountersSize - 2 == Ctx.counters().size());
626 Ctx.resizeCounters(NewCountersSize);
631 if (!Ctx.hasCallsite(CSIndex))
633 auto &CSData = Ctx.callsite(CSIndex);
636 for (
const auto &[
_, V] : CSData)
637 TotalCount += V.getEntrycount();
642 if (
auto It = CSData.find(CalleeGUID); It != CSData.end()) {
643 assert(CalleeGUID == It->second.guid());
644 DirectCount = It->second.getEntrycount();
647 assert(Ctx.callsites().count(NewCSID) == 0);
648 Ctx.ingestContext(NewCSID, std::move(It->second));
649 CSData.erase(CalleeGUID);
652 assert(TotalCount >= DirectCount);
653 uint64_t IndirectCount = TotalCount - DirectCount;
656 Ctx.counters()[DirectID] = DirectCount;
657 Ctx.counters()[IndirectID] = IndirectCount;
660 CtxProf.
update(ProfileUpdater, Caller);
668 assert(!AddressPoints.
empty() &&
"Caller should guarantee");
671 for (
auto &AddressPoint : AddressPoints)
691 LoadInst *VTableEntryLoad = dyn_cast<LoadInst>(Callee);
692 if (!VTableEntryLoad)
695 APInt VTableOffset(
DL.getIndexTypeSizeInBits(VTableEntryPtr->
getType()), 0);
697 DL, VTableOffset,
true);
698 LoadInst *VTablePtrLoad = dyn_cast<LoadInst>(VTableBasePtr);
702 APInt ObjectOffset(
DL.getIndexTypeSizeInBits(Object->getType()), 0);
703 Value *ObjectBase = Object->stripAndAccumulateConstantOffsets(
704 DL, ObjectOffset,
true);
705 if (!(isa<AllocaInst>(ObjectBase) && ObjectOffset == 0))
712 VTablePtrLoad, VTablePtrLoad->
getParent(), BBI, 0,
nullptr,
nullptr);
715 APInt VTableOffsetGVBase(
DL.getIndexTypeSizeInBits(VTablePtr->
getType()), 0);
717 DL, VTableOffsetGVBase,
true);
723 APInt VTableGVOffset = VTableOffsetGVBase + VTableOffset;
728 std::tie(DirectCallee, std::ignore) =
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Module.h This file contains the declarations for the Module class.
Reader for contextual iFDO profile, which comes in bitstream format.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned getActiveBits() const
Compute the number of active bits in the value.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
static uint64_t getGUID(const Function &F)
Type * getByValType() const
Retrieve the byval type.
AttrBuilder & addByValAttr(Type *Ty)
This turns a byval type into the form used internally in Attribute.
Type * getInAllocaType() const
Retrieve the inalloca type.
AttrBuilder & addInAllocaAttr(Type *Ty)
This turns an inalloca type into the form used internally in Attribute.
AttrBuilder & remove(const AttributeMask &AM)
Remove the attributes from the builder.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Return true if the attribute exists for the given argument.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
InstListType::iterator iterator
Instruction iterators...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
void mutateFunctionType(FunctionType *FTy)
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
Function * getCaller()
Helper to get the caller (the parent function).
This is the base class for all instructions that perform data casts.
static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
static CastInst * CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
static InstrProfIncrementInst * getBBInstrumentation(BasicBlock &BB)
Get the instruction instrumenting a BB, or nullptr if not present.
static InstrProfCallsite * getCallsiteInstrumentation(CallBase &CB)
Get the instruction instrumenting a callsite, or nullptr if that cannot be found.
A parsed version of the target data layout string in and methods for querying it.
Module * getParent()
Get the module that this global value is contained inside of...
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
BranchInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
BasicBlock * getUnwindDest() const
BasicBlock * getNormalDest() const
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
A Module instance is used to store all the information related to an LLVM module.
The instrumented contextual profile, produced by the CtxProfAnalysis.
void update(Visitor, const Function &F)
uint32_t allocateNextCounterIndex(const Function &F)
uint32_t allocateNextCallsiteIndex(const Function &F)
bool isFunctionKnown(const Function &F) const
A node (context) in the loaded contextual profile, suitable for mutation during IPO passes.
Class to represent pointers.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Return a value (possibly void), from a function.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isVoidTy() const
Return true if this is 'void'.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
void setName(const Twine &Name)
Change the name of the value.
iterator_range< user_iterator > users()
const ParentTy * getParent() const
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
AttributeMask typeIncompatible(Type *Ty, AttributeSet AS, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
This is an optimization pass for GlobalISel generic memory operations.
bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)
Promote the given indirect call site to conditionally call Callee.
Value * FindAvailableLoadedValue(LoadInst *Load, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan=DefMaxInstsToScan, BatchAAResults *AA=nullptr, bool *IsLoadCSE=nullptr, unsigned *NumScanedInst=nullptr)
Scan backwards to see if we have the value of the given load available locally within a small number ...
void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)
Predicate and clone the given call site.
CallBase & promoteCall(CallBase &CB, Function *Callee, CastInst **RetBitCast=nullptr)
Promote the given indirect call site to unconditionally call Callee.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
CallBase & promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr, Function *Callee, ArrayRef< Constant * > AddressPoints, MDNode *BranchWeights)
This is similar to promoteCallWithIfThenElse except that the condition to promote a virtual call is t...
std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)
Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...