81#define DEBUG_TYPE "argpromotion"
83STATISTIC(NumArgumentsPromoted,
"Number of pointer arguments promoted");
84STATISTIC(NumArgumentsDead,
"Number of dead pointer args eliminated");
96using OffsetAndArgPart = std::pair<int64_t, ArgPart>;
119 std::vector<Type *> Params;
131 unsigned ArgNo = 0, NewArgNo = 0;
134 if (!ArgsToPromote.count(&*
I)) {
136 Params.push_back(
I->getType());
139 }
else if (
I->use_empty()) {
144 const auto &ArgParts = ArgsToPromote.find(&*
I)->second;
145 for (
const auto &Pair : ArgParts) {
146 Params.push_back(Pair.second.Ty);
149 ++NumArgumentsPromoted;
151 NewArgNo += ArgParts.size();
170 F->setSubprogram(
nullptr);
172 LLVM_DEBUG(
dbgs() <<
"ARG PROMOTION: Promoting to:" << *NF <<
"\n"
176 for (
auto *
I : Params)
177 if (
auto *VT = dyn_cast<llvm::VectorType>(
I))
178 LargestVectorWidth = std::max(
179 LargestVectorWidth, VT->getPrimitiveSizeInBits().getKnownMinValue());
188 unsigned Arg1 = NewArgIndices[AllocSize->first];
189 assert(Arg1 != (
unsigned)-1 &&
"allocsize cannot be promoted argument");
190 std::optional<unsigned> Arg2;
191 if (AllocSize->second) {
192 Arg2 = NewArgIndices[*AllocSize->second];
193 assert(Arg2 != (
unsigned)-1 &&
"allocsize cannot be promoted argument");
201 F->getParent()->getFunctionList().insert(
F->getIterator(), NF);
210 while (!
F->use_empty()) {
211 CallBase &CB = cast<CallBase>(*
F->user_back());
221 ++
I, ++AI, ++ArgNo) {
222 if (!ArgsToPromote.count(&*
I)) {
225 }
else if (!
I->use_empty()) {
227 const auto &ArgParts = ArgsToPromote.find(&*
I)->second;
228 for (
const auto &Pair : ArgParts) {
232 Pair.second.Alignment, V->getName() +
".val");
233 if (Pair.second.MustExecInstr) {
234 LI->
setAAMetadata(Pair.second.MustExecInstr->getAAMetadata());
236 {LLVMContext::MD_dereferenceable,
237 LLVMContext::MD_dereferenceable_or_null,
238 LLVMContext::MD_noundef,
239 LLVMContext::MD_nontemporal});
246 {LLVMContext::MD_range, LLVMContext::MD_nonnull,
247 LLVMContext::MD_align});
253 assert(ArgsToPromote.count(&*
I) &&
I->use_empty());
259 for (; AI != CB.
arg_end(); ++AI, ++ArgNo) {
274 NewCall->setTailCallKind(cast<CallInst>(&CB)->getTailCallKind());
281 NewCS->
copyMetadata(CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg});
313 if (!ArgsToPromote.count(&Arg)) {
335 assert(Arg.getType()->isPointerTy() &&
336 "Only arguments with a pointer type are promotable");
342 for (
const auto &Pair : ArgsToPromote.find(&Arg)->second) {
343 int64_t
Offset = Pair.first;
344 const ArgPart &Part = Pair.second;
350 Part.Ty,
nullptr, Arg.getName() +
"." +
Twine(
Offset) +
".allc");
362 assert(
Ptr == &Arg &&
"Not constant offset from arg?");
372 while (!Worklist.
empty()) {
374 if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V)) {
375 DeadInsts.
push_back(cast<Instruction>(V));
380 if (
auto *LI = dyn_cast<LoadInst>(V)) {
381 Value *
Ptr = LI->getPointerOperand();
386 if (
auto *SI = dyn_cast<StoreInst>(V)) {
387 assert(!SI->isVolatile() &&
"Volatile operations can't be promoted.");
388 Value *
Ptr = SI->getPointerOperand();
398 I->eraseFromParent();
402 for (
const auto &Pair : OffsetToAlloca) {
404 "By design, only promotable allocas should be produced.");
410 <<
" alloca(s) are promotable by Mem2Reg\n");
412 if (!Allocas.
empty()) {
431 APInt Bytes(64, NeededDerefBytes);
440 CallBase &CB = cast<CallBase>(*U);
441 return isDereferenceableAndAlignedPointer(CB.getArgOperand(Arg->getArgNo()),
442 NeededAlign, Bytes, DL);
449 unsigned MaxElements,
bool IsRecursive,
469 Align NeededAlign(1);
481 auto HandleEndUser = [&](
auto *
I,
Type *Ty,
482 bool GuaranteedToExecute) -> std::optional<bool> {
494 if (
Offset.getSignificantBits() >= 64)
499 if (
Size.isScalable())
504 if (IsRecursive && Ty->isPointerTy())
507 int64_t Off =
Offset.getSExtValue();
509 Off, ArgPart{Ty,
I->getAlign(), GuaranteedToExecute ?
I :
nullptr});
510 ArgPart &Part = Pair.first->second;
511 bool OffsetNotSeenBefore = Pair.second;
515 if (MaxElements > 0 && ArgParts.
size() > MaxElements) {
517 <<
"more than " << MaxElements <<
" parts\n");
525 <<
"accessed as both " << *Part.Ty <<
" and " << *Ty
526 <<
" at offset " << Off <<
"\n");
536 if (!GuaranteedToExecute &&
537 (OffsetNotSeenBefore || Part.Alignment <
I->getAlign())) {
546 NeededDerefBytes = std::max(NeededDerefBytes, Off +
Size.getFixedValue());
547 NeededAlign = std::max(NeededAlign,
I->getAlign());
550 Part.Alignment = std::max(Part.Alignment,
I->getAlign());
556 std::optional<bool> Res{};
557 if (
LoadInst *LI = dyn_cast<LoadInst>(&
I))
558 Res = HandleEndUser(LI, LI->getType(),
true);
559 else if (
StoreInst *SI = dyn_cast<StoreInst>(&
I))
560 Res = HandleEndUser(SI, SI->getValueOperand()->getType(),
574 auto AppendUses = [&](
const Value *V) {
575 for (
const Use &U : V->uses())
576 if (Visited.
insert(&U).second)
580 while (!Worklist.
empty()) {
582 Value *V = U->getUser();
583 if (isa<BitCastInst>(V)) {
588 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(V)) {
589 if (!
GEP->hasAllConstantIndices())
595 if (
auto *LI = dyn_cast<LoadInst>(V)) {
596 if (!*HandleEndUser(LI, LI->getType(),
false))
603 auto *SI = dyn_cast<StoreInst>(V);
604 if (AreStoresAllowed && SI &&
606 if (!*HandleEndUser(SI, SI->getValueOperand()->getType(),
616 <<
"unknown user " << *V <<
"\n");
620 if (NeededDerefBytes || NeededAlign > 1) {
625 <<
"not dereferenceable or aligned\n");
630 if (ArgParts.
empty())
638 int64_t
Offset = ArgPartsVec[0].first;
639 for (
const auto &Pair : ArgPartsVec) {
643 Offset = Pair.first +
DL.getTypeStoreSize(Pair.second.Ty);
649 if (AreStoresAllowed)
687 CallBase *CB = dyn_cast<CallBase>(U.getUser());
691 const Function *Caller = CB->getCaller();
692 const Function *Callee = CB->getCalledFunction();
693 return TTI.areTypesABICompatible(Caller, Callee, Types);
702 unsigned MaxElements,
bool IsRecursive) {
706 if (
F->hasFnAttribute(Attribute::Naked))
710 if (!
F->hasLocalLinkage())
723 if (
F->getAttributes().hasAttrSomewhere(Attribute::InAlloca))
729 if (
I.getType()->isPointerTy())
731 if (PointerArgs.
empty())
737 for (
Use &U :
F->uses()) {
738 CallBase *CB = dyn_cast<CallBase>(U.getUser());
740 if (CB ==
nullptr || !CB->
isCallee(&U) ||
755 if (BB.getTerminatingMustTailCall())
765 unsigned NumArgsAfterPromote =
F->getFunctionType()->getNumParams();
766 for (
Argument *PtrArg : PointerArgs) {
769 if (PtrArg->hasStructRetAttr()) {
770 unsigned ArgNo = PtrArg->getArgNo();
771 F->removeParamAttr(ArgNo, Attribute::StructRet);
772 F->addParamAttr(ArgNo, Attribute::NoAlias);
773 for (
Use &U :
F->uses()) {
774 CallBase &CB = cast<CallBase>(*U.getUser());
783 if (
findArgParts(PtrArg,
DL, AAR, MaxElements, IsRecursive, ArgParts)) {
785 for (
const auto &Pair : ArgParts)
786 Types.push_back(Pair.second.Ty);
789 NumArgsAfterPromote += ArgParts.size() - 1;
790 ArgsToPromote.
insert({PtrArg, std::move(ArgParts)});
796 if (ArgsToPromote.
empty())
809 bool Changed =
false, LocalChange;
818 bool IsRecursive =
C.size() > 1;
831 C.getOuterRefSCC().replaceNodeFunction(
N, *NewF);
837 for (
auto *U : NewF->
users()) {
838 auto *UserF = cast<CallBase>(U)->getFunction();
843 Changed |= LocalChange;
844 }
while (LocalChange);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file provides utility analysis objects describing memory locations.
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This defines the Use class.
A manager for alias analyses.
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2, const MemoryLocation &Loc, const ModRefInfo Mode)
Check if it is possible for the execution of the specified instructions to mod(according to the mode)...
bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc)
Check if it is possible for execution of the specified basic block to modify the location Loc.
Class for arbitrary precision integers.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
const Function * getParent() const
Type * getParamByValType() const
If this is a byval argument, return its type.
MaybeAlign getParamAlign() const
If this is a byval or inalloca argument, return its alignment.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A function analysis which provides an AssumptionCache.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
std::optional< std::pair< unsigned, std::optional< unsigned > > > getAllocSizeArgs() const
static Attribute getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, const std::optional< unsigned > &NumElemsArg)
LLVM Basic Block Representation.
const Instruction & front() const
Represents analyses that only rely on functions' control flow.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
FunctionType * getFunctionType() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Function * getCaller()
Helper to get the caller (the parent function).
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Analysis pass which computes a DominatorTree.
A proxy from a FunctionAnalysisManager to an SCC.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
void splice(Function::iterator ToIt, Function *FromF)
Transfer all blocks from FromF to this function at ToIt.
const BasicBlock & getEntryBlock() const
AttributeList getAttributes() const
Return the attribute list for this Function.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
void setIsNewDbgInfoFormat(bool NewVal)
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
void copyMetadata(const GlobalObject *Src, unsigned Offset)
Copy metadata from Src, adjusting offsets by Offset.
Common base class shared among various IRBuilders.
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
An instruction for reading from memory.
static unsigned getPointerOperandIndex()
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
static unsigned getPointerOperandIndex()
Analysis pass providing the TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void updateMinLegalVectorWidthAttr(Function &Fn, uint64_t Width)
Update min-legal-vector-width if it is in Attribute and less than Width.
@ C
The default llvm calling convention, compatible with C.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
void sort(IteratorTy Start, IteratorTy End)
iterator_range< idf_iterator< T > > inverse_depth_first(const T &G)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
auto predecessors(const MachineBasicBlock *BB)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Function object to check whether the first component of a container supported by std::get (like std::...