67#define DEBUG_TYPE "expand-variadics"
75 cl::init(ExpandVariadicsMode::Unspecified),
77 "Use the implementation defaults"),
78 clEnumValN(ExpandVariadicsMode::Disable,
"disable",
79 "Disable the pass entirely"),
80 clEnumValN(ExpandVariadicsMode::Optimize,
"optimize",
81 "Optimise without changing ABI"),
82 clEnumValN(ExpandVariadicsMode::Lowering,
"lowering",
83 "Change variadic calling convention")));
85bool commandLineOverride() {
86 return ExpandVariadicsModeOption != ExpandVariadicsMode::Unspecified;
94class VariadicABIInfo {
96 VariadicABIInfo() =
default;
99 static std::unique_ptr<VariadicABIInfo> create(
const Triple &
T);
102 virtual bool enableForTarget() = 0;
107 virtual bool vaListPassedInSSARegister() = 0;
113 virtual Type *vaListParameterType(
Module &M) = 0;
122 struct VAArgSlotInfo {
129 bool vaEndIsNop() {
return true; }
130 bool vaCopyIsMemcpy() {
return true; }
132 virtual ~VariadicABIInfo() =
default;
165 std::unique_ptr<VariadicABIInfo> ABI;
169 Mode(commandLineOverride() ? ExpandVariadicsModeOption : Mode) {}
197 template <Intrinsic::ID ID,
typename InstructionType>
200 bool Changed =
false;
203 getPreexistingDeclaration(&M,
ID, {IntrinsicArgType})) {
205 if (
auto *
I = dyn_cast<InstructionType>(U))
206 Changed |= expandVAIntrinsicCall(Builder,
DL,
I);
208 if (Intrinsic->use_empty())
209 Intrinsic->eraseFromParent();
215 unsigned Addrspace) {
216 auto &Ctx = M.getContext();
218 bool Changed =
false;
221 Changed |= expandIntrinsicUsers<Intrinsic::vastart, VAStartInst>(
222 M, Builder, IntrinsicArgType);
223 Changed |= expandIntrinsicUsers<Intrinsic::vaend, VAEndInst>(
224 M, Builder, IntrinsicArgType);
225 Changed |= expandIntrinsicUsers<Intrinsic::vacopy, VACopyInst>(
226 M, Builder, IntrinsicArgType);
242 ArgTypes.
push_back(ABI->vaListParameterType(M));
250 uint64_t AsInt = AllocaTypeSize ? AllocaTypeSize->getFixedValue() : 0;
255 if (
F->isIntrinsic() || !
F->isVarArg() ||
256 F->hasFnAttribute(Attribute::Naked))
265 if (!
F->hasExactDefinition())
271 bool expansionApplicableToFunctionCall(
CallBase *CB) {
272 if (
CallInst *CI = dyn_cast<CallInst>(CB)) {
273 if (CI->isMustTailCall()) {
284 if (isa<InvokeInst>(CB)) {
293 class ExpandedCallFrame {
299 enum Tag { Store, Memcpy, Padding };
304 Source.push_back({V, Bytes, tag});
311 append<Memcpy>(
T, V, Bytes);
318 size_t size()
const {
return FieldTypes.
size(); }
319 bool empty()
const {
return FieldTypes.
empty(); }
322 const bool IsPacked =
true;
324 (
Twine(
Name) +
".vararg").str(), IsPacked);
332 for (
size_t I = 0;
I <
size();
I++) {
334 auto [V, bytes, tag] = Source[
I];
336 if (tag == Padding) {
355bool ExpandVariadics::runOnModule(
Module &M) {
356 bool Changed =
false;
357 if (Mode == ExpandVariadicsMode::Disable)
360 Triple TT(M.getTargetTriple());
361 ABI = VariadicABIInfo::create(TT);
365 if (!ABI->enableForTarget())
368 auto &Ctx = M.getContext();
385 unsigned Addrspace = 0;
386 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
388 Addrspace =
DL.getAllocaAddrSpace();
390 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
393 if (Mode != ExpandVariadicsMode::Lowering)
397 if (
F.isDeclaration())
405 if (
CallBase *CB = dyn_cast<CallBase>(&
I)) {
406 if (CB->isIndirectCall()) {
409 Changed |= expandCall(M, Builder, CB, FTy, 0);
421 bool Changed =
false;
423 if (!expansionApplicableToFunction(M, OriginalFunction))
426 [[maybe_unused]]
const bool OriginalFunctionIsDeclaration =
428 assert(rewriteABI() || !OriginalFunctionIsDeclaration);
432 replaceAllUsesWithNewDeclaration(M, OriginalFunction);
439 deriveFixedArityReplacement(M, Builder, OriginalFunction);
442 OriginalFunctionIsDeclaration);
446 [[maybe_unused]]
Function *VariadicWrapperDefine =
447 defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement);
448 assert(VariadicWrapperDefine == VariadicWrapper);
458 if (
CallBase *CB = dyn_cast<CallBase>(U)) {
459 Value *CalledOperand = CB->getCalledOperand();
460 if (VariadicWrapper == CalledOperand)
463 FixedArityReplacement);
471 Function *
const ExternallyAccessible =
472 rewriteABI() ? FixedArityReplacement : VariadicWrapper;
474 rewriteABI() ? VariadicWrapper : FixedArityReplacement;
480 ExternallyAccessible->
takeName(OriginalFunction);
502ExpandVariadics::replaceAllUsesWithNewDeclaration(
Module &M,
504 auto &Ctx = M.getContext();
509 NF->
setName(
F.getName() +
".varargs");
512 F.getParent()->getFunctionList().insert(
F.getIterator(), NF);
516 Attrs = Attrs.addParamAttributes(Ctx, FTy->getNumParams(), ParamAttrs);
532 assert(expansionApplicableToFunction(M, &
F));
534 auto &Ctx = M.getContext();
538 const bool FunctionIsDefinition = !
F.isDeclaration();
542 ArgTypes.
push_back(ABI->vaListParameterType(M));
544 FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy);
550 F.getParent()->getFunctionList().insert(
F.getIterator(), NF);
551 NF->
setName(
F.getName() +
".valist");
557 Attrs = Attrs.addParamAttributes(Ctx, NFTy->getNumParams() - 1, ParamAttrs);
561 if (FunctionIsDefinition) {
567 NewArg->setName(Arg.getName());
570 NewArg->setName(
"varargs");
574 F.getAllMetadata(MDs);
575 for (
auto [KindID,
Node] : MDs)
592 Type *VaListTy = ABI->vaListType(Ctx);
601 sizeOfAlloca(Ctx,
DL, VaListInstance));
610 Type *ParameterType = ABI->vaListParameterType(M);
611 if (ABI->vaListPassedInSSARegister())
612 Args.push_back(Builder.
CreateLoad(ParameterType, VaListInstance));
621 sizeOfAlloca(Ctx,
DL, VaListInstance));
623 if (Result->getType()->isVoidTy())
628 return VariadicWrapper;
634 bool Changed =
false;
637 if (!expansionApplicableToFunctionCall(CB)) {
647 if (FuncType != VarargFunctionType) {
650 FuncType = VarargFunctionType;
655 Align MaxFieldAlign(1);
665 ExpandedCallFrame Frame;
669 for (
unsigned I = FuncType->getNumParams(), E = CB->
arg_size();
I < E; ++
I) {
680 DL.getTypeAllocSize(UnderlyingType).getFixedValue();
683 Type *FrameFieldType = UnderlyingType;
686 Value *SourceValue = ArgVal;
688 VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(
DL, UnderlyingType);
690 if (SlotInfo.Indirect) {
696 Builder.
CreateAlloca(UnderlyingType,
nullptr,
"IndirectAlloca");
700 Builder.
CreateMemCpy(CallerCopy, {}, ArgVal, {}, UnderlyingSize);
705 FrameFieldType =
DL.getAllocaPtrType(Ctx);
706 SourceValue = CallerCopy;
711 Align DataAlign = SlotInfo.DataAlign;
713 MaxFieldAlign = std::max(MaxFieldAlign, DataAlign);
716 if (
uint64_t Rem = CurrentOffset % DataAlignV) {
718 uint64_t Padding = DataAlignV - Rem;
719 Frame.padding(Ctx, Padding);
720 CurrentOffset += Padding;
723 if (SlotInfo.Indirect) {
724 Frame.store(Ctx, FrameFieldType, SourceValue);
727 Frame.memcpy(Ctx, FrameFieldType, SourceValue, UnderlyingSize);
729 Frame.store(Ctx, FrameFieldType, SourceValue);
732 CurrentOffset +=
DL.getTypeAllocSize(FrameFieldType).getFixedValue();
740 Frame.padding(Ctx, 1);
752 Align AllocaAlign = MaxFieldAlign;
753 if (
DL.exceedsNaturalStackAlignment(
Align(1024)))
754 AllocaAlign = std::max(AllocaAlign,
DL.getStackAlignment());
764 new AllocaInst(VarargsTy,
DL.getAllocaAddrSpace(),
nullptr, AllocaAlign),
772 Frame.initializeStructAlloca(
DL, Builder, Alloced);
774 const unsigned NumArgs = FuncType->getNumParams();
781 if (!ABI->vaListPassedInSSARegister()) {
782 Type *VaListTy = ABI->vaListType(Ctx);
785 VaList = Builder.
CreateAlloca(VaListTy,
nullptr,
"va_argument");
790 Args.push_back(ABI->initializeVaList(M, Ctx, Builder, VaList, Alloced));
797 for (
unsigned ArgNo = 0; ArgNo < NumArgs; ArgNo++)
808 if (
CallInst *CI = dyn_cast<CallInst>(CB)) {
809 Value *Dst = NF ? NF : CI->getCalledOperand();
810 FunctionType *NFTy = inlinableVariadicFunctionType(M, VarargFunctionType);
812 NewCB =
CallInst::Create(NFTy, Dst, Args, OpBundles,
"", CI->getIterator());
820 CI->setTailCallKind(TCK);
837 NewCB->
copyMetadata(*CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg});
844bool ExpandVariadics::expandVAIntrinsicCall(
IRBuilder<> &Builder,
857 if (ContainingFunction->
isVarArg()) {
863 bool PassedByValue = ABI->vaListPassedInSSARegister();
876 assert(ABI->vaCopyIsMemcpy());
884 {VaStartArg, PassedVaList});
893 assert(ABI->vaEndIsNop());
898bool ExpandVariadics::expandVAIntrinsicCall(
IRBuilder<> &Builder,
901 assert(ABI->vaCopyIsMemcpy());
905 Type *VaListTy = ABI->vaListType(Ctx);
915struct Amdgpu final :
public VariadicABIInfo {
917 bool enableForTarget()
override {
return true; }
919 bool vaListPassedInSSARegister()
override {
return true; }
922 return PointerType::getUnqual(Ctx);
925 Type *vaListParameterType(
Module &M)
override {
926 return PointerType::getUnqual(M.getContext());
937 return {
Align(4),
false};
941struct NVPTX final :
public VariadicABIInfo {
943 bool enableForTarget()
override {
return true; }
945 bool vaListPassedInSSARegister()
override {
return true; }
951 Type *vaListParameterType(
Module &M)
override {
963 Align A =
DL.getABITypeAlign(Parameter);
968struct Wasm final :
public VariadicABIInfo {
970 bool enableForTarget()
override {
972 return commandLineOverride();
975 bool vaListPassedInSSARegister()
override {
return true; }
978 return PointerType::getUnqual(Ctx);
981 Type *vaListParameterType(
Module &M)
override {
982 return PointerType::getUnqual(M.getContext());
993 Align A =
DL.getABITypeAlign(Parameter);
997 if (
auto *S = dyn_cast<StructType>(Parameter)) {
998 if (S->getNumElements() > 1) {
999 return {
DL.getABITypeAlign(PointerType::getUnqual(Ctx)),
true};
1007std::unique_ptr<VariadicABIInfo> VariadicABIInfo::create(
const Triple &
T) {
1008 switch (
T.getArch()) {
1011 return std::make_unique<Amdgpu>();
1015 return std::make_unique<Wasm>();
1020 return std::make_unique<NVPTX>();
1030char ExpandVariadics::ID = 0;
1036 return new ExpandVariadics(M);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool runOnFunction(Function &F, bool PostInlining)
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
an instruction to allocate memory on the stack
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
A container for analyses that lazily runs them and caches their results.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
bool isEmpty() const
Return true if there are no attributes.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Type * getParamByRefType(unsigned ArgNo) const
Extract the byref type for a call or parameter.
CallingConv::ID getCallingConv() const
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This is the shared class of boolean and integer constants.
void removeDeadConstantUsers() const
If there are any dead constant users dangling off of this constant, remove them.
A parsed version of the target data layout string in and methods for querying it.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
ExpandVariadicsPass(ExpandVariadicsMode Mode)
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
void splice(Function::iterator ToIt, Function *FromF)
Transfer all blocks from FromF to this function at ToIt.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool IsNewDbgInfoFormat
Is this function using intrinsics to record the position of debugging information,...
AttributeList getAttributes() const
Return the attribute list for this Function.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
Argument * getArg(unsigned i) const
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
void setComdat(Comdat *C)
const Comdat * getComdat() const
void addMetadata(unsigned KindID, MDNode &MD)
Add a metadata attachment.
VisibilityTypes getVisibility() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
@ DefaultVisibility
The GV is visible.
void setVisibility(VisibilityTypes V)
@ InternalLinkage
Rename collisions when linking (static functions).
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
CallInst * CreateLifetimeStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.start intrinsic.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
ReturnInst * CreateRet(Value *V)
Create a 'ret <val>' instruction.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
void SetInsertPointPastAllocas(Function *F)
This specifies that created instructions should inserted at the beginning end of the specified functi...
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
LLVMContext & getContext() const
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateLifetimeEnd(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.end intrinsic.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const DebugLoc & getStableDebugLoc() const
Fetch the debug location for this node, unless this is a debug intrinsic, in which case fetch the deb...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
This is an important class for using LLVM in a threaded context.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_end intrinsic.
This represents the llvm.va_start intrinsic.
Value * getArgList() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=std::nullopt)
Return the function type for an intrinsic.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
ModulePass * createExpandVariadicsPass(ExpandVariadicsMode)
@ Wasm
WebAssembly Exception Handling.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr T MinAlign(U A, V B)
A and B are either alignments or offsets.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.