31#include "llvm/IR/IntrinsicsSPIRV.h"
40class SPIRVPrepareFunctions :
public ModulePass {
41 const SPIRVTargetMachine &TM;
42 bool substituteIntrinsicCalls(Function *
F);
43 Function *removeAggregateTypesFromSignature(Function *
F);
47 SPIRVPrepareFunctions(
const SPIRVTargetMachine &TM)
48 : ModulePass(ID), TM(TM) {}
50 bool runOnModule(
Module &M)
override;
52 StringRef getPassName()
const override {
return "SPIRV prepare functions"; }
54 void getAnalysisUsage(AnalysisUsage &AU)
const override {
55 ModulePass::getAnalysisUsage(AU);
61char SPIRVPrepareFunctions::ID = 0;
64 "SPIRV prepare functions",
false,
false)
67 Function *IntrinsicFunc =
II->getCalledFunction();
68 assert(IntrinsicFunc &&
"Missing function");
69 std::string FuncName = IntrinsicFunc->
getName().
str();
71 FuncName =
"spirv." + FuncName;
80 if (
F &&
F->getFunctionType() == FT)
99 std::string FuncName = lowerLLVMIntrinsicName(
Intrinsic);
101 FuncName +=
".volatile";
110 M->getOrInsertFunction(FuncName,
Intrinsic->getFunctionType());
111 auto IntrinsicID =
Intrinsic->getIntrinsicID();
115 assert(
F &&
"Callee must be a function");
117 switch (IntrinsicID) {
118 case Intrinsic::memset: {
127 IsVolatile->setName(
"isvolatile");
130 auto *MemSet = IRB.
CreateMemSet(Dest, Val, Len, MSI->getDestAlign(),
134 MemSet->eraseFromParent();
137 case Intrinsic::bswap: {
155 AnnoVal =
Ref->getOperand(0);
157 OptAnnoVal =
Ref->getOperand(0);
168 C &&
C->getNumOperands()) {
169 Value *MaybeStruct =
C->getOperand(0);
171 for (
unsigned I = 0,
E =
Struct->getNumOperands();
I !=
E; ++
I) {
173 Anno += (
I == 0 ?
": " :
", ") +
174 std::to_string(CInt->getType()->getIntegerBitWidth() == 1
175 ? CInt->getZExtValue()
176 : CInt->getSExtValue());
180 for (
unsigned I = 0,
E =
Struct->getType()->getStructNumElements();
182 Anno +=
I == 0 ?
": 0" :
", 0";
189 const std::string &Anno,
196 static const std::regex R(
197 "\\{(\\d+)(?:[:,](\\d+|\"[^\"]*\")(?:,(\\d+|\"[^\"]*\"))*)?\\}");
200 for (std::sregex_iterator
201 It = std::sregex_iterator(Anno.begin(), Anno.end(), R),
202 ItEnd = std::sregex_iterator();
204 if (It->position() != Pos)
206 Pos = It->position() + It->length();
207 std::smatch Match = *It;
209 for (std::size_t i = 1; i < Match.size(); ++i) {
210 std::ssub_match SMatch = Match[i];
211 std::string Item = SMatch.str();
212 if (Item.length() == 0)
214 if (Item[0] ==
'"') {
215 Item = Item.substr(1, Item.length() - 2);
217 static const std::regex RStr(
"^(\\d+)(?:,(\\d+))*$");
218 if (std::smatch MatchStr; std::regex_match(Item, MatchStr, RStr)) {
219 for (std::size_t SubIdx = 1; SubIdx < MatchStr.size(); ++SubIdx)
220 if (std::string SubStr = MatchStr[SubIdx].str(); SubStr.length())
222 ConstantInt::get(
Int32Ty, std::stoi(SubStr))));
233 if (MDsItem.
size() == 0)
237 return Pos ==
static_cast<int>(Anno.length()) ? std::move(MDs)
246 Value *PtrArg =
nullptr;
248 PtrArg = BI->getOperand(0);
250 PtrArg =
II->getOperand(0);
253 4 <
II->arg_size() ?
II->getArgOperand(4) :
nullptr);
262 if (MDs.
size() == 0) {
272 Intrinsic::spv_assign_decoration, {PtrArg->
getType()},
274 II->replaceAllUsesWith(
II->getOperand(0));
284 Type *FSHRetTy = FSHFuncTy->getReturnType();
285 const std::string FuncName = lowerLLVMIntrinsicName(FSHIntrinsic);
289 if (!FSHFunc->
empty()) {
301 unsigned BitWidth = IntTy->getIntegerBitWidth();
303 Value *BitWidthForInsts =
307 Value *RotateModVal =
309 Value *FirstShift =
nullptr, *SecShift =
nullptr;
322 Value *SubRotateVal = IRB.
CreateSub(BitWidthForInsts, RotateModVal);
348 if (
II->getIntrinsicID() == Intrinsic::assume) {
350 II->getModule(), Intrinsic::SPVIntrinsics::spv_assume);
351 II->setCalledFunction(
F);
352 }
else if (
II->getIntrinsicID() == Intrinsic::expect) {
354 II->getModule(), Intrinsic::SPVIntrinsics::spv_expect,
355 {II->getOperand(0)->getType()});
356 II->setCalledFunction(
F);
365 std::optional<TypeSize>
Size =
366 Alloca->getAllocationSize(Alloca->getDataLayout());
368 Builder.CreateIntrinsic(NewID, Alloca->getType(),
369 {SizeVal, II->getArgOperand(0)});
370 II->eraseFromParent();
376bool SPIRVPrepareFunctions::substituteIntrinsicCalls(
Function *
F) {
378 const SPIRVSubtarget &STI = TM.
getSubtarget<SPIRVSubtarget>(*F);
379 for (BasicBlock &BB : *
F) {
388 switch (
II->getIntrinsicID()) {
389 case Intrinsic::memset:
390 case Intrinsic::bswap:
393 case Intrinsic::fshl:
394 case Intrinsic::fshr:
398 case Intrinsic::assume:
399 case Intrinsic::expect:
404 case Intrinsic::lifetime_start:
407 II, Intrinsic::SPVIntrinsics::spv_lifetime_start);
409 II->eraseFromParent();
413 case Intrinsic::lifetime_end:
416 II, Intrinsic::SPVIntrinsics::spv_lifetime_end);
418 II->eraseFromParent();
422 case Intrinsic::ptr_annotation:
436SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *
F) {
437 bool IsRetAggr =
F->getReturnType()->isAggregateType();
439 if (
F->isIntrinsic() && IsRetAggr)
444 bool HasAggrArg =
llvm::any_of(
F->args(), [](Argument &Arg) {
445 return Arg.getType()->isAggregateType();
447 bool DoClone = IsRetAggr || HasAggrArg;
451 Type *RetType = IsRetAggr ?
B.getInt32Ty() :
F->getReturnType();
453 ChangedTypes.
push_back(std::pair<int, Type *>(-1,
F->getReturnType()));
455 for (
const auto &Arg :
F->args()) {
456 if (Arg.getType()->isAggregateType()) {
459 std::pair<int, Type *>(Arg.getArgNo(), Arg.getType()));
463 FunctionType *NewFTy =
464 FunctionType::get(RetType, ArgTypes,
F->getFunctionType()->isVarArg());
470 for (
auto &Arg :
F->args()) {
471 StringRef ArgName = Arg.getName();
472 NewFArgIt->setName(ArgName);
473 VMap[&Arg] = &(*NewFArgIt++);
481 NamedMDNode *FuncMD =
482 F->getParent()->getOrInsertNamedMetadata(
"spv.cloned_funcs");
485 for (
auto &ChangedTyP : ChangedTypes)
488 {ConstantAsMetadata::get(B.getInt32(ChangedTyP.first)),
489 ValueAsMetadata::get(Constant::getNullValue(ChangedTyP.second))}));
490 MDNode *ThisFuncMD =
MDNode::get(
B.getContext(), MDArgs);
496 U->replaceUsesOfWith(
F, NewF);
500 if (RetType !=
F->getReturnType())
501 TM.
getSubtarget<SPIRVSubtarget>(*F).getSPIRVGlobalRegistry()->addMutated(
502 NewF,
F->getReturnType());
506bool SPIRVPrepareFunctions::runOnModule(
Module &M) {
508 for (Function &
F : M) {
509 Changed |= substituteIntrinsicCalls(&
F);
513 std::vector<Function *> FuncsWorklist;
515 FuncsWorklist.push_back(&
F);
517 for (
auto *
F : FuncsWorklist) {
518 Function *NewF = removeAggregateTypesFromSignature(
F);
521 F->eraseFromParent();
530 return new SPIRVPrepareFunctions(TM);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Machine Check Debug Module
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic)
static std::string getAnnotation(Value *AnnoVal, Value *OptAnnoVal)
static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic)
static void lowerPtrAnnotation(IntrinsicInst *II)
static SmallVector< Metadata * > parseAnnotation(Value *I, const std::string &Anno, LLVMContext &Ctx, Type *Int32Ty)
static bool toSpvLifetimeIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID)
static void lowerExpectAssume(IntrinsicInst *II)
static Function * getOrCreateFunction(Module *M, Type *RetTy, ArrayRef< Type * > ArgTypes, StringRef Name)
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
FunctionType * getFunctionType() const
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This is the shared class of boolean and integer constants.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Type * getReturnType() const
Returns the type of the ret val.
void setCallingConv(CallingConv::ID CC)
Argument * getArg(unsigned i) const
void setDSOLocal(bool Local)
@ ExternalLinkage
Externally visible function.
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
ReturnInst * CreateRet(Value *V)
Create a 'ret <val>' instruction.
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, const AAMDNodes &AAInfo=AAMDNodes())
Create and insert a memset to the specified pointer and the specified value.
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
void LowerIntrinsicCall(CallInst *CI)
Replace a call to the specified intrinsic function.
This is an important class for using LLVM in a threaded context.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
A Module instance is used to store all the information related to an LLVM module.
LLVM_ABI void addOperand(MDNode *M)
bool canUseExtension(SPIRV::Extension::Extension E) const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Type * getElementType() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ SPIR_FUNC
Used for SPIR non-kernel device functions.
@ C
The default llvm calling convention, compatible with C.
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
LLVM_ABI bool getConstantStringInfo(const Value *V, StringRef &Str, bool TrimAtNul=true)
This function computes the length of a null-terminated C string pointed to by V.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool sortBlocks(Function &F)
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
@ Ref
The access may reference the value stored in memory.
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
void replace(R &&Range, const T &OldValue, const T &NewValue)
Provide wrappers to std::replace which take ranges instead of having to pass begin/end explicitly.
constexpr unsigned BitWidth
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
LLVM_ABI void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool to_integer(StringRef S, N &Num, unsigned Base=0)
Convert the string S to an integer of the specified type using the radix Base. If Base is 0,...
ModulePass * createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM)
LLVM_ABI void expandMemSetAsLoop(MemSetInst *MemSet)
Expand MemSet as a loop. MemSet is not deleted.
Implement std::hash so that hash_code can be used in STL containers.