31#include "llvm/IR/IntrinsicsSPIRV.h"
67#define DEBUG_TYPE "spirv-emit-intrinsics"
70#define GET_BuiltinGroup_DECL
71#include "SPIRVGenTables.inc"
76class GlobalVariableUsers {
77 template <
typename T1,
typename T2>
78 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
80 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
82 void collectGlobalUsers(
83 const GlobalVariable *GV,
84 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
85 &GlobalIsUsedByGlobal) {
87 while (!
Stack.empty()) {
91 GlobalIsUsedByFun[GV].insert(
I->getFunction());
96 GlobalIsUsedByGlobal[GV].insert(UserGV);
101 Stack.append(
C->user_begin(),
C->user_end());
105 bool propagateGlobalToGlobalUsers(
106 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
107 &GlobalIsUsedByGlobal) {
110 for (
auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
111 OldUsersGlobals.
assign(UserGlobals.begin(), UserGlobals.end());
112 for (
const GlobalVariable *UserGV : OldUsersGlobals) {
113 auto It = GlobalIsUsedByGlobal.find(UserGV);
114 if (It == GlobalIsUsedByGlobal.end())
122 void propagateGlobalToFunctionReferences(
123 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
124 &GlobalIsUsedByGlobal) {
125 for (
auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
126 auto &UserFunctions = GlobalIsUsedByFun[GV];
127 for (
const GlobalVariable *UserGV : UserGlobals) {
128 auto It = GlobalIsUsedByFun.find(UserGV);
129 if (It == GlobalIsUsedByFun.end())
140 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
141 GlobalIsUsedByGlobal;
142 GlobalIsUsedByFun.clear();
143 for (GlobalVariable &GV :
M.globals())
144 collectGlobalUsers(&GV, GlobalIsUsedByGlobal);
147 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
150 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
153 using FunctionSetType =
decltype(GlobalIsUsedByFun)::mapped_type;
154 const FunctionSetType &
155 getTransitiveUserFunctions(
const GlobalVariable &GV)
const {
156 auto It = GlobalIsUsedByFun.find(&GV);
157 if (It != GlobalIsUsedByFun.end())
160 static const FunctionSetType
Empty{};
165static bool isaGEP(
const Value *
V) {
171static std::optional<uint64_t> getByteAddressingMultiplier(
Type *Ty) {
177 return AT->getNumElements();
183class SPIRVEmitIntrinsicsImpl
184 :
public InstVisitor<SPIRVEmitIntrinsicsImpl, Instruction *> {
185 const SPIRVTargetMachine &TM;
186 SPIRVGlobalRegistry *GR =
nullptr;
188 bool TrackConstants =
true;
189 bool HaveFunPtrs =
false;
190 bool CanUseAnyVectorRank =
false;
191 DenseMap<Instruction *, Constant *> AggrConsts;
192 DenseMap<Instruction *, Type *> AggrConstTypes;
193 SmallPtrSet<Instruction *, 0> AggrStores;
194 GlobalVariableUsers GVUsers;
195 SmallPtrSet<Value *, 0> Named;
197 SmallPtrSet<GetElementPtrInst *, 4> InvalidGEPs;
200 MapVector<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
203 bool CanTodoType =
true;
204 unsigned TodoTypeSz = 0;
205 DenseMap<Value *, bool> TodoType;
206 void insertTodoType(
Value *
Op) {
208 if (CanTodoType && !isaGEP(
Op)) {
209 auto It = TodoType.try_emplace(
Op,
true);
214 void eraseTodoType(
Value *
Op) {
215 auto It = TodoType.find(
Op);
216 if (It != TodoType.end() && It->second) {
224 auto It = TodoType.find(
Op);
225 return It != TodoType.end() && It->second;
229 SmallPtrSet<Instruction *, 0> TypeValidated;
232 SmallVector<Instruction *> RetypedToRevalidate;
233 bool RevalidatingRetyped =
false;
236 enum WellKnownTypes { Event };
239 Type *deduceElementType(
Value *
I,
bool UnknownElemTypeI8);
240 Type *deduceElementTypeHelper(
Value *
I,
bool UnknownElemTypeI8);
241 Type *deduceElementTypeHelper(
Value *
I, SmallPtrSetImpl<Value *> &Visited,
242 bool UnknownElemTypeI8,
243 bool IgnoreKnownType =
false);
244 Type *deduceElementTypeByValueDeep(
Type *ValueTy,
Value *Operand,
245 bool UnknownElemTypeI8);
246 Type *deduceElementTypeByValueDeep(
Type *ValueTy,
Value *Operand,
247 SmallPtrSetImpl<Value *> &Visited,
248 bool UnknownElemTypeI8,
249 std::optional<unsigned> AddrSpace = {});
251 SmallPtrSetImpl<Value *> &Visited,
252 bool UnknownElemTypeI8);
254 bool UnknownElemTypeI8);
257 Type *deduceNestedTypeHelper(User *U,
bool UnknownElemTypeI8);
258 Type *deduceNestedTypeHelper(User *U,
Type *Ty,
259 SmallPtrSetImpl<Value *> &Visited,
260 bool UnknownElemTypeI8);
264 deduceOperandElementType(Instruction *
I,
265 SmallPtrSetImpl<Instruction *> *IncompleteRets,
266 const SmallPtrSetImpl<Value *> *AskOps =
nullptr,
267 bool IsPostprocessing =
false);
272 void insertCompositeAggregateArms(Instruction *
I,
IRBuilder<> &
B);
273 void simplifyNullAddrSpaceCasts();
275 Type *reconstructType(
Value *
Op,
bool UnknownElemTypeI8,
276 bool IsPostprocessing);
278 void replaceMemInstrUses(Instruction *Old, Instruction *New,
IRBuilder<> &
B);
280 bool insertAssignPtrTypeIntrs(Instruction *
I,
IRBuilder<> &
B,
281 bool UnknownElemTypeI8);
283 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType,
Value *
V,
285 void replacePointerOperandWithPtrCast(Instruction *
I,
Value *Pointer,
286 Type *ExpectedElementType,
287 unsigned OperandToReplace,
289 void insertPtrCastOrAssignTypeInstr(Instruction *
I,
IRBuilder<> &
B);
290 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
292 void insertConstantsForFPFastMathDefault(
Module &M);
295 void processGlobalValue(GlobalVariable &GV,
IRBuilder<> &
B);
298 Type *deduceFunParamElementType(
Function *
F,
unsigned OpIdx);
300 SmallPtrSetImpl<Function *> &FVisited);
302 bool deduceOperandElementTypeCalledFunction(
304 Type *&KnownElemTy,
bool &Incomplete);
305 void deduceOperandElementTypeFunctionPointer(
307 Type *&KnownElemTy,
bool IsPostprocessing);
308 bool deduceOperandElementTypeFunctionRet(
309 Instruction *
I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
310 const SmallPtrSetImpl<Value *> *AskOps,
bool IsPostprocessing,
314 void replaceUsesOfWithSpvPtrcast(
Value *
Op,
Type *ElemTy, Instruction *
I,
315 DenseMap<Function *, CallInst *> Ptrcasts);
317 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
320 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
321 void propagateElemTypeRec(
Value *
Op,
Type *PtrElemTy,
Type *CastElemTy,
322 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
323 SmallPtrSetImpl<Value *> &Visited,
324 DenseMap<Function *, CallInst *> Ptrcasts);
327 void replaceAllUsesWithAndErase(
IRBuilder<> &
B, Instruction *Src,
328 Instruction *Dest,
bool DeleteOld =
true);
332 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *
GEP);
335 bool postprocessTypes(
Module &M);
336 bool processFunctionPointers(
Module &M);
337 void parseFunDeclarations(
Module &M);
338 void useRoundingMode(ConstrainedFPIntrinsic *FPI,
IRBuilder<> &
B);
339 bool processMaskedMemIntrinsic(IntrinsicInst &
I);
340 bool convertMaskedMemIntrinsics(
Module &M);
341 bool preprocessModule(
Module &M);
342 void preprocessBoolVectorBitcasts(
Function &
F);
343 void scalarizeVectorPtrCasts(
Function &
F);
362 Expected<bool> walkLogicalAccessChain(
363 GetElementPtrInst &
GEP,
364 const std::function<
void(
Type *PointedType,
uint64_t Index)>
367 uint64_t Multiplier)> &OnDynamicIndexing);
369 bool walkLogicalAccessChainDynamic(
371 const std::function<
void(
Type *,
uint64_t)> &OnLiteralIndexing,
374 Expected<bool> walkLogicalAccessChainConstant(
376 const std::function<
void(
Type *,
uint64_t)> &OnLiteralIndexing);
378 void recordAccessChainError(GetElementPtrInst &
GEP,
Error Err);
384 Type *getGEPType(GetElementPtrInst *
GEP);
391 Type *getGEPTypeLogical(GetElementPtrInst *
GEP);
393 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &
GEP);
396 SPIRVEmitIntrinsicsImpl(
const SPIRVTargetMachine &TM) : TM(TM) {}
399 Instruction *visitGetElementPtrInst(GetElementPtrInst &
I);
402 Instruction *visitInsertElementInst(InsertElementInst &
I);
403 Instruction *visitExtractElementInst(ExtractElementInst &
I);
405 Instruction *visitExtractValueInst(ExtractValueInst &
I);
409 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &
I);
414 bool runOnModule(
Module &M);
417class SPIRVEmitIntrinsicsLegacy :
public ModulePass {
418 const SPIRVTargetMachine &TM;
422 SPIRVEmitIntrinsicsLegacy(
const SPIRVTargetMachine &TM)
423 : ModulePass(ID), TM(TM) {}
425 StringRef getPassName()
const override {
return "SPIRV emit intrinsics"; }
427 bool runOnModule(
Module &M)
override {
428 return SPIRVEmitIntrinsicsImpl(TM).runOnModule(M);
434 Intrinsic::experimental_convergence_loop,
435 Intrinsic::experimental_convergence_anchor>());
438bool expectIgnoredInIRTranslation(
const Instruction *
I) {
440 Intrinsic::spv_resource_handlefrombinding,
441 Intrinsic::spv_resource_getbasepointer,
442 Intrinsic::spv_resource_getpointer>());
449 return getPointerRoot(
V);
455char SPIRVEmitIntrinsicsLegacy::ID = 0;
458 "SPIRV emit intrinsics",
false,
false)
485 B.SetInsertPoint(
I->getParent()->getFirstNonPHIOrAlloca());
491 B.SetCurrentDebugLocation(
I->getDebugLoc());
492 if (
I->getType()->isVoidTy())
493 B.SetInsertPoint(
I->getNextNode());
495 B.SetInsertPoint(*
I->getInsertionPointAfterDef());
505 if (
I->getType()->isTokenTy())
507 "does not support token type",
513 if (!
I->hasName() ||
I->getType()->isAggregateType() ||
514 expectIgnoredInIRTranslation(
I))
519 bool KeepName = CLOpts.emit_op_names;
525 if (
F &&
F->getName().starts_with(
"llvm.spv.alloca"))
536 std::vector<Value *> Args = {
539 B.CreateIntrinsic(Intrinsic::spv_assign_name, {
I->getType()}, Args);
542void SPIRVEmitIntrinsicsImpl::replaceAllUsesWith(
Value *Src,
Value *Dest,
546 if (isTodoType(Src)) {
549 insertTodoType(Dest);
553void SPIRVEmitIntrinsicsImpl::replaceAllUsesWithAndErase(
IRBuilder<> &
B,
558 std::string
Name = Src->hasName() ? Src->getName().str() :
"";
559 Src->eraseFromParent();
562 if (Named.
insert(Dest).second)
577 V =
V->stripPointerCasts();
607Type *SPIRVEmitIntrinsicsImpl::reconstructType(
Value *
Op,
608 bool UnknownElemTypeI8,
609 bool IsPostprocessing) {
613 if (
auto It = AggrConstTypes.
find(OpI); It != AggrConstTypes.
end())
627 if (UnknownElemTypeI8) {
628 if (!IsPostprocessing)
644 B.SetInsertPointPastAllocas(OpA->getParent());
647 B.SetInsertPoint(
F->getEntryBlock().getFirstNonPHIOrAlloca());
649 Type *OpTy =
Op->getType();
651 SmallVector<Value *, 2>
Args = {
654 CallInst *PtrCasted =
655 B.CreateIntrinsicWithoutFolding(Intrinsic::spv_ptrcast, {
Types},
Args);
660void SPIRVEmitIntrinsicsImpl::replaceUsesOfWithSpvPtrcast(
662 DenseMap<Function *, CallInst *> Ptrcasts) {
664 CallInst *PtrCastedI =
nullptr;
665 auto It = Ptrcasts.
find(
F);
666 if (It == Ptrcasts.
end()) {
667 PtrCastedI = buildSpvPtrcast(
F,
Op, ElemTy);
668 Ptrcasts[
F] = PtrCastedI;
670 PtrCastedI = It->second;
672 I->replaceUsesOfWith(
Op, PtrCastedI);
675void SPIRVEmitIntrinsicsImpl::propagateElemType(
677 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
678 DenseMap<Function *, CallInst *> Ptrcasts;
680 for (
auto *U :
Users) {
683 if (!VisitedSubst.insert(std::make_pair(U,
Op)).second)
688 if (isaGEP(UI) || TypeValidated.
find(UI) != TypeValidated.
end())
689 replaceUsesOfWithSpvPtrcast(
Op, ElemTy, UI, Ptrcasts);
693void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
695 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
696 SmallPtrSet<Value *, 0> Visited;
697 DenseMap<Function *, CallInst *> Ptrcasts;
698 propagateElemTypeRec(
Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
699 std::move(Ptrcasts));
702void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
704 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
705 SmallPtrSetImpl<Value *> &Visited,
706 DenseMap<Function *, CallInst *> Ptrcasts) {
710 for (
auto *U :
Users) {
713 if (!VisitedSubst.insert(std::make_pair(U,
Op)).second)
718 if (isaGEP(UI) || TypeValidated.
find(UI) != TypeValidated.
end())
719 replaceUsesOfWithSpvPtrcast(
Op, CastElemTy, UI, Ptrcasts);
726Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
727 Type *ValueTy,
Value *Operand,
bool UnknownElemTypeI8) {
728 SmallPtrSet<Value *, 0> Visited;
729 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
733Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
734 Type *ValueTy,
Value *Operand, SmallPtrSetImpl<Value *> &Visited,
735 bool UnknownElemTypeI8, std::optional<unsigned> AddrSpace) {
740 deduceElementTypeHelper(Operand, Visited, UnknownElemTypeI8))
742 NestedTy, AddrSpace.value_or(PtrTy->getAddressSpace()));
752Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByUsersDeep(
753 Value *
Op, SmallPtrSetImpl<Value *> &Visited,
bool UnknownElemTypeI8) {
765 for (User *OpU :
Op->users()) {
767 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited, UnknownElemTypeI8))
779 Function *CalledF,
unsigned OpIdx) {
784 if (OpIdx == 0 && (Name ==
"printf" || Name ==
"get_fence"))
787 if (OpIdx == 2 && (Name ==
"capture_event_profiling_info" ||
788 Name ==
"__spirv_CaptureEventProfilingInfo"))
795Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
Value *
I,
796 bool UnknownElemTypeI8) {
797 SmallPtrSet<Value *, 0> Visited;
798 return deduceElementTypeHelper(
I, Visited, UnknownElemTypeI8);
801void SPIRVEmitIntrinsicsImpl::maybeAssignPtrType(
Type *&Ty,
Value *
Op,
803 bool UnknownElemTypeI8) {
805 if (!UnknownElemTypeI8)
814bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainDynamic(
816 const std::function<
void(
Type *,
uint64_t)> &OnLiteralIndexing,
823 if (
ST->getNumElements() == 0)
825 CurType =
ST->getElementType(0);
826 OnLiteralIndexing(CurType, 0);
834 OnDynamicIndexing(AT->getElementType(), Operand, Multiplier);
835 return AT ==
nullptr;
838Expected<bool> SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainConstant(
840 const std::function<
void(
Type *,
uint64_t)> &OnLiteralIndexing) {
846 uint64_t EltTypeSize =
DL.getTypeAllocSize(AT->getElementType());
850 CurType = AT->getElementType();
851 OnLiteralIndexing(CurType, Index);
853 uint32_t StructSize =
DL.getTypeSizeInBits(ST) / 8;
856 const auto &STL =
DL.getStructLayout(ST);
857 unsigned Element = STL->getElementContainingOffset(
Offset);
858 Offset -= STL->getElementOffset(Element);
859 CurType =
ST->getElementType(Element);
860 OnLiteralIndexing(CurType, Element);
862 Type *EltTy = VT->getElementType();
863 TypeSize EltSizeBits =
DL.getTypeSizeInBits(EltTy);
864 assert(EltSizeBits % 8 == 0 &&
865 "Element type size in bits must be a multiple of 8.");
866 uint32_t EltTypeSize = EltSizeBits / 8;
867 if (
Offset >= VT->getNumElements() * EltTypeSize)
869 "Cannot lower an access to vector padding for Logical SPIR-V");
873 OnLiteralIndexing(CurType, Index);
878 "Cannot lower a byte offset within a scalar or padding for "
886Expected<bool> SPIRVEmitIntrinsicsImpl::walkLogicalAccessChain(
887 GetElementPtrInst &
GEP,
888 const std::function<
void(
Type *,
uint64_t)> &OnLiteralIndexing,
892 std::optional<uint64_t> MultiplierOpt =
893 getByteAddressingMultiplier(
GEP.getSourceElementType());
894 assert(MultiplierOpt &&
"We only rewrite byte-addressing GEP");
895 uint64_t Multiplier = *MultiplierOpt;
898 Value *Src = getPointerRoot(
GEP.getPointerOperand());
899 Type *CurType = deduceElementType(Src,
true);
902 "Cannot lower byte addressing with an unsized type for Logical SPIR-V");
906 return walkLogicalAccessChainConstant(
907 CurType, CI->getZExtValue() * Multiplier, OnLiteralIndexing);
909 return walkLogicalAccessChainDynamic(CurType, Operand, Multiplier,
910 OnLiteralIndexing, OnDynamicIndexing);
913void SPIRVEmitIntrinsicsImpl::recordAccessChainError(GetElementPtrInst &
GEP,
917 std::move(AccessChainErrors),
919 GEP.getFunction()->getName() +
"')"));
922Instruction *SPIRVEmitIntrinsicsImpl::buildLogicalAccessChainFromGEP(
923 GetElementPtrInst &
GEP) {
929 B.SetInsertPoint(&
GEP);
931 std::vector<Value *> Indices;
932 Indices.push_back(ConstantInt::get(
933 IntegerType::getInt32Ty(CurrF->
getContext()), 0,
false));
934 auto Result = walkLogicalAccessChain(
938 ConstantInt::get(
B.getInt64Ty(), Index,
false));
943 uint32_t EltTypeSize =
DL.getTypeSizeInBits(EltType) / 8;
945 if (Multiplier == EltTypeSize) {
947 }
else if (EltTypeSize % Multiplier == 0) {
950 EltTypeSize / Multiplier,
954 ConstantInt::get(
Offset->getType(), Multiplier,
957 Index =
B.CreateUDiv(Index,
958 ConstantInt::get(
Offset->getType(), EltTypeSize,
962 Indices.push_back(Index);
965 recordAccessChainError(
GEP,
Result.takeError());
969 B.SetInsertPoint(&
GEP);
971 SmallVector<Value *, 4>
Args;
972 Args.push_back(
B.getInt1(
GEP.isInBounds()));
973 Args.push_back(
GEP.getOperand(0));
976 B.CreateIntrinsicWithoutFolding(Intrinsic::spv_gep, {
Types}, {
Args});
977 replaceAllUsesWithAndErase(
B, &
GEP, NewI);
981Type *SPIRVEmitIntrinsicsImpl::getGEPTypeLogical(GetElementPtrInst *
GEP) {
985 Type *CurType =
GEP->getResultElementType();
987 auto Result = walkLogicalAccessChain(
988 *
GEP, [&CurType](
Type *EltType,
uint64_t Index) { CurType = EltType; },
991 recordAccessChainError(*
GEP,
Result.takeError());
995 return *
Result ?
GEP->getResultElementType() : CurType;
998Type *SPIRVEmitIntrinsicsImpl::getGEPType(GetElementPtrInst *
Ref) {
999 if (getByteAddressingMultiplier(
Ref->getSourceElementType()) &&
1001 return getGEPTypeLogical(
Ref);
1008 Ty =
Ref->getSourceElementType();
1012 Ty =
Ref->getResultElementType();
1017Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
1018 Value *
I, SmallPtrSetImpl<Value *> &Visited,
bool UnknownElemTypeI8,
1019 bool IgnoreKnownType) {
1025 if (!IgnoreKnownType)
1030 if (!Visited.
insert(
I).second)
1037 maybeAssignPtrType(Ty,
I,
Ref->getAllocatedType(), UnknownElemTypeI8);
1039 Ty = getGEPType(
Ref);
1041 Ty = SGEP->getResultElementType();
1046 KnownTy =
Op->getType();
1048 maybeAssignPtrType(Ty,
I, ElemTy, UnknownElemTypeI8);
1051 Ty = SPIRV::getOriginalFunctionType(*Fn);
1058 std::optional<unsigned> ProgramAS;
1063 SPIRV::Extension::SPV_INTEL_function_pointers))
1067 Ty = deduceElementTypeByValueDeep(
Ref->getValueType(),
Op, Visited,
1068 UnknownElemTypeI8, ProgramAS);
1071 Type *RefTy = deduceElementTypeHelper(
Ref->getPointerOperand(), Visited,
1073 maybeAssignPtrType(Ty,
I, RefTy, UnknownElemTypeI8);
1075 maybeAssignPtrType(Ty,
I,
Ref->getDestTy(), UnknownElemTypeI8);
1077 if (
Type *Src =
Ref->getSrcTy(), *Dest =
Ref->getDestTy();
1079 Ty = deduceElementTypeHelper(
Ref->getOperand(0), Visited,
1085 reconstructType(
Ref->getAggregateOperand(),
false,
false),
1090 Ty = deduceElementTypeHelper(
Op, Visited, UnknownElemTypeI8);
1094 Ty = deduceElementTypeHelper(
Op, Visited, UnknownElemTypeI8);
1096 Type *BestTy =
nullptr;
1098 DenseMap<Type *, unsigned> PhiTys;
1099 for (
int i =
Ref->getNumIncomingValues() - 1; i >= 0; --i) {
1100 Ty = deduceElementTypeByUsersDeep(
Ref->getIncomingValue(i), Visited,
1107 if (It.first->second > MaxN) {
1108 MaxN = It.first->second;
1116 for (
Value *
Op : {
Ref->getTrueValue(),
Ref->getFalseValue()}) {
1120 ? deduceElementTypeHelper(
Op, Visited, UnknownElemTypeI8)
1121 : deduceElementTypeByUsersDeep(
Op, Visited, UnknownElemTypeI8);
1126 static StringMap<unsigned> ResTypeByArg = {
1132 {
"__to_private", 0},
1133 {
"__spirv_GenericCastToPtr_ToGlobal", 0},
1134 {
"__spirv_GenericCastToPtr_ToLocal", 0},
1135 {
"__spirv_GenericCastToPtr_ToPrivate", 0},
1136 {
"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
1137 {
"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
1138 {
"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
1142 if (
II && (
II->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
1143 II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
1145 if (HandleType->getTargetExtName() ==
"spirv.Image" ||
1146 HandleType->getTargetExtName() ==
"spirv.SignedImage") {
1147 for (User *U :
II->users()) {
1152 }
else if (HandleType->getTargetExtName() ==
"spirv.VulkanBuffer") {
1154 Ty = HandleType->getTypeParameter(0);
1155 if (
II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1169 }
else if (
II &&
II->getIntrinsicID() ==
1170 Intrinsic::spv_generic_cast_to_ptr_explicit) {
1174 std::string DemangledName =
1176 if (DemangledName.length() > 0)
1177 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName);
1178 auto AsArgIt = ResTypeByArg.
find(DemangledName);
1179 if (AsArgIt != ResTypeByArg.
end())
1180 Ty = deduceElementTypeHelper(CI->
getArgOperand(AsArgIt->second),
1181 Visited, UnknownElemTypeI8);
1188 if (Ty && !IgnoreKnownType) {
1199Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(User *U,
1200 bool UnknownElemTypeI8) {
1201 SmallPtrSet<Value *, 0> Visited;
1202 return deduceNestedTypeHelper(U,
U->getType(), Visited, UnknownElemTypeI8);
1205Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(
1206 User *U,
Type *OrigTy, SmallPtrSetImpl<Value *> &Visited,
1207 bool UnknownElemTypeI8) {
1217 if (!Visited.
insert(U).second)
1222 bool Change =
false;
1223 for (
unsigned i = 0; i <
U->getNumOperands(); ++i) {
1225 assert(
Op &&
"Operands should not be null.");
1226 Type *OpTy =
Op->getType();
1229 if (
Type *NestedTy =
1230 deduceElementTypeHelper(
Op, Visited, UnknownElemTypeI8))
1237 Change |= Ty != OpTy;
1241 Tys, OrigStructTy->isLiteral() ?
"" : OrigStructTy->getName(),
1242 OrigStructTy->isPacked());
1247 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
1248 Type *OpTy = ArrTy->getElementType();
1251 if (
Type *NestedTy =
1252 deduceElementTypeHelper(
Op, Visited, UnknownElemTypeI8))
1259 Type *NewTy = ArrayType::get(Ty, ArrTy->getNumElements());
1265 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
1266 Type *OpTy = VecTy->getElementType();
1269 if (
Type *NestedTy =
1270 deduceElementTypeHelper(
Op, Visited, UnknownElemTypeI8))
1277 Type *NewTy = VectorType::get(Ty, VecTy->getElementCount());
1288Type *SPIRVEmitIntrinsicsImpl::deduceElementType(
Value *
I,
1289 bool UnknownElemTypeI8) {
1290 if (
Type *Ty = deduceElementTypeHelper(
I, UnknownElemTypeI8))
1292 if (!UnknownElemTypeI8)
1295 return IntegerType::getInt8Ty(
I->getContext());
1299 Value *PointerOperand) {
1305 return I->getType();
1313bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeCalledFunction(
1315 Type *&KnownElemTy,
bool &Incomplete) {
1319 std::string DemangledName =
1321 if (DemangledName.length() > 0 &&
1323 const SPIRVSubtarget &
ST = TM.
getSubtarget<SPIRVSubtarget>(*CalledF);
1324 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1325 DemangledName,
ST.getPreferredInstructionSet());
1326 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1327 for (
unsigned i = 0, PtrCnt = 0; i < CI->
arg_size() && PtrCnt < 2; ++i) {
1333 KnownElemTy = ElemTy;
1334 Ops.push_back(std::make_pair(
Op, i));
1336 }
else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1343 case SPIRV::OpAtomicFAddEXT:
1344 case SPIRV::OpAtomicFMinEXT:
1345 case SPIRV::OpAtomicFMaxEXT:
1346 case SPIRV::OpAtomicLoad:
1347 case SPIRV::OpAtomicCompareExchangeWeak:
1348 case SPIRV::OpAtomicCompareExchange:
1349 case SPIRV::OpAtomicExchange:
1350 case SPIRV::OpAtomicIAdd:
1351 case SPIRV::OpAtomicISub:
1352 case SPIRV::OpAtomicOr:
1353 case SPIRV::OpAtomicXor:
1354 case SPIRV::OpAtomicAnd:
1355 case SPIRV::OpAtomicUMin:
1356 case SPIRV::OpAtomicUMax:
1357 case SPIRV::OpAtomicSMin:
1358 case SPIRV::OpAtomicSMax: {
1363 Incomplete = isTodoType(
Op);
1364 Ops.push_back(std::make_pair(
Op, 0));
1366 case SPIRV::OpAtomicStore: {
1375 Incomplete = isTodoType(
Op);
1376 Ops.push_back(std::make_pair(
Op, 0));
1385void SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionPointer(
1387 Type *&KnownElemTy,
bool IsPostprocessing) {
1391 Ops.push_back(std::make_pair(
Op, std::numeric_limits<unsigned>::max()));
1392 FunctionType *FTy = SPIRV::getOriginalFunctionType(*CI);
1393 bool IsNewFTy =
false, IsIncomplete =
false;
1396 Type *ArgTy = Arg->getType();
1401 if (isTodoType(Arg))
1402 IsIncomplete =
true;
1404 IsIncomplete =
true;
1407 ArgTy = FTy->getFunctionParamType(ParmIdx);
1411 Type *RetTy = FTy->getReturnType();
1418 IsIncomplete =
true;
1420 IsIncomplete =
true;
1423 if (!IsPostprocessing && IsIncomplete)
1426 IsNewFTy ? FunctionType::get(RetTy, ArgTys, FTy->isVarArg()) : FTy;
1429bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionRet(
1430 Instruction *
I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1431 const SmallPtrSetImpl<Value *> *AskOps,
bool IsPostprocessing,
1443 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(
I,
Op)};
1444 for (User *U :
F->users()) {
1453 propagateElemType(CI, PrevElemTy, VisitedSubst);
1463 for (Instruction *IncompleteRetI : *IncompleteRets)
1464 deduceOperandElementType(IncompleteRetI,
nullptr, AskOps,
1466 }
else if (IncompleteRets) {
1477void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
1478 Instruction *
I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1479 const SmallPtrSetImpl<Value *> *AskOps,
bool IsPostprocessing) {
1481 Type *KnownElemTy =
nullptr;
1482 bool Incomplete =
false;
1488 Incomplete = isTodoType(
I);
1489 for (
unsigned i = 0; i <
Ref->getNumIncomingValues(); i++) {
1492 Ops.push_back(std::make_pair(
Op, i));
1498 Incomplete = isTodoType(
I);
1499 Ops.push_back(std::make_pair(
Ref->getPointerOperand(), 0));
1506 Incomplete = isTodoType(
I);
1507 Ops.push_back(std::make_pair(
Ref->getOperand(0), 0));
1511 KnownElemTy =
Ref->getSourceElementType();
1512 Ops.push_back(std::make_pair(
Ref->getPointerOperand(),
1517 KnownElemTy =
Ref->getBaseType();
1518 Ops.push_back(std::make_pair(
Ref->getPointerOperand(),
1521 KnownElemTy =
I->getType();
1528 Value *Root =
Ref->getPointerOperand()->stripPointerCasts();
1537 Ops.push_back(std::make_pair(
Ref->getPointerOperand(),
1541 reconstructType(
Ref->getValueOperand(),
false, IsPostprocessing)))
1546 Ops.push_back(std::make_pair(
Ref->getPointerOperand(),
1554 Incomplete = isTodoType(
Ref->getPointerOperand());
1555 Ops.push_back(std::make_pair(
Ref->getPointerOperand(),
1563 Incomplete = isTodoType(
Ref->getPointerOperand());
1564 Ops.push_back(std::make_pair(
Ref->getPointerOperand(),
1570 Incomplete = isTodoType(
I);
1571 for (
unsigned i = 0; i <
Ref->getNumOperands(); i++) {
1574 Ops.push_back(std::make_pair(
Op, i));
1582 if (deduceOperandElementTypeFunctionRet(
I, IncompleteRets, AskOps,
1583 IsPostprocessing, KnownElemTy,
Op,
1586 Incomplete = isTodoType(CurrF);
1587 Ops.push_back(std::make_pair(
Op, 0));
1593 bool Incomplete0 = isTodoType(Op0);
1594 bool Incomplete1 = isTodoType(Op1);
1596 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1598 : GR->findDeducedElementType(Op0);
1600 KnownElemTy = ElemTy0;
1601 Incomplete = Incomplete0;
1602 Ops.push_back(std::make_pair(Op1, 1));
1603 }
else if (ElemTy1) {
1604 KnownElemTy = ElemTy1;
1605 Incomplete = Incomplete1;
1606 Ops.push_back(std::make_pair(Op0, 0));
1610 deduceOperandElementTypeCalledFunction(CI,
Ops, KnownElemTy, Incomplete);
1611 else if (HaveFunPtrs)
1612 deduceOperandElementTypeFunctionPointer(CI,
Ops, KnownElemTy,
1617 if (!KnownElemTy ||
Ops.size() == 0)
1621 for (
auto &OpIt :
Ops) {
1625 Type *AskTy =
nullptr;
1626 CallInst *AskCI =
nullptr;
1627 if (IsPostprocessing && AskOps) {
1633 if (Ty == KnownElemTy)
1636 Type *OpTy =
Op->getType();
1642 if (
Op->hasUseList() && !WouldClobberPtrWithNonPtr &&
1650 else if (!IsPostprocessing)
1654 if (AssignCI ==
nullptr) {
1663 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1664 std::make_pair(
I,
Op)};
1665 propagateElemTypeRec(
Op, KnownElemTy, PrevElemTy, VisitedSubst);
1676 CallInst *PtrCastI =
1677 buildSpvPtrcast(
I->getParent()->getParent(),
Op, KnownElemTy);
1678 if (OpIt.second == std::numeric_limits<unsigned>::max())
1681 I->setOperand(OpIt.second, PtrCastI);
1688 if (RevalidatingRetyped)
1690 RevalidatingRetyped =
true;
1691 while (!RetypedToRevalidate.
empty())
1692 deduceOperandElementType(RetypedToRevalidate.
pop_back_val(),
nullptr,
1693 nullptr, IsPostprocessing);
1694 RevalidatingRetyped =
false;
1697void SPIRVEmitIntrinsicsImpl::replaceMemInstrUses(Instruction *Old,
1702 if (isAssignTypeInstr(U)) {
1703 B.SetInsertPoint(U);
1704 SmallVector<Value *, 2>
Args = {
New,
U->getOperand(1)};
1705 CallInst *AssignCI =
B.CreateIntrinsicWithoutFolding(
1706 Intrinsic::spv_assign_type, {
New->getType()},
Args);
1708 U->eraseFromParent();
1711 U->replaceUsesOfWith(Old, New);
1719 Type *NewArgTy =
New->getType();
1721 if (NewArgTy != ExpectedArgTy) {
1724 M, Intrinsic::spv_abort, {NewArgTy});
1734 "aggregate PHI/select/freeze should have been mutated to value-id "
1736 U->replaceUsesOfWith(Old, New);
1741 New->copyMetadata(*Old);
1743 TypeValidated.
erase(Old);
1749 bool HasPoisonExt) {
1756 LLVM_DEBUG(
dbgs() <<
"SPV_KHR_poison_freeze is not enabled. Poison is "
1757 "lowered as undef\n");
1759 Intrinsic::ID IID = AsPoison ? Intrinsic::spv_poison : Intrinsic::spv_undef;
1760 Type *Ty = UV->getType();
1766 AsPoison ?
B.CreateIntrinsicWithoutFolding(IID, {
B.getInt32Ty()}, {})
1767 :
B.CreateIntrinsicWithoutFolding(IID, {});
1768 AggrConsts[
Call] = UV;
1769 AggrConstTypes[
Call] = Ty;
1774 return B.CreateIntrinsic(IID, {Ty}, {});
1781void SPIRVEmitIntrinsicsImpl::preprocessUndefsAndPoisons(
IRBuilder<> &
B) {
1786 SmallVector<Instruction *, 16> Insts;
1790 for (Instruction *
I : Insts) {
1791 bool BPrepared =
false;
1793 for (
unsigned Idx = 0;
Idx <
I->getNumOperands(); ++
Idx) {
1797 bool IsScalar = !
Op->getType()->isAggregateType();
1800 if (IsScalar && !AsPoison)
1804 if (IsScalar && Phi)
1805 B.SetInsertPoint(
Phi->getIncomingBlock(Idx)->getTerminator());
1806 else if (!BPrepared) {
1810 if (
Value *Repl = lowerUndefOrPoison(
Op,
B, HasPoisonExt))
1811 I->setOperand(Idx, Repl);
1820void SPIRVEmitIntrinsicsImpl::simplifyNullAddrSpaceCasts() {
1824 ASC->replaceAllUsesWith(
1826 ASC->eraseFromParent();
1834 if (!
V->getType()->isAggregateType())
1843 I.getType()->isAggregateType();
1849void SPIRVEmitIntrinsicsImpl::insertCompositeAggregateArms(Instruction *
I,
1852 for (Use &U :
I->operands()) {
1859 B.SetInsertPoint(
Phi->getIncomingBlock(U)->getTerminator());
1864 for (
unsigned Idx = 0,
E = AggrTy->getNumElements(); Idx !=
E; ++Idx) {
1866 Composite =
B.CreateInsertValue(Composite,
Field, Idx);
1872void SPIRVEmitIntrinsicsImpl::preprocessCompositeConstants(
IRBuilder<> &
B) {
1876 std::queue<Instruction *> Worklist;
1880 while (!Worklist.empty()) {
1881 auto *
I = Worklist.front();
1884 bool KeepInst =
false;
1885 for (
const auto &
Op :
I->operands()) {
1887 Type *ResTy =
nullptr;
1890 ResTy = COp->getType();
1902 ResTy =
Op->getType()->isVectorTy() ? COp->getType() :
B.getInt32Ty();
1905 auto PrepareInsert = [&]() {
1908 IsPhi ?
B.SetInsertPointPastAllocas(
I->getParent()->getParent())
1909 :
B.SetInsertPoint(
I);
1914 for (
unsigned i = 0; i < COp->getNumElements(); ++i)
1915 Args.push_back(COp->getElementAsConstant(i));
1921 CE &&
CE->getOpcode() == Instruction::AddrSpaceCast &&
1930 if (
Value *Repl = lowerUndefOrPoison(
Op,
B, HasPoisonExt))
1936 auto *CI =
B.CreateIntrinsicWithoutFolding(
1937 Intrinsic::spv_const_composite, {ResTy}, {
Args});
1941 AggrConsts[CI] = AggrConst;
1942 AggrConstTypes[CI] = deduceNestedTypeHelper(AggrConst,
false);
1954 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {
I->getType()},
1959 unsigned RoundingModeDeco,
1966 ConstantInt::get(Int32Ty, SPIRV::Decoration::FPRoundingMode)),
1975 MDNode *SaturatedConversionNode =
1977 Int32Ty, SPIRV::Decoration::SaturatedConversion))});
1997 MDString *ConstraintString =
2002 for (
unsigned OpIdx = 0; OpIdx <
Call.
arg_size(); OpIdx++)
2006 B.SetInsertPoint(&
Call);
2007 B.CreateIntrinsic(Intrinsic::spv_inline_asm, {
Args});
2012void SPIRVEmitIntrinsicsImpl::useRoundingMode(ConstrainedFPIntrinsic *FPI,
2015 if (!
RM.has_value())
2017 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
2018 switch (
RM.value()) {
2022 case RoundingMode::NearestTiesToEven:
2023 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
2025 case RoundingMode::TowardNegative:
2026 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
2028 case RoundingMode::TowardPositive:
2029 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
2031 case RoundingMode::TowardZero:
2032 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
2034 case RoundingMode::Dynamic:
2035 case RoundingMode::NearestTiesToAway:
2039 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
2045Instruction *SPIRVEmitIntrinsicsImpl::visitSwitchInst(SwitchInst &
I) {
2049 B.SetInsertPoint(&
I);
2050 SmallVector<Value *, 4>
Args;
2052 Args.push_back(
I.getCondition());
2055 for (
auto &Case :
I.cases()) {
2056 Args.push_back(Case.getCaseValue());
2057 BBCases.
push_back(Case.getCaseSuccessor());
2060 CallInst *NewI =
B.CreateIntrinsicWithoutFolding(
2061 Intrinsic::spv_switch, {
I.getOperand(0)->getType()}, {
Args});
2065 I.eraseFromParent();
2068 B.SetInsertPoint(ParentBB);
2069 IndirectBrInst *BrI =
B.CreateIndirectBr(
2072 for (BasicBlock *BBCase : BBCases)
2081Instruction *SPIRVEmitIntrinsicsImpl::visitIntrinsicInst(IntrinsicInst &
I) {
2087 B.SetInsertPoint(&
I);
2089 SmallVector<Value *, 4>
Args;
2090 Args.push_back(
B.getInt1(
true));
2091 Args.push_back(
I.getOperand(0));
2092 Args.push_back(
B.getInt32(0));
2093 for (
unsigned J = 0; J < SGEP->getNumIndices(); ++J)
2094 Args.push_back(SGEP->getIndexOperand(J));
2097 B.CreateIntrinsicWithoutFolding(Intrinsic::spv_gep, Types, Args);
2098 replaceAllUsesWithAndErase(
B, &
I, NewI);
2103SPIRVEmitIntrinsicsImpl::visitGetElementPtrInst(GetElementPtrInst &
I) {
2105 B.SetInsertPoint(&
I);
2110 unsigned N = RetVTy->getNumElements();
2111 Value *PtrOp =
I.getPointerOperand();
2113 Type *ResultPtrTy = RetVTy->getElementType();
2116 Value *InBounds =
B.getInt1(
I.isInBounds());
2117 Type *LanePointeeTy = getGEPType(&
I);
2120 Type *SrcElemTy =
I.getSourceElementType();
2129 for (
unsigned Lane = 0; Lane <
N; ++Lane) {
2130 Value *LaneIdx =
B.getInt32(Lane);
2131 Value *ScalarPtr = PtrOp;
2135 ScalarPtr =
B.CreateIntrinsic(Intrinsic::spv_extractelt, {ExtractTypes},
2139 SmallVector<Value *, 4>
Args;
2140 Args.push_back(InBounds);
2141 Args.push_back(ScalarPtr);
2142 for (
Value *Idx :
I.indices()) {
2150 Args.push_back(visitExtractElementInst(*EI));
2154 Args.push_back(Idx);
2157 Value *ScalarGep =
B.CreateIntrinsic(Intrinsic::spv_gep, GepTypes, Args);
2159 VecResult =
B.CreateInsertElement(VecResult, ScalarGep, LaneIdx);
2163 replaceAllUsesWithAndErase(
B, &
I, NewI);
2181 if (getByteAddressingMultiplier(
I.getSourceElementType())) {
2182 return buildLogicalAccessChainFromGEP(
I);
2187 Value *PtrOp =
I.getPointerOperand();
2188 Type *SrcElemTy =
I.getSourceElementType();
2189 Type *DeducedPointeeTy = deduceElementType(PtrOp,
true);
2192 if (ArrTy->getElementType() == SrcElemTy) {
2194 Type *FirstIdxType =
I.getOperand(1)->getType();
2195 NewIndices.
push_back(ConstantInt::get(FirstIdxType, 0));
2196 for (
Value *Idx :
I.indices())
2200 SmallVector<Value *, 4>
Args;
2201 Args.push_back(
B.getInt1(
I.isInBounds()));
2202 Args.push_back(
I.getPointerOperand());
2205 Instruction *NewI =
B.CreateIntrinsicWithoutFolding(Intrinsic::spv_gep,
2207 replaceAllUsesWithAndErase(
B, &
I, NewI);
2214 SmallVector<Value *, 4>
Args;
2215 Args.push_back(
B.getInt1(
I.isInBounds()));
2218 B.CreateIntrinsicWithoutFolding(Intrinsic::spv_gep, {
Types}, {
Args});
2219 replaceAllUsesWithAndErase(
B, &
I, NewI);
2223Instruction *SPIRVEmitIntrinsicsImpl::visitBitCastInst(BitCastInst &
I) {
2225 B.SetInsertPoint(&
I);
2234 I.eraseFromParent();
2241 B.CreateIntrinsicWithoutFolding(Intrinsic::spv_bitcast, {
Types}, {
Args});
2242 replaceAllUsesWithAndErase(
B, &
I, NewI);
2246void SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeTargetExt(
2253 if (ElemTy != AssignedType)
2266 if (CurrentType == AssignedType)
2273 " for value " +
V->getName(),
2282void SPIRVEmitIntrinsicsImpl::replacePointerOperandWithPtrCast(
2283 Instruction *
I,
Value *Pointer,
Type *ExpectedElementType,
2288 Type *PointerElemTy = deduceElementTypeHelper(Pointer,
false);
2289 if (PointerElemTy == ExpectedElementType ||
2294 Value *ExpectedElementVal =
2296 MetadataAsValue *VMD =
buildMD(ExpectedElementVal);
2298 bool FirstPtrCastOrAssignPtrType =
true;
2304 for (
auto User :
Pointer->users()) {
2307 (
II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
2308 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
2309 II->getOperand(0) != Pointer)
2314 FirstPtrCastOrAssignPtrType =
false;
2315 if (
II->getOperand(1) != VMD ||
2322 if (
II->getIntrinsicID() != Intrinsic::spv_ptrcast)
2327 if (
II->getParent() !=
I->getParent())
2330 I->setOperand(OperandToReplace,
II);
2345 if (FirstPtrCastOrAssignPtrType) {
2350 }
else if (isTodoType(Pointer)) {
2351 eraseTodoType(Pointer);
2359 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
2360 std::make_pair(
I, Pointer)};
2362 propagateElemType(Pointer, PrevElemTy, VisitedSubst);
2375 B.CreateIntrinsicWithoutFolding(Intrinsic::spv_ptrcast, {
Types},
Args);
2381void SPIRVEmitIntrinsicsImpl::insertPtrCastOrAssignTypeInstr(Instruction *
I,
2386 replacePointerOperandWithPtrCast(
2387 I,
SI->getValueOperand(), IntegerType::getInt8Ty(CurrF->
getContext()),
2393 Type *OpTy =
Op->getType();
2396 if (
auto It = AggrConstTypes.
find(OpI); It != AggrConstTypes.
end())
2399 if (OpTy ==
Op->getType())
2400 OpTy = deduceElementTypeByValueDeep(OpTy,
Op,
false);
2401 replacePointerOperandWithPtrCast(
I, Pointer, OpTy, 1,
B);
2406 Type *OpTy = LI->getType();
2417 Type *NewOpTy = OpTy;
2418 OpTy = deduceElementTypeByValueDeep(OpTy, LI,
false);
2419 if (OpTy == NewOpTy)
2420 insertTodoType(Pointer);
2423 replacePointerOperandWithPtrCast(
I, Pointer, OpTy, 0,
B);
2428 Type *OpTy =
nullptr;
2440 OpTy = GEPI->getSourceElementType();
2442 replacePointerOperandWithPtrCast(
I, Pointer, OpTy, 0,
B);
2444 insertTodoType(Pointer);
2450 Value *
Op = IVI->getInsertedValueOperand();
2452 replacePointerOperandWithPtrCast(
2466 std::string DemangledName =
2470 bool HaveTypes =
false;
2471 for (
unsigned OpIdx = 0; OpIdx < CalledF->
arg_size(); ++OpIdx) {
2489 for (User *U : CalledArg->
users()) {
2491 if ((ElemTy = deduceElementTypeHelper(Inst,
false)) !=
nullptr)
2497 HaveTypes |= ElemTy !=
nullptr;
2502 if (DemangledName.empty() && !HaveTypes)
2506 TM.
getSubtarget<SPIRVSubtarget>(*CalledF).getPreferredInstructionSet();
2508 bool ParseBuiltinTypes =
2509 !DemangledName.empty() &&
2510 (!CalledF->
isDeclaration() || SPIRV::isBuiltin(DemangledName, Set));
2511 for (
unsigned OpIdx = 0; OpIdx < CI->
arg_size(); OpIdx++) {
2526 Type *ExpectedType =
2527 OpIdx < CalledArgTys.
size() ? CalledArgTys[OpIdx] :
nullptr;
2528 if (!ExpectedType && ParseBuiltinTypes)
2529 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2530 DemangledName, OpIdx,
I->getContext());
2531 if (!ExpectedType || ExpectedType->
isVoidTy())
2539 replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx,
B);
2544SPIRVEmitIntrinsicsImpl::visitInsertElementInst(InsertElementInst &
I) {
2547 if (
isVector1(
I.getType()) && !CanUseAnyVectorRank)
2551 I.getOperand(1)->getType(),
2552 I.getOperand(2)->getType()};
2554 B.SetInsertPoint(&
I);
2556 Instruction *NewI =
B.CreateIntrinsicWithoutFolding(Intrinsic::spv_insertelt,
2558 replaceAllUsesWithAndErase(
B, &
I, NewI);
2563SPIRVEmitIntrinsicsImpl::visitExtractElementInst(ExtractElementInst &
I) {
2566 if (
isVector1(
I.getVectorOperandType()) && !CanUseAnyVectorRank)
2570 B.SetInsertPoint(&
I);
2572 I.getIndexOperand()->getType()};
2573 SmallVector<Value *, 2>
Args = {
I.getVectorOperand(),
I.getIndexOperand()};
2574 Instruction *NewI =
B.CreateIntrinsicWithoutFolding(Intrinsic::spv_extractelt,
2576 replaceAllUsesWithAndErase(
B, &
I, NewI);
2584void SPIRVEmitIntrinsicsImpl::scalarizeVectorPtrCasts(
Function &
F) {
2588 if (
ST.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
2591 SmallVector<Instruction *, 8> ToErase;
2596 if (!VecTy || !VecTy->getElementType()->isPointerTy())
2600 unsigned NumElems = VecTy->getNumElements();
2604 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2607 ElemPtr =
B.CreateExtractElement(PTI->getOperand(0),
B.getInt32(Idx));
2608 Value *Conv =
B.CreatePtrToInt(ElemPtr, ElemIntTy);
2609 Result =
B.CreateInsertElement(Result, Conv,
B.getInt32(Idx));
2612 PTI->replaceAllUsesWith(Result);
2616 if (!VecTy || !VecTy->getElementType()->isPointerTy())
2622 bool AllUsersExtract =
llvm::all_of(ITP->users(), [&](User *U) {
2623 auto *EE = dyn_cast<ExtractElementInst>(U);
2624 return EE && isa<ConstantInt>(EE->getIndexOperand());
2626 if (!AllUsersExtract)
2630 Type *ElemPtrTy = VecTy->getElementType();
2636 B.SetInsertPoint(EE);
2638 B.CreateExtractElement(ITP->getOperand(0),
B.getInt32(Idx));
2639 Value *Conv =
B.CreateIntToPtr(Elem, ElemPtrTy);
2648 if (ToErase.
empty())
2651 for (Instruction *
I : ToErase) {
2652 for (
Value *
Op :
I->operand_values())
2654 I->eraseFromParent();
2659Instruction *SPIRVEmitIntrinsicsImpl::visitInsertValueInst(InsertValueInst &
I) {
2661 B.SetInsertPoint(&
I);
2664 Value *AggregateOp =
I.getAggregateOperand();
2668 Args.push_back(AggregateOp);
2669 Args.push_back(
I.getInsertedValueOperand());
2670 for (
auto &
Op :
I.indices())
2671 Args.push_back(
B.getInt32(
Op));
2673 B.CreateIntrinsicWithoutFolding(Intrinsic::spv_insertv, {
Types}, {
Args});
2674 replaceMemInstrUses(&
I, NewI,
B);
2679SPIRVEmitIntrinsicsImpl::visitExtractValueInst(ExtractValueInst &
I) {
2681 B.SetInsertPoint(&
I);
2682 if (
I.getAggregateOperand()->getType()->isAggregateType()) {
2691 for (
auto &
Op :
I.indices())
2692 Args.push_back(
B.getInt32(
Op));
2693 Instruction *NewI =
B.CreateIntrinsicWithoutFolding(Intrinsic::spv_extractv,
2694 {
I.getType()}, {
Args});
2700 any_of(
I.users(), [](User *U) { return isa<InsertValueInst>(U); })) {
2701 AggrConstTypes[NewI] =
I.getType();
2703 replaceMemInstrUses(&
I, NewI,
B);
2706 replaceAllUsesWithAndErase(
B, &
I, NewI);
2710 for (
const Use &U : NewI->
uses()) {
2711 User *Usr =
U.getUser();
2713 if (RI->getFunction()->getReturnType() != NewI->
getType()) {
2724 if (ArgNo < FT->getNumParams() &&
2725 !FT->getParamType(ArgNo)->isAggregateType()) {
2734Instruction *SPIRVEmitIntrinsicsImpl::visitLoadInst(LoadInst &
I) {
2735 if (!
I.getType()->isAggregateType())
2738 B.SetInsertPoint(&
I);
2739 TrackConstants =
false;
2744 unsigned IntrinsicId;
2745 SmallVector<Value *, 4>
Args = {
I.getPointerOperand(),
B.getInt16(Flags)};
2746 if (!
I.isAtomic()) {
2747 IntrinsicId = Intrinsic::spv_load;
2748 Args.push_back(
B.getInt32(
I.getAlign().value()));
2750 IntrinsicId = Intrinsic::spv_atomic_load;
2751 Args.push_back(
B.getInt8(
static_cast<uint8_t
>(
I.getOrdering())));
2753 CallInst *NewI =
B.CreateIntrinsicWithoutFolding(
2754 IntrinsicId, {
I.getOperand(0)->getType()},
Args);
2756 replaceMemInstrUses(&
I, NewI,
B);
2760Instruction *SPIRVEmitIntrinsicsImpl::visitStoreInst(StoreInst &
I) {
2764 B.SetInsertPoint(&
I);
2765 TrackConstants =
false;
2769 auto *PtrOp =
I.getPointerOperand();
2771 if (
I.getValueOperand()->getType()->isAggregateType()) {
2779 "Unexpected argument of aggregate type, should be spv_extractv!");
2783 unsigned IntrinsicId;
2784 SmallVector<Value *, 4>
Args = {
I.getValueOperand(), PtrOp,
2786 if (!
I.isAtomic()) {
2787 IntrinsicId = Intrinsic::spv_store;
2788 Args.push_back(
B.getInt32(
I.getAlign().value()));
2790 IntrinsicId = Intrinsic::spv_atomic_store;
2791 Args.push_back(
B.getInt8(
static_cast<uint8_t
>(
I.getOrdering())));
2794 IntrinsicId, {
I.getValueOperand()->getType(), PtrOp->
getType()},
Args);
2796 I.eraseFromParent();
2800Instruction *SPIRVEmitIntrinsicsImpl::visitAllocaInst(AllocaInst &
I) {
2801 Value *ArraySize =
nullptr;
2802 if (
I.isArrayAllocation()) {
2805 SPIRV::Extension::SPV_INTEL_variable_length_array))
2807 "array allocation: this instruction requires the following "
2808 "SPIR-V extension: SPV_INTEL_variable_length_array",
2810 ArraySize =
I.getArraySize();
2813 B.SetInsertPoint(&
I);
2814 TrackConstants =
false;
2815 Type *PtrTy =
I.getType();
2818 ?
B.CreateIntrinsicWithoutFolding(
2819 Intrinsic::spv_alloca_array, {PtrTy, ArraySize->
getType()},
2820 {ArraySize,
B.getInt32(
I.getAlign().value())})
2821 :
B.CreateIntrinsicWithoutFolding(
Intrinsic::spv_alloca, {PtrTy},
2822 {
B.getInt32(
I.getAlign().value())});
2823 replaceAllUsesWithAndErase(
B, &
I, NewI);
2828SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &
I) {
2829 assert(
I.getType()->isAggregateType() &&
"Aggregate result is expected");
2831 B.SetInsertPoint(&
I);
2834 Args.push_back(
B.getInt32(
static_cast<uint32_t
>(
2838 const SPIRVSubtarget &
ST = TM.
getSubtarget<SPIRVSubtarget>(*
I.getFunction());
2839 unsigned AS =
I.getPointerOperand()->getType()->getPointerAddressSpace();
2840 uint32_t ScSem =
static_cast<uint32_t
>(
2849 Intrinsic::spv_cmpxchg, {
I.getPointerOperand()->getType()}, {
Args});
2850 replaceMemInstrUses(&
I, NewI,
B);
2854Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &
I) {
2855 auto Op =
I.getOperation();
2866 B.SetInsertPoint(&
I);
2868 const SPIRVSubtarget &
ST = TM.
getSubtarget<SPIRVSubtarget>(*
I.getFunction());
2869 unsigned AS =
I.getPointerOperand()->getType()->getPointerAddressSpace();
2871 uint32_t
Scope =
static_cast<uint32_t
>(
2872 getMemScope(
M->getTargetTriple(),
I.getContext(),
I.getSyncScopeID()));
2873 uint32_t ScSem =
static_cast<uint32_t
>(
2876 M->getTargetTriple(),
2880 ?
"__translate_spirv_atomic_uinc_wrap"
2881 :
"__translate_spirv_atomic_udec_wrap");
2883 Type *ValTy =
I.getValOperand()->getType();
2884 Type *PtrTy =
I.getPointerOperand()->getType();
2886 raw_svector_ostream OS(FuncName);
2887 OS <<
"_p" << AS <<
"_";
2889 OS <<
"v" << VecTy->getNumElements();
2892 Type *Int32Ty =
B.getInt32Ty();
2893 Type *BoolTy =
B.getInt1Ty();
2895 ValTy, BoolTy, BoolTy};
2896 FunctionType *FT = FunctionType::get(ValTy, ArgTys,
false);
2897 FunctionCallee
FC =
M->getOrInsertFunction(FuncName, FT);
2901 I.getPointerOperand(),
B.getInt32(Scope),
2902 B.getInt32(MemSem),
I.getValOperand(),
2903 B.getInt1(
I.isVolatile()),
B.getInt1(
I.isElementwise())};
2904 CallInst *CI =
B.CreateCall(FC, Args);
2910 replaceAllUsesWithAndErase(
B, &
I, CI);
2919 case Intrinsic::spv_abort:
2921 case Intrinsic::trap:
2922 case Intrinsic::ubsantrap:
2924 return ST.canUseExtension(SPIRV::Extension::SPV_KHR_abort);
2940 [&ST](
const Instruction &
II) { return isAbortCall(II, ST); }) &&
2941 "abort-like call must be the last non-debug instruction before its "
2942 "block's terminator");
2946Instruction *SPIRVEmitIntrinsicsImpl::visitUnreachableInst(UnreachableInst &
I) {
2947 const SPIRVSubtarget &
ST = TM.
getSubtarget<SPIRVSubtarget>(*
I.getFunction());
2951 B.CreateIntrinsic(Intrinsic::spv_unreachable, {});
2958 return Name ==
"llvm.compiler.used" || Name ==
"llvm.used";
2972 while (!Stack.empty()) {
2973 const Value *
V = Stack.pop_back_val();
2974 if (!Visited.
insert(
V).second)
2982 Stack.append(
C->user_begin(),
C->user_end());
2998 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2999 if (UserFunctions.contains(
F))
3004 if (!UserFunctions.empty())
3009 const Module &M = *
F->getParent();
3010 const Function &FirstDefinition = *M.getFunctionDefs().
begin();
3011 return F == &FirstDefinition;
3014Value *SPIRVEmitIntrinsicsImpl::buildSpvUndefComposite(
Type *AggrTy,
3016 auto MakeLeaf = [&](
Type *ElemTy) -> Instruction * {
3017 CallInst *Leaf =
B.CreateIntrinsicWithoutFolding(Intrinsic::spv_undef, {});
3019 AggrConstTypes[Leaf] = ElemTy;
3022 SmallVector<Value *, 4> Elems;
3024 Elems.
assign(ArrTy->getNumElements(), MakeLeaf(ArrTy->getElementType()));
3027 DenseMap<Type *, Instruction *> LeafByType;
3028 for (
unsigned I = 0;
I < StructTy->getNumElements(); ++
I) {
3030 auto &
Entry = LeafByType[ElemTy];
3032 Entry = MakeLeaf(ElemTy);
3036 CallInst *Composite =
B.CreateIntrinsicWithoutFolding(
3037 Intrinsic::spv_const_composite, {
B.getInt32Ty()}, Elems);
3039 AggrConstTypes[Composite] = AggrTy;
3048void SPIRVEmitIntrinsicsImpl::reconstructAggregateReturns(
Function &Func,
3053 for (BasicBlock &BB : Func) {
3057 Value *RetVal = RI->getReturnValue();
3064 B.SetInsertPoint(RI);
3067 Value *Elt =
B.CreateExtractValue(RetVal,
I);
3068 Rebuilt =
B.CreateInsertValue(Rebuilt, Elt,
I);
3070 RI->setOperand(0, Rebuilt);
3074void SPIRVEmitIntrinsicsImpl::processGlobalValue(GlobalVariable &GV,
3084 deduceElementTypeHelper(&GV,
false);
3089 Value *InitOp = Init;
3096 CallInst *
Call =
B.CreateIntrinsicWithoutFolding(Intrinsic::spv_poison,
3097 {
B.getInt32Ty()}, {});
3102 InitOp = buildSpvUndefComposite(Init->
getType(),
B);
3107 CallInst *InitInst =
B.CreateIntrinsicWithoutFolding(
3108 Intrinsic::spv_init_global, {GV.
getType(), Ty}, {&GV,
Const});
3114 B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.
getType(), &GV);
3120bool SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeIntrs(Instruction *
I,
3122 bool UnknownElemTypeI8) {
3128 if (
Type *ElemTy = deduceElementType(
I, UnknownElemTypeI8)) {
3135void SPIRVEmitIntrinsicsImpl::insertAssignTypeIntrs(Instruction *
I,
3138 static StringMap<unsigned> ResTypeWellKnown = {
3139 {
"async_work_group_copy", WellKnownTypes::Event},
3140 {
"async_work_group_strided_copy", WellKnownTypes::Event},
3141 {
"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
3145 bool IsKnown =
false;
3150 std::string DemangledName =
3153 if (DemangledName.length() > 0)
3155 SPIRV::lookupBuiltinNameHelper(DemangledName, &DecorationId);
3156 auto ResIt = ResTypeWellKnown.
find(DemangledName);
3157 if (ResIt != ResTypeWellKnown.
end()) {
3160 switch (ResIt->second) {
3161 case WellKnownTypes::Event:
3164 CanUseAnyVectorRank);
3169 switch (DecorationId) {
3172 case FPDecorationId::SAT:
3175 case FPDecorationId::RTE:
3177 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTE,
B);
3179 case FPDecorationId::RTZ:
3181 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTZ,
B);
3183 case FPDecorationId::RTP:
3185 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTP,
B);
3187 case FPDecorationId::RTN:
3189 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTN,
B);
3195 Type *Ty =
I->getType();
3198 Type *TypeToAssign = Ty;
3201 auto It = AggrConstTypes.
find(
II);
3202 if (It == AggrConstTypes.
end())
3204 TypeToAssign = It->second;
3205 }
else if (
II->getIntrinsicID() == Intrinsic::spv_poison) {
3206 if (
auto It = AggrConstTypes.
find(
II); It != AggrConstTypes.
end())
3207 TypeToAssign = It->second;
3209 }
else if (
auto It = AggrConstTypes.
find(
I); It != AggrConstTypes.
end())
3210 TypeToAssign = It->second;
3214 for (
const auto &
Op :
I->operands()) {
3222 Type *OpTy =
Op->getType();
3224 CallInst *AssignCI =
3229 Type *OpTy =
Op->getType();
3245 Intrinsic::spv_assign_type, {OpTy},
3255bool SPIRVEmitIntrinsicsImpl::shouldTryToAddMemAliasingDecoration(
3256 Instruction *Inst) {
3258 if (!STI->
canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
3268void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *
I,
3270 if (MDNode *MD =
I->getMetadata(
"spirv.Decorations")) {
3272 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {
I->getType()},
3277 auto processMemAliasingDecoration = [&](
unsigned Kind) {
3278 if (MDNode *AliasListMD =
I->getMetadata(Kind)) {
3279 if (shouldTryToAddMemAliasingDecoration(
I)) {
3280 uint32_t Dec =
Kind == LLVMContext::MD_alias_scope
3281 ? SPIRV::Decoration::AliasScopeINTEL
3282 : SPIRV::Decoration::NoAliasINTEL;
3284 I, ConstantInt::get(
B.getInt32Ty(), Dec),
3287 B.CreateIntrinsic(Intrinsic::spv_assign_aliasing_decoration,
3288 {
I->getType()}, {
Args});
3292 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
3293 processMemAliasingDecoration(LLVMContext::MD_noalias);
3296 if (MDNode *MD =
I->getMetadata(LLVMContext::MD_fpmath)) {
3298 bool AllowFPMaxError =
3300 if (!AllowFPMaxError)
3304 B.CreateIntrinsic(Intrinsic::spv_assign_fpmaxerror_decoration,
3310 LLVMContext &Ctx =
B.getContext();
3312 B.getInt32Ty(), SPIRV::Decoration::AuxDataInstructionMetadata));
3315 if (
I->hasMetadata(
"amdgpu.no.fine.grained.memory"))
3317 Ctx, {AuxMD,
MDString::get(Ctx,
"amdgpu.no.fine.grained.memory")}));
3318 if (
I->hasMetadata(
"amdgpu.no.remote.memory"))
3320 Ctx, {AuxMD,
MDString::get(Ctx,
"amdgpu.no.remote.memory")}));
3321 if (
I->hasMetadata(LLVMContext::MD_atomic_ignore_denormal_mode))
3323 Ctx, {AuxMD,
MDString::get(Ctx,
"atomic.ignore.denormal.mode")}));
3326 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {
I->getType()},
3335 &FPFastMathDefaultInfoMap,
3337 auto it = FPFastMathDefaultInfoMap.
find(
F);
3338 if (it != FPFastMathDefaultInfoMap.
end())
3346 SPIRV::FPFastMathMode::None);
3348 SPIRV::FPFastMathMode::None);
3350 SPIRV::FPFastMathMode::None);
3351 return FPFastMathDefaultInfoMap[
F] = std::move(FPFastMathDefaultInfoVec);
3357 size_t BitWidth = Ty->getScalarSizeInBits();
3361 assert(Index >= 0 && Index < 3 &&
3362 "Expected FPFastMathDefaultInfo for half, float, or double");
3363 assert(FPFastMathDefaultInfoVec.
size() == 3 &&
3364 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3365 return FPFastMathDefaultInfoVec[Index];
3368void SPIRVEmitIntrinsicsImpl::insertConstantsForFPFastMathDefault(
Module &M) {
3370 if (!
ST->canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2))
3379 auto Node =
M.getNamedMetadata(
"spirv.ExecutionMode");
3381 if (!
M.getNamedMetadata(
"opencl.enable.FP_CONTRACT")) {
3389 ConstantInt::get(Type::getInt32Ty(
M.getContext()), 0);
3392 [[maybe_unused]] GlobalVariable *GV =
3393 new GlobalVariable(M,
3394 Type::getInt32Ty(
M.getContext()),
3408 MapVector<Function *, SPIRV::FPFastMathDefaultInfoVector>
3409 FPFastMathDefaultInfoMap;
3411 for (
unsigned i = 0; i <
Node->getNumOperands(); i++) {
3420 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3422 "Expected 4 operands for FPFastMathDefault");
3428 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3430 SPIRV::FPFastMathDefaultInfo &
Info =
3433 Info.FPFastMathDefault =
true;
3434 }
else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3436 "Expected no operands for ContractionOff");
3440 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3442 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3443 Info.ContractionOff =
true;
3445 }
else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3447 "Expected 1 operand for SignedZeroInfNanPreserve");
3448 unsigned TargetWidth =
3453 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3457 assert(Index >= 0 && Index < 3 &&
3458 "Expected FPFastMathDefaultInfo for half, float, or double");
3459 assert(FPFastMathDefaultInfoVec.
size() == 3 &&
3460 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3461 FPFastMathDefaultInfoVec[
Index].SignedZeroInfNanPreserve =
true;
3465 DenseMap<unsigned, GlobalVariable *> GlobalVars;
3466 for (
auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
3467 if (FPFastMathDefaultInfoVec.
empty())
3470 for (
const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3471 assert(
Info.Ty &&
"Expected target type for FPFastMathDefaultInfo");
3474 if (Flags == SPIRV::FPFastMathMode::None && !
Info.ContractionOff &&
3475 !
Info.SignedZeroInfNanPreserve && !
Info.FPFastMathDefault)
3479 if (
Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
3481 "and AllowContract");
3483 if (
Info.SignedZeroInfNanPreserve &&
3485 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
3486 SPIRV::FPFastMathMode::NSZ))) {
3487 if (
Info.FPFastMathDefault)
3489 "SignedZeroInfNanPreserve but at least one of "
3490 "NotNaN/NotInf/NSZ is enabled.");
3493 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
3494 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
3495 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
3497 "AllowTransform requires AllowReassoc and "
3498 "AllowContract to be set.");
3501 auto it = GlobalVars.
find(Flags);
3502 GlobalVariable *GV =
nullptr;
3503 if (it != GlobalVars.
end()) {
3509 ConstantInt::get(Type::getInt32Ty(
M.getContext()), Flags);
3512 GV =
new GlobalVariable(M,
3513 Type::getInt32Ty(
M.getContext()),
3518 GlobalVars[
Flags] = GV;
3524void SPIRVEmitIntrinsicsImpl::processInstrAfterVisit(Instruction *
I,
3527 bool IsConstComposite =
3528 II &&
II->getIntrinsicID() == Intrinsic::spv_const_composite;
3529 if (IsConstComposite && TrackConstants) {
3531 auto t = AggrConsts.
find(
I);
3535 {
II->getType(),
II->getType()}, t->second,
I, {},
B);
3537 NewOp->setArgOperand(0,
I);
3540 for (
const auto &
Op :
I->operands()) {
3544 unsigned OpNo =
Op.getOperandNo();
3545 if (
II && ((
II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
3546 (!
II->isBundleOperand(OpNo) &&
3547 II->paramHasAttr(OpNo, Attribute::ImmArg))))
3551 IsPhi ?
B.SetInsertPointPastAllocas(
I->getParent()->getParent())
3552 :
B.SetInsertPoint(
I);
3555 Type *OpTy =
Op->getType();
3563 {OpTy, OpTyVal->
getType()},
Op, OpTyVal, {},
B);
3565 if (!IsConstComposite &&
isPointerTy(OpTy) && OpElemTy !=
nullptr &&
3566 OpElemTy != IntegerType::getInt8Ty(
I->getContext())) {
3568 SmallVector<Value *, 2>
Args = {
3572 CallInst *PtrCasted =
B.CreateIntrinsicWithoutFolding(
3578 I->setOperand(OpNo, NewOp);
3584Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
Function *
F,
3586 SmallPtrSet<Function *, 0> FVisited;
3587 return deduceFunParamElementType(
F, OpIdx, FVisited);
3590Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
3591 Function *
F,
unsigned OpIdx, SmallPtrSetImpl<Function *> &FVisited) {
3593 if (!FVisited.
insert(
F).second)
3596 SmallPtrSet<Value *, 0> Visited;
3599 for (User *U :
F->users()) {
3601 if (!CI || OpIdx >= CI->
arg_size())
3611 if (
Type *Ty = deduceElementTypeHelper(OpArg, Visited,
false))
3614 for (User *OpU : OpArg->
users()) {
3616 if (!Inst || Inst == CI)
3619 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited,
false))
3626 if (FVisited.
find(OuterF) != FVisited.
end())
3628 for (
unsigned i = 0; i < OuterF->
arg_size(); ++i) {
3629 if (OuterF->
getArg(i) == OpArg) {
3630 Lookup.push_back(std::make_pair(OuterF, i));
3637 for (
auto &Pair :
Lookup) {
3638 if (
Type *Ty = deduceFunParamElementType(Pair.first, Pair.second, FVisited))
3645void SPIRVEmitIntrinsicsImpl::processParamTypesByFunHeader(
Function *
F,
3647 B.SetInsertPointPastAllocas(
F);
3648 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
3654 for (User *U : Arg->
users()) {
3656 if (
GEP &&
GEP->getPointerOperand() == Arg) {
3674 for (User *U :
F->users()) {
3676 if (!CI || OpIdx >= CI->
arg_size())
3690 for (User *U : Arg->
users()) {
3694 CI->
getParent()->getParent() == CurrF) {
3696 deduceOperandElementTypeFunctionPointer(CI,
Ops, ElemTy,
false);
3708 B.SetInsertPointPastAllocas(
F);
3709 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
3714 if (!ElemTy && (ElemTy = deduceFunParamElementType(
F, OpIdx)) !=
nullptr) {
3716 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3720 propagateElemType(Arg, IntegerType::getInt8Ty(
F->getContext()),
3732 bool IsNewFTy =
false;
3748bool SPIRVEmitIntrinsicsImpl::processFunctionPointers(
Module &M) {
3751 if (
F.isIntrinsic())
3753 if (
F.isDeclaration()) {
3754 for (User *U :
F.users()) {
3757 UF &&
UF->hasPersonalityFn() &&
UF->getPersonalityFn() == &
F)
3771 for (User *U :
F.users()) {
3773 if (!
II ||
II->arg_size() != 3 ||
II->getOperand(0) != &
F)
3775 if (
II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
3776 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
3784 if (Worklist.
empty())
3787 LLVMContext &Ctx =
M.getContext();
3794 for (
const auto &Arg :
F->args())
3797 IRB.CreateCall(
F, Args)->setTailCall(
true);
3799 IRB.CreateRetVoid();
3805void SPIRVEmitIntrinsicsImpl::applyDemangledPtrArgTypes(
IRBuilder<> &
B) {
3806 DenseMap<Function *, CallInst *> Ptrcasts;
3807 for (
auto It : FDeclPtrTys) {
3809 for (
auto *U :
F->users()) {
3814 for (
auto [Idx, ElemTy] : It.second) {
3822 B.SetInsertPointPastAllocas(Arg->
getParent());
3829 replaceUsesOfWithSpvPtrcast(
3830 Param,
normalizeType(ElemTy, CanUseAnyVectorRank), CI, Ptrcasts);
3839 .getFirstNonPHIOrAlloca());
3857 FDeclPtrTys.
clear();
3860GetElementPtrInst *SPIRVEmitIntrinsicsImpl::simplifyZeroLengthArrayGepInst(
3861 GetElementPtrInst *
GEP) {
3868 Type *SrcTy =
GEP->getSourceElementType();
3869 SmallVector<Value *, 8> Indices(
GEP->indices());
3871 if (ArrTy && ArrTy->getNumElements() == 0 &&
match(Indices[0],
m_Zero())) {
3872 Indices.erase(Indices.begin());
3873 SrcTy = ArrTy->getElementType();
3875 GEP->getNoWrapFlags(),
"",
3876 GEP->getIterator());
3881void SPIRVEmitIntrinsicsImpl::emitUnstructuredLoopControls(
Function &
F,
3888 if (
ST->canUseExtension(
3889 SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3890 for (BasicBlock &BB :
F) {
3892 MDNode *LoopMD =
Term->getMetadata(LLVMContext::MD_loop);
3896 SmallVector<unsigned, 1>
Ops =
3898 unsigned LC =
Ops[0];
3899 if (LC == SPIRV::LoopControl::None)
3903 B.SetInsertPoint(Term);
3904 SmallVector<Value *, 4> IntrArgs;
3905 for (
unsigned Op :
Ops)
3907 B.CreateIntrinsic(Intrinsic::spv_loop_control_intel, IntrArgs);
3919 SmallPtrSet<BasicBlock *, 8> UsedMergeBlocks;
3926 SmallVector<unsigned, 1> LoopControlOps =
3928 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3941 for (BasicBlock *BB : {Latch, Header}) {
3945 if (!
L->contains(Succ) && !UsedMergeBlocks.
contains(Succ))
3948 if (!MergeBlock || !UsedMergeBlocks.
insert(MergeBlock).second)
3951 B.SetInsertPoint(Header->getTerminator());
3954 SmallVector<Value *, 4>
Args = {MergeAddress, ContinueAddress};
3955 for (
unsigned Imm : LoopControlOps)
3956 Args.emplace_back(
B.getInt32(
Imm));
3957 B.CreateIntrinsic(Intrinsic::spv_loop_merge, {
Args});
3961bool SPIRVEmitIntrinsicsImpl::runOnFunction(
Function &Func) {
3962 if (
Func.isDeclaration())
3971 GR =
ST.getSPIRVGlobalRegistry();
3975 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers);
3977 CanUseAnyVectorRank =
3978 ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector);
3982 AggrConstTypes.
clear();
3985 scalarizeVectorPtrCasts(Func);
3986 processParamTypesByFunHeader(CurrF,
B);
3990 SmallPtrSet<Instruction *, 4> DeadInsts;
3993 Type *ElTy =
SI->getValueOperand()->getType();
4008 normalizeType(SGEP->getResultElementType(), CanUseAnyVectorRank));
4012 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(
GEP);
4014 GEP->replaceAllUsesWith(NewGEP);
4018 if (
Type *GepTy = getGEPType(
GEP))
4022 for (
auto *
I : DeadInsts) {
4023 assert(
I->use_empty() &&
"Dead instruction should not have any uses left");
4024 I->eraseFromParent();
4026 if (AccessChainErrors)
4029 B.SetInsertPoint(
Func.getEntryBlock().begin());
4030 for (
auto &GV :
Func.getParent()->globals())
4031 processGlobalValue(GV,
B);
4033 reconstructAggregateReturns(Func,
B);
4034 preprocessUndefsAndPoisons(
B);
4035 simplifyNullAddrSpaceCasts();
4036 preprocessCompositeConstants(
B);
4044 Type *I32Ty =
B.getInt32Ty();
4049 insertCompositeAggregateArms(&
I,
B);
4050 AggrConstTypes[&
I] =
I.getType();
4051 I.mutateType(I32Ty);
4054 preprocessBoolVectorBitcasts(Func);
4055 SmallVector<Instruction *> Worklist(
4058 applyDemangledPtrArgTypes(
B);
4061 for (
auto &
I : Worklist) {
4063 if (isConvergenceIntrinsic(
I))
4066 bool Postpone = insertAssignPtrTypeIntrs(
I,
B,
false);
4068 insertAssignTypeIntrs(
I,
B);
4069 insertPtrCastOrAssignTypeInstr(
I,
B);
4074 insertAssignPtrTypeIntrs(
I,
B,
true);
4077 useRoundingMode(FPI,
B);
4082 SmallPtrSet<Instruction *, 4> IncompleteRets;
4084 deduceOperandElementType(&
I, &IncompleteRets);
4088 for (BasicBlock &BB : Func)
4089 for (PHINode &Phi : BB.
phis())
4091 deduceOperandElementType(&Phi,
nullptr);
4093 for (
auto *
I : Worklist) {
4094 TrackConstants =
true;
4100 if (AccessChainErrors)
4106 if (isConvergenceIntrinsic(
I))
4110 processInstrAfterVisit(
I,
B);
4113 emitUnstructuredLoopControls(Func,
B);
4119bool SPIRVEmitIntrinsicsImpl::postprocessTypes(
Module &M) {
4120 if (!GR || TodoTypeSz == 0)
4123 unsigned SzTodo = TodoTypeSz;
4124 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
4131 if (!KnownTy || !AssignCI)
4137 SmallPtrSet<Value *, 0> Visited;
4138 if (
Type *ElemTy = deduceElementTypeHelper(
Op, Visited,
false,
true)) {
4139 if (ElemTy != KnownTy) {
4140 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
4141 propagateElemType(CI, ElemTy, VisitedSubst);
4148 if (
Op->hasUseList()) {
4149 for (User *U :
Op->users()) {
4156 if (TodoTypeSz == 0)
4161 SmallPtrSet<Instruction *, 4> IncompleteRets;
4163 auto It = ToProcess.
find(&
I);
4164 if (It == ToProcess.
end())
4166 It->second.remove_if([
this](
Value *
V) {
return !isTodoType(
V); });
4167 if (It->second.size() == 0)
4169 deduceOperandElementType(&
I, &IncompleteRets, &It->second,
true);
4170 if (TodoTypeSz == 0)
4175 return SzTodo > TodoTypeSz;
4189 .Cases({
"char",
"signed char",
"unsigned char",
"short",
4190 "unsigned short",
"int",
"unsigned int",
"_Float16",
"float",
4198 unsigned NumElts = 0;
4199 if (
Count.getAsInteger(10, NumElts) || NumElts == 0)
4206 TypeStr.
split(Quals,
' ', -1,
false);
4209 if (Qual !=
"const" && Qual !=
"volatile" && Qual !=
"restrict" &&
4210 !(Qual.consume_front(
"AS") && !Qual.getAsInteger(10, AS)))
4217void SPIRVEmitIntrinsicsImpl::parseFunDeclarations(
Module &M) {
4219 if (!
F.isDeclaration() ||
F.isIntrinsic())
4222 if (DemangledName.empty())
4225 auto Set =
ST.getPreferredInstructionSet();
4226 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(DemangledName, Set);
4228 const bool IsBuiltin = Grp >= 0 || SPIRV::isBuiltin(DemangledName, Set);
4229 if (IsBuiltin && Opcode != SPIRV::OpGroupAsyncCopy)
4231 SmallVector<unsigned> Idxs;
4232 for (Argument &Arg :
F.args())
4237 LLVMContext &Ctx =
F.getContext();
4239 SPIRV::parseBuiltinTypeStr(TypeStrs, DemangledName, Ctx);
4240 if (TypeStrs.
empty())
4243 if (!IsBuiltin && (!
F.getName().starts_with(
"_Z") ||
F.hasStructRetAttr() ||
4244 StringRef(DemangledName).
count(
'(') != 1 ||
4245 StringRef(DemangledName).
count(
')') != 1 ||
4246 StringRef(DemangledName).
contains(
'<') ||
4247 TypeStrs.
size() !=
F.arg_size()))
4249 bool AllowLongVectors =
4250 ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector);
4251 for (
unsigned Idx : Idxs) {
4252 if (Idx >= TypeStrs.
size())
4254 StringRef TypeStr = TypeStrs[
Idx].trim();
4255 Type *ElemTy = IsBuiltin
4256 ? SPIRV::parseBuiltinCallArgumentType(TypeStr, Ctx)
4266 FDeclPtrTys[&
F].emplace_back(Idx, ElemTy);
4268 ST.getSPIRVGlobalRegistry()->addDeducedElementType(
F.getArg(Idx),
4274bool SPIRVEmitIntrinsicsImpl::processMaskedMemIntrinsic(IntrinsicInst &
I) {
4275 const SPIRVSubtarget &
ST = TM.
getSubtarget<SPIRVSubtarget>(*
I.getFunction());
4277 if (
I.getIntrinsicID() == Intrinsic::masked_gather) {
4278 if (!
ST.canUseExtension(
4279 SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4280 I.getContext().emitError(
4281 &
I,
"llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
4285 I.eraseFromParent();
4291 Value *Ptrs =
I.getArgOperand(0);
4293 Value *Passthru =
I.getArgOperand(2);
4296 uint32_t
Alignment =
I.getParamAlign(0).valueOrOne().value();
4298 SmallVector<Value *, 4>
Args = {Ptrs,
B.getInt32(Alignment),
Mask,
4303 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_masked_gather, Types, Args);
4305 I.eraseFromParent();
4309 if (
I.getIntrinsicID() == Intrinsic::masked_scatter) {
4310 if (!
ST.canUseExtension(
4311 SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4312 I.getContext().emitError(
4313 &
I,
"llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
4316 I.eraseFromParent();
4323 Value *Ptrs =
I.getArgOperand(1);
4328 uint32_t
Alignment =
I.getParamAlign(1).valueOrOne().value();
4330 SmallVector<Value *, 4>
Args = {
Values, Ptrs,
B.getInt32(Alignment),
Mask};
4334 B.CreateIntrinsic(Intrinsic::spv_masked_scatter, Types, Args);
4335 I.eraseFromParent();
4346void SPIRVEmitIntrinsicsImpl::preprocessBoolVectorBitcasts(
Function &
F) {
4347 struct BoolVecBitcast {
4349 FixedVectorType *BoolVecTy;
4353 auto getAsBoolVec = [](
Type *Ty) -> FixedVectorType * {
4355 return (VTy && VTy->getElementType()->
isIntegerTy(1)) ? VTy :
nullptr;
4363 if (
auto *BVTy = getAsBoolVec(BC->getSrcTy()))
4365 else if (
auto *BVTy = getAsBoolVec(BC->getDestTy()))
4369 for (
auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
4371 Value *Src = BC->getOperand(0);
4372 unsigned BoolVecN = BoolVecTy->getNumElements();
4375 Type *OtherTy = SrcIsBoolVec ? BC->getDestTy() : Src->getType();
4376 unsigned NumElts = 1;
4378 NumElts = VTy->getNumElements();
4384 if (!SrcIsBoolVec) {
4387 auto GetLane = [&](
unsigned I) {
4390 Elt =
B.CreateExtractElement(IntVal,
B.getInt32(
I / EltBits));
4394 return B.CreateICmpNE(
And, ConstantInt::get(EltTy, 0));
4402 if (!Idx ||
Idx->getValue().uge(BoolVecN))
4404 B.SetInsertPoint(EE);
4405 EE->replaceAllUsesWith(GetLane(
Idx->getZExtValue()));
4406 EE->eraseFromParent();
4408 if (BC->use_empty()) {
4409 BC->eraseFromParent();
4413 B.SetInsertPoint(BC);
4415 for (
unsigned I = 0;
I < BoolVecN; ++
I)
4416 Result =
B.CreateInsertElement(Result, GetLane(
I),
B.getInt32(
I));
4420 for (
unsigned J = 0; J < NumElts; ++J) {
4421 Value *Elt = ConstantInt::get(EltTy, 0);
4422 for (
unsigned Bit = 0;
Bit < EltBits; ++
Bit) {
4424 B.CreateExtractElement(Src,
B.getInt32(J * EltBits + Bit));
4425 Value *Ext =
B.CreateZExt(Lane, EltTy);
4427 Ext =
B.CreateShl(Ext, ConstantInt::get(EltTy, Bit));
4428 Elt =
B.CreateOr(Elt, Ext);
4430 IntVal = NumElts > 1 ?
B.CreateInsertElement(IntVal, Elt,
B.getInt32(J))
4433 Result =
B.CreateBitCast(IntVal, BC->getDestTy());
4436 BC->replaceAllUsesWith(Result);
4437 BC->eraseFromParent();
4441bool SPIRVEmitIntrinsicsImpl::convertMaskedMemIntrinsics(
Module &M) {
4445 if (!
F.isIntrinsic())
4448 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
4453 Changed |= processMaskedMemIntrinsic(*
II);
4457 F.eraseFromParent();
4463bool SPIRVEmitIntrinsicsImpl::preprocessModule(
Module &M) {
4467 Type *I32Ty = Type::getInt32Ty(
M.getContext());
4469 if (
F.isDeclaration())
4475 CE &&
CE->getOpcode() == Instruction::GetElementPtr)
4482 if (!AI || !AI->isStaticAlloca() || !AI->isArrayAllocation() ||
4483 AI->getAddressSpace() !=
4488 if (
Count->isZero() || !
Count->getValue().isIntN(32))
4490 AI->setAllocatedType(
4491 ArrayType::get(AI->getAllocatedType(),
Count->getZExtValue()));
4492 AI->setOperand(0, ConstantInt::get(I32Ty, 1));
4496 Changed |= convertMaskedMemIntrinsics(M);
4504 GEPs,
nullptr,
false,
true);
4509bool SPIRVEmitIntrinsicsImpl::runOnModule(
Module &M) {
4510 bool Changed = preprocessModule(M);
4512 parseFunDeclarations(M);
4513 insertConstantsForFPFastMathDefault(M);
4519 if (AccessChainErrors)
4526 if (!
F.isDeclaration() && !
F.isIntrinsic()) {
4528 processParamTypes(&
F,
B);
4532 CanTodoType =
false;
4533 Changed |= postprocessTypes(M);
4536 Changed |= processFunctionPointers(M);
4543 if (SPIRVEmitIntrinsicsImpl(TM).runOnModule(M))
4549 return new SPIRVEmitIntrinsicsLegacy(TM);
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static void replaceAllUsesWith(Value *Old, Value *New, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHuge)
Replace all old uses with new ones, and push the updated BBs into FreshBBs.
static Type * getPointeeType(Value *Ptr, const DataLayout &DL)
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
static bool runOnFunction(Function &F, bool PostInlining)
iv Induction Variable Users
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
static bool isMemInstrToReplace(Instruction *I)
static bool isAggrConstForceInt32(const Value *V)
static Type * getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I, Value *PointerOperand)
static void reportFatalOnTokenType(const Instruction *I)
static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I)
static SPIRV::FPFastMathDefaultInfoVector & getOrCreateFPFastMathDefaultInfoVec(const Module &M, MapVector< Function *, SPIRV::FPFastMathDefaultInfoVector > &FPFastMathDefaultInfoMap, Function *F)
static bool isArtificialGlobal(StringRef Name)
static Type * getPointeeTypeByCallInst(StringRef DemangledName, Function *CalledF, unsigned OpIdx)
static void createRoundingModeDecoration(Instruction *I, unsigned RoundingModeDeco, IRBuilder<> &B)
static void createDecorationIntrinsic(Instruction *I, MDNode *Node, IRBuilder<> &B)
static void emitAssignName(const SPIRVOptions &CLOpts, Instruction *I, IRBuilder<> &B)
static bool hasOnlyArtificialUses(const GlobalVariable &GV)
static bool isAggregateValueIdInstr(const Instruction &I)
static SPIRV::FPFastMathDefaultInfo & getFPFastMathDefaultInfo(SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec, const Type *Ty)
static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST)
static bool tracesToPointerAlloca(Value *V)
static bool isUseListGlobal(StringRef Name)
static bool IsKernelArgInt8(Function *F, StoreInst *SI)
static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B)
static bool isFirstIndexZero(const GetElementPtrInst *GEP)
static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I)
static bool isSpvAggrPlaceholder(const Value *V)
static Type * getMemberPointeeType(Type *AggrTy, ArrayRef< unsigned > Indices)
static bool precededByAbortIntrinsic(const UnreachableInst &I, const SPIRVSubtarget &ST)
static FunctionType * getFunctionPointerElemType(Function *F, SPIRVGlobalRegistry *GR)
static bool isMultiRegisterAggregate(Value *V)
static void createSaturatedConversionDecoration(Instruction *I, IRBuilder<> &B)
static Type * parseDeclPtrElemType(StringRef TypeStr, LLVMContext &Ctx)
static bool shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers, const GlobalVariable &GV, const Function *F)
static Type * restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I, Type *Ty)
static bool requireAssignType(Instruction *I)
static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR, MachineIRBuilder MIB)
static void visit(BasicBlock &Start, std::function< bool(BasicBlock *)> op)
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallPtrSet class.
This file defines the SmallString class.
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
This class represents an incoming formal argument to a Function.
const Function * getParent() const
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
static unsigned getPointerOperandIndex()
static unsigned getPointerOperandIndex()
@ UIncWrap
Increment one up to a maximum value.
@ UDecWrap
Decrement one until a minimum value or zero.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
const Function * getParent() const
Return the enclosing method, or null if none.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
static LLVM_ABI BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
bool isInlineAsm() const
Check if this call is an inline asm statement.
void setCallingConv(CallingConv::ID CC)
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
unsigned arg_size() const
bool isArgOperand(const Use *U) const
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVM_ABI std::optional< RoundingMode > getRoundingMode() const
uint64_t getNumOperands() const
unsigned getProgramAddressSpace() const
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
static ErrorSuccess success()
Create a success value.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Type * getParamType(unsigned i) const
Parameter type accessors.
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static LLVM_ABI Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static unsigned getPointerOperandIndex()
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
@ InternalLinkage
Rename collisions when linking (static functions).
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI void addDestination(BasicBlock *Dest)
Add a destination.
static unsigned getInsertedValueOperandIndex()
Base class for instruction visitors.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
This is an important class for using LLVM in a threaded context.
static unsigned getPointerOperandIndex()
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
void analyze(ParentT F)
Create the loop forest for a function.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Flags
Flags values. These may be or'd together.
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
A Module instance is used to store all the information related to an LLVM module.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg, bool CanUseAnyVectorRank)
void addAssignPtrTypeInstr(Value *Val, CallInst *AssignPtrTyCI)
void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg)
Type * findDeducedCompositeType(const Value *Val)
void replaceAllUsesWith(Value *Old, Value *New, bool DeleteOld=true)
void addDeducedElementType(Value *Val, Type *Ty)
void addReturnType(const Function *ArgF, TypedPointerType *DerivedTy)
Type * findMutated(const Value *Val)
void addDeducedCompositeType(Value *Val, Type *Ty)
void removeAssignPtrTypeInstr(Value *OldVal)
Type * findDeducedElementType(const Value *Val)
void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType)
CallInst * findAssignPtrTypeInstr(const Value *Val)
const SPIRVTargetLowering * getTargetLowering() const override
bool isLogicalSPIRV() const
bool canUseExtension(SPIRV::Extension::Extension E) const
const SPIRVOptions & getCLOpts() const
const SPIRVSubtarget * getSubtargetImpl() const
bool erase(PtrType Ptr)
Remove pointer from the set.
iterator find(ConstPtrType Ptr) const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
static unsigned getPointerOperandIndex()
iterator find(StringRef Key)
Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
constexpr size_t size() const
Get the string size.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
StringRef drop_back(size_t N=1) const
Return a StringRef equal to 'this' but with the last N elements dropped.
A switch()-like statement whose cases are string literals.
static LLVM_ABI StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
static unsigned getPointerOperandIndex()
static LLVM_ABI TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})
Return a target extension type having the specified name and optional type and integer parameters.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr, CodeGenOptLevel OptLevel=CodeGenOptLevel::Default) const
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
const Triple & getTargetTriple() const
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
VendorType getVendor() const
Get the parsed vendor type of this triple.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isSized() const
Return true if it makes sense to take the size of this type.
LLVM_ABI StringRef getTargetExtName() const
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isStructTy() const
True if this is an instance of StructType.
bool isTargetExtTy() const
Return true if this is a target extension type.
bool isAggregateType() const
Return true if the type is an aggregate type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
static LLVM_ABI bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
static LLVM_ABI TypedPointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
void setOperand(unsigned i, Value *Val)
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
iterator_range< use_iterator > uses()
void mutateType(Type *Ty)
Mutate the type of this Value to be of the specified type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
bool match(Val *V, const Pattern &P)
auto m_Value()
Match an arbitrary value and ignore it.
auto m_AnyIntrinsic()
Matches any intrinsic call and ignore it.
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
std::string lookupBuiltinNameHelper(StringRef DemangledCall, FPDecorationId *DecorationId)
Parses the name part of the demangled builtin call.
@ CE
Windows NT (Windows on ARM)
initializer< Ty > init(const Ty &Val)
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
@ User
could "use" a pointer
NodeAddr< PhiNode * > Phi
NodeAddr< NodeBase * > Node
NodeAddr< FuncNode * > Func
friend class Instruction
Iterator for Instructions in a `BasicBlock.
unsigned getNumElements(Type *Ty)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
ModulePass * createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM)
bool isTypedPointerWrapper(const TargetExtType *ExtTy)
RelativeUniformCounterPtr Values
uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem, uint32_t StorageClassSem)
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
unsigned getPointerAddressSpace(const Type *T)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
auto successors(const MachineBasicBlock *BB)
CallInst * buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef< Type * > Types, Value *Arg, Value *Arg2, ArrayRef< Constant * > Imms, IRBuilder<> &B)
bool isUntypedPointerVectorTy(const Type *T)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool sortBlocks(Function &F)
bool isa_and_nonnull(const Y &Val)
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx, SyncScope::ID Id)
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
bool isNestedPointer(const Type *Ty)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Function * getOrCreateBackendServiceFunction(Module &M)
MetadataAsValue * buildMD(Value *Arg)
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name)
LLVM_ABI bool convertUsersOfConstantsToInstructions(ArrayRef< Constant * > Consts, Function *RestrictToFunc=nullptr, bool RemoveDeadConstants=true, bool IncludeSelf=false)
Replace constant expressions users of the given constants with instructions.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< unsigned, 1 > getSpirvLoopControlOperandsFromLoopMetadata(MDNode *LoopMD)
Type * normalizeType(Type *Ty, bool CanUseAnyVectorRank)
auto reverse(ContainerTy &&C)
Type * getTypedPointerWrapper(Type *ElemTy, unsigned AS)
Error joinErrors(Error E1, Error E2)
Concatenate errors.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
bool set_union(S1Ty &S1, const S2Ty &S2)
set_union(A, B) - Compute A := A u B, return whether A changed.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)
@ Ref
The access may reference the value stored in memory.
@ And
Bitwise or logical AND of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
LLVM_ABI Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
bool isLongVectorEXT(const Type *Ty, bool CanUseVector16=true)
Type * parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx)
DWARFExpression::Operation Op
Type * getPointeeTypeByAttr(Argument *Arg)
std::string toString(const APInt &I, unsigned Radix, bool Signed, bool formatAsCLiteral=false, bool UpperCase=true, bool InsertSeparators=false)
bool hasPointeeTypeAttr(Argument *Arg)
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
constexpr unsigned BitWidth
bool isEquivalentTypes(Type *Ty1, Type *Ty2)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
bool hasInitializer(const GlobalVariable *GV)
bool isPointerTyOrWrapper(const Type *Ty)
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
PoisonValue * getNormalizedPoisonValue(Type *Ty, bool CanUseAnyVectorRank)
bool isUntypedPointerTy(const Type *T)
Type * reconstitutePeeledArrayType(Type *Ty)
bool spirvPreserveAuxData(const SPIRVOptions &CLOpts, const Triple &TT)
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
static size_t computeFPFastMathDefaultInfoVecIndex(size_t BitWidth)