23#include "llvm/IR/IntrinsicsSPIRV.h"
60class SPIRVEmitIntrinsics
62 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
66 bool TrackConstants =
true;
72 Type *deduceElementType(
Value *
I,
bool UnknownElemTypeI8);
74 Type *deduceElementTypeHelper(
Value *
I, std::unordered_set<Value *> &Visited);
75 Type *deduceElementTypeByValueDeep(
Type *ValueTy,
Value *Operand,
76 std::unordered_set<Value *> &Visited);
78 std::unordered_set<Value *> &Visited);
81 Type *deduceNestedTypeHelper(
User *U);
83 std::unordered_set<Value *> &Visited);
96 Args.push_back(buildMD(Arg));
97 for (
auto *Imm : Imms)
99 return B.CreateIntrinsic(IntrID, {
Types},
Args);
109 bool UnknownElemTypeI8);
114 Type *ExpectedElementType,
115 unsigned OperandToReplace,
122 Type *deduceFunParamElementType(
Function *F,
unsigned OpIdx);
123 Type *deduceFunParamElementType(
Function *F,
unsigned OpIdx,
124 std::unordered_set<Function *> &FVisited);
160 const auto *
II = dyn_cast<IntrinsicInst>(
I);
164 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
165 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
166 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
170char SPIRVEmitIntrinsics::ID = 0;
176 return isa<IntrinsicInst>(
I) &&
177 cast<IntrinsicInst>(
I)->getIntrinsicID() == Intrinsic::spv_assign_type;
181 return isa<StoreInst>(
I) || isa<LoadInst>(
I) || isa<InsertValueInst>(
I) ||
182 isa<ExtractValueInst>(
I) || isa<AtomicCmpXchgInst>(
I);
186 return isa<ConstantArray>(V) || isa<ConstantStruct>(V) ||
187 isa<ConstantDataArray>(V) ||
188 (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
193 B.SetInsertPoint(
I->getParent()->getFirstNonPHIOrDbgOrAlloca());
199 B.SetCurrentDebugLocation(
I->getDebugLoc());
200 if (
I->getType()->isVoidTy())
201 B.SetInsertPoint(
I->getNextNode());
203 B.SetInsertPoint(*
I->getInsertionPointAfterDef());
209 switch (
Intr->getIntrinsicID()) {
210 case Intrinsic::invariant_start:
211 case Intrinsic::invariant_end:
219 if (
I->getType()->isTokenTy())
221 "does not support token type",
228 CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
229 {Arg->
getType()}, OfType, Arg, {},
B);
237 if (AssignPtrTyCI ==
nullptr ||
238 AssignPtrTyCI->
getParent()->getParent() !=
F) {
239 AssignPtrTyCI = buildIntrWithMD(
240 Intrinsic::spv_assign_ptr_type, {Arg->
getType()}, OfType, Arg,
246 updateAssignType(AssignPtrTyCI, Arg, OfType);
250void SPIRVEmitIntrinsics::updateAssignType(
CallInst *AssignCI,
Value *Arg,
254 Intrinsic::spv_assign_ptr_type)
265Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
266 Type *ValueTy,
Value *Operand, std::unordered_set<Value *> &Visited) {
269 if (
auto *PtrTy = dyn_cast<PointerType>(Ty)) {
270 if (
Type *NestedTy = deduceElementTypeHelper(Operand, Visited))
273 Ty = deduceNestedTypeHelper(dyn_cast<User>(Operand), Ty, Visited);
280Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
281 Value *
Op, std::unordered_set<Value *> &Visited) {
285 if (
auto PType = dyn_cast<TypedPointerType>(
Op->getType()))
286 return PType->getElementType();
292 for (
User *OpU :
Op->users()) {
293 if (
Instruction *Inst = dyn_cast<Instruction>(OpU)) {
294 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited))
306 Function *CalledF,
unsigned OpIdx) {
307 if ((DemangledName.
starts_with(
"__spirv_ocl_printf(") ||
310 return IntegerType::getInt8Ty(CalledF->
getContext());
316Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
Value *
I) {
317 std::unordered_set<Value *> Visited;
318 return deduceElementTypeHelper(
I, Visited);
321Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
322 Value *
I, std::unordered_set<Value *> &Visited) {
332 if (Visited.find(
I) != Visited.end())
339 if (
auto *
Ref = dyn_cast<AllocaInst>(
I)) {
340 Ty =
Ref->getAllocatedType();
341 }
else if (
auto *
Ref = dyn_cast<GetElementPtrInst>(
I)) {
342 Ty =
Ref->getResultElementType();
343 }
else if (
auto *
Ref = dyn_cast<GlobalValue>(
I)) {
344 Ty = deduceElementTypeByValueDeep(
346 Ref->getNumOperands() > 0 ?
Ref->getOperand(0) :
nullptr, Visited);
347 }
else if (
auto *
Ref = dyn_cast<AddrSpaceCastInst>(
I)) {
348 Ty = deduceElementTypeHelper(
Ref->getPointerOperand(), Visited);
349 }
else if (
auto *
Ref = dyn_cast<BitCastInst>(
I)) {
350 if (
Type *Src =
Ref->getSrcTy(), *Dest =
Ref->getDestTy();
352 Ty = deduceElementTypeHelper(
Ref->getOperand(0), Visited);
353 }
else if (
auto *
Ref = dyn_cast<AtomicCmpXchgInst>(
I)) {
355 Ty = deduceElementTypeByValueDeep(
Op->getType(),
Op, Visited);
356 }
else if (
auto *
Ref = dyn_cast<AtomicRMWInst>(
I)) {
358 Ty = deduceElementTypeByValueDeep(
Op->getType(),
Op, Visited);
359 }
else if (
auto *
Ref = dyn_cast<PHINode>(
I)) {
360 for (
unsigned i = 0; i <
Ref->getNumIncomingValues(); i++) {
361 Ty = deduceElementTypeByUsersDeep(
Ref->getIncomingValue(i), Visited);
365 }
else if (
auto *
Ref = dyn_cast<SelectInst>(
I)) {
366 for (
Value *
Op : {
Ref->getTrueValue(),
Ref->getFalseValue()}) {
367 Ty = deduceElementTypeByUsersDeep(
Op, Visited);
371 }
else if (
auto *CI = dyn_cast<CallInst>(
I)) {
376 {
"__spirv_GenericCastToPtr_ToGlobal", 0},
377 {
"__spirv_GenericCastToPtr_ToLocal", 0},
378 {
"__spirv_GenericCastToPtr_ToPrivate", 0},
379 {
"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
380 {
"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
381 {
"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
383 if (
Function *CalledF = CI->getCalledFunction()) {
384 std::string DemangledName =
386 auto AsArgIt = ResTypeByArg.
find(DemangledName);
387 if (AsArgIt != ResTypeByArg.
end())
388 Ty = deduceElementTypeHelper(CI->getArgOperand(AsArgIt->second),
405Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
User *U) {
406 std::unordered_set<Value *> Visited;
407 return deduceNestedTypeHelper(U,
U->getType(), Visited);
410Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
411 User *U,
Type *OrigTy, std::unordered_set<Value *> &Visited) {
420 if (Visited.find(U) != Visited.end())
424 if (dyn_cast<StructType>(OrigTy)) {
427 for (
unsigned i = 0; i <
U->getNumOperands(); ++i) {
429 Type *OpTy =
Op->getType();
432 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
433 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
436 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
440 Change |= Ty != OpTy;
447 }
else if (
auto *ArrTy = dyn_cast<ArrayType>(OrigTy)) {
448 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
449 Type *OpTy = ArrTy->getElementType();
451 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
452 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
455 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
458 Type *NewTy = ArrayType::get(Ty, ArrTy->getNumElements());
463 }
else if (
auto *VecTy = dyn_cast<VectorType>(OrigTy)) {
464 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
465 Type *OpTy = VecTy->getElementType();
467 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
468 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
471 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
474 Type *NewTy = VectorType::get(Ty, VecTy->getElementCount());
484Type *SPIRVEmitIntrinsics::deduceElementType(
Value *
I,
bool UnknownElemTypeI8) {
485 if (
Type *Ty = deduceElementTypeHelper(
I))
487 return UnknownElemTypeI8 ? IntegerType::getInt8Ty(
I->getContext()) : nullptr;
494void SPIRVEmitIntrinsics::deduceOperandElementType(
Instruction *
I) {
496 Type *KnownElemTy =
nullptr;
498 if (
auto *
Ref = dyn_cast<PHINode>(
I)) {
502 for (
unsigned i = 0; i <
Ref->getNumIncomingValues(); i++) {
507 }
else if (
auto *
Ref = dyn_cast<SelectInst>(
I)) {
511 for (
unsigned i = 0; i <
Ref->getNumOperands(); i++) {
516 }
else if (
auto *
Ref = dyn_cast<ReturnInst>(
I)) {
533 }
else if (
auto *
Ref = dyn_cast<ICmpInst>(
I)) {
541 KnownElemTy = ElemTy0;
543 }
else if (ElemTy1) {
544 KnownElemTy = ElemTy1;
550 if (!KnownElemTy || Ops.
size() == 0)
555 for (
auto &OpIt : Ops) {
560 if (Ty == KnownElemTy)
563 Type *OpTy =
Op->getType();
568 if (AssignCI ==
nullptr) {
572 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal,
Op,
576 updateAssignType(AssignCI,
Op, OpTyVal);
579 if (
auto *OpI = dyn_cast<Instruction>(
Op)) {
582 B.SetInsertPoint(*OpI->getInsertionPointAfterDef());
583 B.SetCurrentDebugLocation(OpI->getDebugLoc());
584 }
else if (
auto *OpA = dyn_cast<Argument>(
Op)) {
585 B.SetInsertPointPastAllocas(OpA->getParent());
588 B.SetInsertPoint(
F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
594 B.CreateIntrinsic(Intrinsic::spv_ptrcast, {
Types},
Args);
595 I->setOperand(OpIt.second, PtrCastI);
600void SPIRVEmitIntrinsics::replaceMemInstrUses(
Instruction *Old,
605 if (isAssignTypeInstr(U)) {
609 B.CreateIntrinsic(Intrinsic::spv_assign_type, {
New->getType()},
Args);
611 U->eraseFromParent();
614 U->replaceUsesOfWith(Old, New);
622void SPIRVEmitIntrinsics::preprocessUndefs(
IRBuilder<> &
B) {
623 std::queue<Instruction *> Worklist;
627 while (!Worklist.empty()) {
629 bool BPrepared =
false;
632 for (
auto &
Op :
I->operands()) {
633 auto *AggrUndef = dyn_cast<UndefValue>(
Op);
634 if (!AggrUndef || !
Op->getType()->isAggregateType())
641 auto *IntrUndef =
B.CreateIntrinsic(Intrinsic::spv_undef, {}, {});
642 Worklist.push(IntrUndef);
643 I->replaceUsesOfWith(
Op, IntrUndef);
644 AggrConsts[IntrUndef] = AggrUndef;
645 AggrConstTypes[IntrUndef] = AggrUndef->getType();
650void SPIRVEmitIntrinsics::preprocessCompositeConstants(
IRBuilder<> &
B) {
651 std::queue<Instruction *> Worklist;
655 while (!Worklist.empty()) {
656 auto *
I = Worklist.front();
657 bool IsPhi = isa<PHINode>(
I), BPrepared =
false;
659 bool KeepInst =
false;
660 for (
const auto &
Op :
I->operands()) {
662 Type *ResTy =
nullptr;
663 if (
auto *COp = dyn_cast<ConstantVector>(
Op)) {
664 AggrConst = cast<Constant>(COp);
665 ResTy = COp->getType();
666 }
else if (
auto *COp = dyn_cast<ConstantArray>(
Op)) {
667 AggrConst = cast<Constant>(COp);
668 ResTy =
B.getInt32Ty();
669 }
else if (
auto *COp = dyn_cast<ConstantStruct>(
Op)) {
670 AggrConst = cast<Constant>(COp);
671 ResTy =
B.getInt32Ty();
672 }
else if (
auto *COp = dyn_cast<ConstantDataArray>(
Op)) {
673 AggrConst = cast<Constant>(COp);
674 ResTy =
B.getInt32Ty();
675 }
else if (
auto *COp = dyn_cast<ConstantAggregateZero>(
Op)) {
676 AggrConst = cast<Constant>(COp);
677 ResTy =
Op->getType()->isVectorTy() ? COp->getType() :
B.getInt32Ty();
681 if (
auto *COp = dyn_cast<ConstantDataSequential>(
Op))
682 for (
unsigned i = 0; i < COp->getNumElements(); ++i)
683 Args.push_back(COp->getElementAsConstant(i));
685 for (
auto &COp : AggrConst->
operands())
688 IsPhi ?
B.SetInsertPointPastAllocas(
I->getParent()->getParent())
689 :
B.SetInsertPoint(
I);
693 B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {
Args});
695 I->replaceUsesOfWith(
Op, CI);
697 AggrConsts[CI] = AggrConst;
698 AggrConstTypes[CI] = deduceNestedTypeHelper(AggrConst);
707 if (!
Call.isInlineAsm())
718 for (
unsigned OpIdx = 0; OpIdx <
Call.arg_size(); OpIdx++)
719 Args.push_back(
Call.getArgOperand(OpIdx));
722 B.SetInsertPoint(&Call);
723 B.CreateIntrinsic(Intrinsic::spv_inline_asm, {}, {
Args});
730 B.SetInsertPoint(&
I);
733 for (
auto &
Op :
I.operands()) {
734 if (
Op.get()->getType()->isSized()) {
736 }
else if (
BasicBlock *BB = dyn_cast<BasicBlock>(
Op.get())) {
743 CallInst *NewI =
B.CreateIntrinsic(Intrinsic::spv_switch,
747 I.replaceAllUsesWith(NewI);
751 B.SetInsertPoint(ParentBB);
762 B.SetInsertPoint(&
I);
765 Args.push_back(
B.getInt1(
I.isInBounds()));
766 for (
auto &
Op :
I.operands())
768 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_gep, {
Types}, {
Args});
769 I.replaceAllUsesWith(NewI);
776 B.SetInsertPoint(&
I);
784 I.replaceAllUsesWith(Source);
791 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_bitcast, {
Types}, {
Args});
792 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
793 I.replaceAllUsesWith(NewI);
799void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
801 Type *VTy =
V->getType();
805 if (
auto PType = dyn_cast<TypedPointerType>(VTy))
806 if (PType->getElementType() != AssignedType)
811 buildAssignType(
B, AssignedType, V);
816 dyn_cast<ConstantAsMetadata>(
817 cast<MetadataAsValue>(AssignCI->
getOperand(1))->getMetadata())
819 if (CurrentType == AssignedType)
826 " for value " +
V->getName(),
834void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
839 while (
BitCastInst *BC = dyn_cast<BitCastInst>(Pointer))
843 Type *PointerElemTy = deduceElementTypeHelper(Pointer);
844 if (PointerElemTy == ExpectedElementType)
850 bool FirstPtrCastOrAssignPtrType =
true;
856 auto *
II = dyn_cast<IntrinsicInst>(
User);
858 (
II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
859 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
860 II->getOperand(0) != Pointer)
865 FirstPtrCastOrAssignPtrType =
false;
866 if (
II->getOperand(1) != VMD ||
867 dyn_cast<ConstantInt>(
II->getOperand(2))->getSExtValue() !=
873 if (
II->getIntrinsicID() != Intrinsic::spv_ptrcast)
878 if (
II->getParent() !=
I->getParent())
881 I->setOperand(OperandToReplace,
II);
892 if (FirstPtrCastOrAssignPtrType &&
893 (isa<Instruction>(Pointer) || isa<Argument>(Pointer))) {
894 buildAssignPtr(
B, ExpectedElementType, Pointer);
901 auto *PtrCastI =
B.CreateIntrinsic(Intrinsic::spv_ptrcast, {
Types},
Args);
902 I->setOperand(OperandToReplace, PtrCastI);
905void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(
Instruction *
I,
911 isa<Argument>(
SI->getValueOperand())) {
912 return replacePointerOperandWithPtrCast(
913 I,
SI->getValueOperand(), IntegerType::getInt8Ty(
F->getContext()), 0,
916 return replacePointerOperandWithPtrCast(
917 I,
SI->getPointerOperand(),
SI->getValueOperand()->getType(), 1,
B);
918 }
else if (
LoadInst *LI = dyn_cast<LoadInst>(
I)) {
919 return replacePointerOperandWithPtrCast(
I, LI->getPointerOperand(),
920 LI->getType(), 0,
B);
922 return replacePointerOperandWithPtrCast(
I, GEPI->getPointerOperand(),
923 GEPI->getSourceElementType(), 0,
B);
933 std::string DemangledName =
937 bool HaveTypes =
false;
938 for (
unsigned OpIdx = 0; OpIdx < CalledF->
arg_size(); ++OpIdx) {
944 CalledArgTys.
push_back(cast<TypedPointerType>(ArgType)->getElementType());
956 if (
Instruction *Inst = dyn_cast<Instruction>(U)) {
957 if ((ElemTy = deduceElementTypeHelper(Inst)) !=
nullptr)
963 HaveTypes |= ElemTy !=
nullptr;
968 if (DemangledName.empty() && !HaveTypes)
971 for (
unsigned OpIdx = 0; OpIdx < CI->
arg_size(); OpIdx++) {
977 if (!isa<Instruction>(ArgOperand) && !isa<Argument>(ArgOperand)) {
987 OpIdx < CalledArgTys.
size() ? CalledArgTys[OpIdx] :
nullptr;
988 if (!ExpectedType && !DemangledName.empty())
990 DemangledName, OpIdx,
I->getContext());
995 insertAssignPtrTypeTargetExt(cast<TargetExtType>(ExpectedType),
998 replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx,
B);
1004 I.getOperand(1)->getType(),
1005 I.getOperand(2)->getType()};
1007 B.SetInsertPoint(&
I);
1009 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_insertelt, {
Types}, {
Args});
1010 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
1011 I.replaceAllUsesWith(NewI);
1012 I.eraseFromParent();
1020 B.SetInsertPoint(&
I);
1022 I.getIndexOperand()->getType()};
1024 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_extractelt, {
Types}, {
Args});
1025 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
1026 I.replaceAllUsesWith(NewI);
1027 I.eraseFromParent();
1034 B.SetInsertPoint(&
I);
1037 for (
auto &
Op :
I.operands())
1038 if (isa<UndefValue>(
Op))
1042 for (
auto &
Op :
I.indices())
1043 Args.push_back(
B.getInt32(
Op));
1045 B.CreateIntrinsic(Intrinsic::spv_insertv, {
Types}, {
Args});
1046 replaceMemInstrUses(&
I, NewI,
B);
1052 B.SetInsertPoint(&
I);
1054 for (
auto &
Op :
I.operands())
1056 for (
auto &
Op :
I.indices())
1057 Args.push_back(
B.getInt32(
Op));
1059 B.CreateIntrinsic(Intrinsic::spv_extractv, {
I.getType()}, {
Args});
1060 I.replaceAllUsesWith(NewI);
1061 I.eraseFromParent();
1066 if (!
I.getType()->isAggregateType())
1069 B.SetInsertPoint(&
I);
1070 TrackConstants =
false;
1071 const auto *TLI =
TM->getSubtargetImpl()->getTargetLowering();
1073 TLI->getLoadMemOperandFlags(
I,
F->getDataLayout());
1075 B.CreateIntrinsic(Intrinsic::spv_load, {
I.getOperand(0)->
getType()},
1076 {
I.getPointerOperand(),
B.getInt16(Flags),
1077 B.getInt8(
I.getAlign().value())});
1078 replaceMemInstrUses(&
I, NewI,
B);
1086 B.SetInsertPoint(&
I);
1087 TrackConstants =
false;
1088 const auto *TLI =
TM->getSubtargetImpl()->getTargetLowering();
1090 TLI->getStoreMemOperandFlags(
I,
F->getDataLayout());
1091 auto *PtrOp =
I.getPointerOperand();
1092 auto *NewI =
B.CreateIntrinsic(
1093 Intrinsic::spv_store, {
I.getValueOperand()->
getType(), PtrOp->getType()},
1094 {
I.getValueOperand(), PtrOp,
B.getInt16(Flags),
1095 B.getInt8(
I.getAlign().value())});
1096 I.eraseFromParent();
1101 Value *ArraySize =
nullptr;
1102 if (
I.isArrayAllocation()) {
1105 SPIRV::Extension::SPV_INTEL_variable_length_array))
1107 "array allocation: this instruction requires the following "
1108 "SPIR-V extension: SPV_INTEL_variable_length_array",
1110 ArraySize =
I.getArraySize();
1113 B.SetInsertPoint(&
I);
1114 TrackConstants =
false;
1115 Type *PtrTy =
I.getType();
1117 ArraySize ?
B.CreateIntrinsic(Intrinsic::spv_alloca_array,
1118 {PtrTy, ArraySize->
getType()}, {ArraySize})
1119 :
B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
1120 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
1121 I.replaceAllUsesWith(NewI);
1122 I.eraseFromParent();
1128 assert(
I.getType()->isAggregateType() &&
"Aggregate result is expected");
1130 B.SetInsertPoint(&
I);
1132 for (
auto &
Op :
I.operands())
1134 Args.push_back(
B.getInt32(
I.getSyncScopeID()));
1135 Args.push_back(
B.getInt32(
1137 Args.push_back(
B.getInt32(
1139 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_cmpxchg,
1141 replaceMemInstrUses(&
I, NewI,
B);
1147 B.SetInsertPoint(&
I);
1148 B.CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
1155 if (GV.
getName() ==
"llvm.global.annotations")
1161 deduceElementTypeHelper(&GV);
1165 auto *InitInst =
B.CreateIntrinsic(Intrinsic::spv_init_global,
1167 InitInst->setArgOperand(1,
Init);
1171 B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.
getType(), &GV);
1177bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(
Instruction *
I,
1179 bool UnknownElemTypeI8) {
1182 isa<BitCastInst>(
I))
1186 if (
Type *ElemTy = deduceElementType(
I, UnknownElemTypeI8)) {
1187 buildAssignPtr(
B, ElemTy,
I);
1193void SPIRVEmitIntrinsics::insertAssignTypeIntrs(
Instruction *
I,
1196 Type *Ty =
I->getType();
1199 Type *TypeToAssign = Ty;
1200 if (
auto *
II = dyn_cast<IntrinsicInst>(
I)) {
1201 if (
II->getIntrinsicID() == Intrinsic::spv_const_composite ||
1202 II->getIntrinsicID() == Intrinsic::spv_undef) {
1203 auto It = AggrConstTypes.
find(
II);
1204 if (It == AggrConstTypes.
end())
1206 TypeToAssign = It->second;
1209 buildAssignType(
B, TypeToAssign,
I);
1211 for (
const auto &
Op :
I->operands()) {
1212 if (isa<ConstantPointerNull>(
Op) || isa<UndefValue>(
Op) ||
1214 (isa<ConstantExpr>(
Op) && isa<GEPOperator>(
Op))) {
1216 Type *OpTy =
Op->getType();
1219 buildIntrWithMD(Intrinsic::spv_assign_type, {
B.getInt32Ty()},
Op,
1222 }
else if (!isa<Instruction>(
Op)) {
1223 Type *OpTy =
Op->getType();
1224 if (
auto PType = dyn_cast<TypedPointerType>(OpTy)) {
1225 buildAssignPtr(
B, PType->getElementType(),
Op);
1228 buildAssignPtr(
B, ElemTy ? ElemTy : deduceElementType(
Op,
true),
Op);
1230 CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
1231 {OpTy},
Op,
Op, {},
B);
1239void SPIRVEmitIntrinsics::insertSpirvDecorations(
Instruction *
I,
1241 if (
MDNode *MD =
I->getMetadata(
"spirv.Decorations")) {
1243 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {
I->getType()},
1248void SPIRVEmitIntrinsics::processInstrAfterVisit(
Instruction *
I,
1250 auto *
II = dyn_cast<IntrinsicInst>(
I);
1251 if (
II &&
II->getIntrinsicID() == Intrinsic::spv_const_composite &&
1254 auto t = AggrConsts.
find(
I);
1257 buildIntrWithMD(Intrinsic::spv_track_constant,
1258 {
II->getType(),
II->getType()}, t->second,
I, {},
B);
1259 I->replaceAllUsesWith(NewOp);
1260 NewOp->setArgOperand(0,
I);
1262 bool IsPhi = isa<PHINode>(
I), BPrepared =
false;
1263 for (
const auto &
Op :
I->operands()) {
1264 if (isa<PHINode>(
I) || isa<SwitchInst>(
I))
1265 TrackConstants =
false;
1266 if ((isa<ConstantData>(
Op) || isa<ConstantExpr>(
Op)) && TrackConstants) {
1267 unsigned OpNo =
Op.getOperandNo();
1268 if (
II && ((
II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
1269 (
II->paramHasAttr(OpNo, Attribute::ImmArg))))
1272 IsPhi ?
B.SetInsertPointPastAllocas(
I->getParent()->getParent())
1273 :
B.SetInsertPoint(
I);
1277 if (
Op->getType()->isTargetExtTy())
1279 auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
1282 I->setOperand(OpNo, NewOp);
1288 std::vector<Value *>
Args = {
I};
1290 B.CreateIntrinsic(Intrinsic::spv_assign_name, {
I->getType()},
Args);
1294Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
Function *
F,
1296 std::unordered_set<Function *> FVisited;
1297 return deduceFunParamElementType(
F, OpIdx, FVisited);
1300Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
1301 Function *
F,
unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
1303 if (FVisited.find(
F) != FVisited.end())
1307 std::unordered_set<Value *> Visited;
1310 for (
User *U :
F->users()) {
1311 CallInst *CI = dyn_cast<CallInst>(U);
1312 if (!CI || OpIdx >= CI->
arg_size())
1322 if (
Type *Ty = deduceElementTypeHelper(OpArg, Visited))
1327 if (!Inst || Inst == CI)
1330 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited))
1337 if (FVisited.find(OuterF) != FVisited.end())
1339 for (
unsigned i = 0; i < OuterF->
arg_size(); ++i) {
1340 if (OuterF->
getArg(i) == OpArg) {
1341 Lookup.push_back(std::make_pair(OuterF, i));
1348 for (
auto &Pair :
Lookup) {
1349 if (
Type *Ty = deduceFunParamElementType(Pair.first, Pair.second, FVisited))
1356void SPIRVEmitIntrinsics::processParamTypesByFunHeader(
Function *
F,
1358 B.SetInsertPointPastAllocas(
F);
1359 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
1366 buildAssignPtr(
B, ElemTy, Arg);
1371 B.SetInsertPointPastAllocas(
F);
1372 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
1377 if (!ElemTy && (ElemTy = deduceFunParamElementType(
F, OpIdx)) !=
nullptr)
1378 buildAssignPtr(
B, ElemTy, Arg);
1382bool SPIRVEmitIntrinsics::runOnFunction(
Function &Func) {
1383 if (
Func.isDeclaration())
1387 GR =
ST.getSPIRVGlobalRegistry();
1392 AggrConstTypes.
clear();
1395 processParamTypesByFunHeader(
F,
B);
1403 Type *ElTy =
SI->getValueOperand()->getType();
1408 B.SetInsertPoint(&
Func.getEntryBlock(),
Func.getEntryBlock().begin());
1409 for (
auto &GV :
Func.getParent()->globals())
1410 processGlobalValue(GV,
B);
1412 preprocessUndefs(
B);
1413 preprocessCompositeConstants(
B);
1418 for (
auto &
I : Worklist) {
1420 if (isConvergenceIntrinsic(
I))
1423 bool Postpone = insertAssignPtrTypeIntrs(
I,
B,
false);
1425 insertAssignTypeIntrs(
I,
B);
1426 insertPtrCastOrAssignTypeInstr(
I,
B);
1430 if (Postpone && !GR->findAssignPtrTypeInstr(
I))
1431 insertAssignPtrTypeIntrs(
I,
B,
true);
1435 deduceOperandElementType(&
I);
1437 for (
auto *
I : Worklist) {
1438 TrackConstants =
true;
1439 if (!
I->getType()->isVoidTy() || isa<StoreInst>(
I))
1448 if (isConvergenceIntrinsic(
I))
1451 processInstrAfterVisit(
I,
B);
1457bool SPIRVEmitIntrinsics::runOnModule(
Module &M) {
1458 bool Changed =
false;
1466 if (!
F.isDeclaration() && !
F.isIntrinsic()) {
1468 GR =
ST.getSPIRVGlobalRegistry();
1470 processParamTypes(&
F,
B);
1478 return new SPIRVEmitIntrinsics(
TM);
static unsigned getIntrinsicID(const SDNode *N)
Expand Atomic instructions
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool runOnFunction(Function &F, bool PostInlining)
uint64_t IntrinsicInst * II
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isMemInstrToReplace(Instruction *I)
static bool isAggrConstForceInt32(const Value *V)
static void reportFatalOnTokenType(const Instruction *I)
static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I)
static Type * getPointeeTypeByCallInst(StringRef DemangledName, Function *CalledF, unsigned OpIdx)
static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I)
static bool requireAssignType(Instruction *I)
static void insertSpirvDecorations(MachineFunction &MF, MachineIRBuilder MIB)
static SymbolRef::Type getType(const Symbol *Sym)
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
an instruction to allocate memory on the stack
Represent the analysis usage information of a pass.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An instruction that atomically checks whether a specified value is in a memory location,...
LLVM Basic Block Representation.
LLVMContext & getContext() const
Get the context in which this basic block lives.
This class represents a no-op cast from one type to another.
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool isIndirectCall() const
Return true if the callsite is an indirect call.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This class represents an Operation in the Expression.
iterator find(const_arg_type_t< KeyT > Val)
Implements a dense probed hash-table based set.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
PointerType * getType() const
Global values are always pointers.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Indirect Branch Instruction.
void addDestination(BasicBlock *Dest)
Add a destination.
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
RetTy visitExtractElementInst(ExtractElementInst &I)
RetTy visitInsertValueInst(InsertValueInst &I)
RetTy visitUnreachableInst(UnreachableInst &I)
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
RetTy visitBitCastInst(BitCastInst &I)
RetTy visitSwitchInst(SwitchInst &I)
RetTy visitExtractValueInst(ExtractValueInst &I)
RetTy visitStoreInst(StoreInst &I)
RetTy visitInsertElementInst(InsertElementInst &I)
RetTy visitAllocaInst(AllocaInst &I)
RetTy visitCallInst(CallInst &I)
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
void visitInstruction(Instruction &I)
RetTy visitLoadInst(LoadInst &I)
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
Flags
Flags values. These may be or'd together.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
void addAssignPtrTypeInstr(Value *Val, CallInst *AssignPtrTyCI)
Type * findDeducedCompositeType(const Value *Val)
void addDeducedElementType(Value *Val, Type *Ty)
void addReturnType(const Function *ArgF, TypedPointerType *DerivedTy)
void addDeducedCompositeType(Value *Val, Type *Ty)
Type * findDeducedElementType(const Value *Val)
CallInst * findAssignPtrTypeInstr(const Value *Val)
bool canUseExtension(SPIRV::Extension::Extension E) const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
iterator find(StringRef Key)
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Class to represent target extensions types, which are generally unintrospectable from target-independ...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
StringRef getTargetExtName() const
bool isTargetExtTy() const
Return true if this is a target extension type.
bool isAggregateType() const
Return true if the type is an aggregate type.
bool isVoidTy() const
Return true if this is 'void'.
A few GPU targets, such as DXIL and SPIR-V, have typed pointers.
static TypedPointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
unsigned getNumUses() const
This method computes the number of uses of this Value.
StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ SPIR_KERNEL
Used for SPIR kernel functions.
Type * parseBuiltinCallArgumentBaseType(const StringRef DemangledCall, unsigned ArgIdx, LLVMContext &Ctx)
Parses the provided ArgIdx argument base type in the DemangledCall skeleton.
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
void initializeSPIRVEmitIntrinsicsPass(PassRegistry &)
ModulePass * createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM)
unsigned getPointerAddressSpace(const Type *T)
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name)
bool isTypedPointerTy(const Type *T)
bool isPointerTy(const Type *T)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
@ Ref
The access may reference the value stored in memory.
DWARFExpression::Operation Op
Type * getPointeeTypeByAttr(Argument *Arg)
bool hasPointeeTypeAttr(Argument *Arg)
void addStringImm(const StringRef &Str, MCInst &Inst)
bool isUntypedPointerTy(const Type *T)
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)