23#include "llvm/IR/IntrinsicsSPIRV.h"
53class SPIRVEmitIntrinsics
55 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
59 bool TrackConstants =
true;
70 Type *deduceElementTypeHelper(
Value *
I, std::unordered_set<Value *> &Visited);
71 Type *deduceElementTypeByValueDeep(
Type *ValueTy,
Value *Operand,
72 std::unordered_set<Value *> &Visited);
74 std::unordered_set<Value *> &Visited);
77 Type *deduceNestedTypeHelper(
User *U);
79 std::unordered_set<Value *> &Visited);
96 for (
auto *Imm : Imms)
98 return B.CreateIntrinsic(IntrID, {
Types},
Args);
107 void insertAssignTypeInstrForTargetExtTypes(
TargetExtType *AssignedType,
110 Type *ExpectedElementType,
111 unsigned OperandToReplace,
118 Type *deduceFunParamElementType(
Function *F,
unsigned OpIdx);
119 Type *deduceFunParamElementType(
Function *F,
unsigned OpIdx,
120 std::unordered_set<Function *> &FVisited);
155char SPIRVEmitIntrinsics::ID = 0;
161 return isa<IntrinsicInst>(
I) &&
162 cast<IntrinsicInst>(
I)->getIntrinsicID() == Intrinsic::spv_assign_type;
166 return isa<StoreInst>(
I) || isa<LoadInst>(
I) || isa<InsertValueInst>(
I) ||
167 isa<ExtractValueInst>(
I) || isa<AtomicCmpXchgInst>(
I);
171 return isa<ConstantAggregate>(V) || isa<ConstantDataArray>(V) ||
172 (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
177 B.SetInsertPoint(
I->getParent(),
I->getParent()->getFirstInsertionPt());
185 switch (
Intr->getIntrinsicID()) {
186 case Intrinsic::invariant_start:
187 case Intrinsic::invariant_end:
195 if (
I->getType()->isTokenTy())
197 "does not support token type",
204 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Arg->
getType()},
209 AssignPtrTypeInstr[Arg] = AssignPtrTyCI;
214Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
215 Type *ValueTy,
Value *Operand, std::unordered_set<Value *> &Visited) {
218 if (
auto *PtrTy = dyn_cast<PointerType>(Ty)) {
219 if (
Type *NestedTy = deduceElementTypeHelper(Operand, Visited))
222 Ty = deduceNestedTypeHelper(dyn_cast<User>(Operand), Ty, Visited);
229Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
230 Value *
Op, std::unordered_set<Value *> &Visited) {
234 if (
auto PType = dyn_cast<TypedPointerType>(
Op->getType()))
235 return PType->getElementType();
241 for (
User *OpU :
Op->users()) {
242 if (
Instruction *Inst = dyn_cast<Instruction>(OpU)) {
243 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited))
255 Function *CalledF,
unsigned OpIdx) {
256 if ((DemangledName.
starts_with(
"__spirv_ocl_printf(") ||
259 return IntegerType::getInt8Ty(CalledF->
getContext());
265Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
Value *
I) {
266 std::unordered_set<Value *> Visited;
267 return deduceElementTypeHelper(
I, Visited);
270Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
271 Value *
I, std::unordered_set<Value *> &Visited) {
281 if (Visited.find(
I) != Visited.end())
288 if (
auto *
Ref = dyn_cast<AllocaInst>(
I)) {
289 Ty =
Ref->getAllocatedType();
290 }
else if (
auto *
Ref = dyn_cast<GetElementPtrInst>(
I)) {
291 Ty =
Ref->getResultElementType();
292 }
else if (
auto *
Ref = dyn_cast<GlobalValue>(
I)) {
293 Ty = deduceElementTypeByValueDeep(
295 Ref->getNumOperands() > 0 ?
Ref->getOperand(0) :
nullptr, Visited);
296 }
else if (
auto *
Ref = dyn_cast<AddrSpaceCastInst>(
I)) {
297 Ty = deduceElementTypeHelper(
Ref->getPointerOperand(), Visited);
298 }
else if (
auto *
Ref = dyn_cast<BitCastInst>(
I)) {
299 if (
Type *Src =
Ref->getSrcTy(), *Dest =
Ref->getDestTy();
301 Ty = deduceElementTypeHelper(
Ref->getOperand(0), Visited);
302 }
else if (
auto *
Ref = dyn_cast<AtomicCmpXchgInst>(
I)) {
304 Ty = deduceElementTypeByValueDeep(
Op->getType(),
Op, Visited);
305 }
else if (
auto *
Ref = dyn_cast<AtomicRMWInst>(
I)) {
307 Ty = deduceElementTypeByValueDeep(
Op->getType(),
Op, Visited);
308 }
else if (
auto *
Ref = dyn_cast<PHINode>(
I)) {
309 for (
unsigned i = 0; i <
Ref->getNumIncomingValues(); i++) {
310 Ty = deduceElementTypeByUsersDeep(
Ref->getIncomingValue(i), Visited);
314 }
else if (
auto *
Ref = dyn_cast<SelectInst>(
I)) {
315 for (
Value *
Op : {
Ref->getTrueValue(),
Ref->getFalseValue()}) {
316 Ty = deduceElementTypeByUsersDeep(
Op, Visited);
334Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
User *U) {
335 std::unordered_set<Value *> Visited;
336 return deduceNestedTypeHelper(U,
U->getType(), Visited);
339Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
340 User *U,
Type *OrigTy, std::unordered_set<Value *> &Visited) {
349 if (Visited.find(U) != Visited.end())
353 if (dyn_cast<StructType>(OrigTy)) {
356 for (
unsigned i = 0; i <
U->getNumOperands(); ++i) {
358 Type *OpTy =
Op->getType();
361 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
362 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
365 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
369 Change |= Ty != OpTy;
376 }
else if (
auto *ArrTy = dyn_cast<ArrayType>(OrigTy)) {
377 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
378 Type *OpTy = ArrTy->getElementType();
380 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
381 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
384 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
387 Type *NewTy = ArrayType::get(Ty, ArrTy->getNumElements());
392 }
else if (
auto *VecTy = dyn_cast<VectorType>(OrigTy)) {
393 if (
Value *
Op =
U->getNumOperands() > 0 ?
U->getOperand(0) :
nullptr) {
394 Type *OpTy = VecTy->getElementType();
396 if (
auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
397 if (
Type *NestedTy = deduceElementTypeHelper(
Op, Visited))
400 Ty = deduceNestedTypeHelper(dyn_cast<User>(
Op), OpTy, Visited);
403 Type *NewTy = VectorType::get(Ty, VecTy->getElementCount());
413Type *SPIRVEmitIntrinsics::deduceElementType(
Value *
I) {
414 if (
Type *Ty = deduceElementTypeHelper(
I))
416 return IntegerType::getInt8Ty(
I->getContext());
423void SPIRVEmitIntrinsics::deduceOperandElementType(
Instruction *
I) {
425 Type *KnownElemTy =
nullptr;
427 if (
auto *
Ref = dyn_cast<PHINode>(
I)) {
431 for (
unsigned i = 0; i <
Ref->getNumIncomingValues(); i++) {
436 }
else if (
auto *
Ref = dyn_cast<SelectInst>(
I)) {
440 for (
unsigned i = 0; i <
Ref->getNumOperands(); i++) {
445 }
else if (
auto *
Ref = dyn_cast<ReturnInst>(
I)) {
462 }
else if (
auto *
Ref = dyn_cast<ICmpInst>(
I)) {
470 KnownElemTy = ElemTy0;
472 }
else if (ElemTy1) {
473 KnownElemTy = ElemTy1;
479 if (!KnownElemTy || Ops.
size() == 0)
484 for (
auto &OpIt : Ops) {
489 if (Ty == KnownElemTy)
496 Type *OpTy =
Op->getType();
500 auto It = AssignPtrTypeInstr.
find(
Op);
501 if (It == AssignPtrTypeInstr.
end()) {
503 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal,
Op,
505 AssignPtrTypeInstr[
Op] = CI;
519 B.CreateIntrinsic(Intrinsic::spv_ptrcast, {
Types},
Args);
520 I->setOperand(OpIt.second, PtrCastI);
525void SPIRVEmitIntrinsics::replaceMemInstrUses(
Instruction *Old,
530 if (isAssignTypeInstr(U)) {
533 B.CreateIntrinsic(Intrinsic::spv_assign_type, {
New->getType()},
Args);
534 U->eraseFromParent();
537 U->replaceUsesOfWith(Old, New);
545void SPIRVEmitIntrinsics::preprocessUndefs(
IRBuilder<> &
B) {
546 std::queue<Instruction *> Worklist;
550 while (!Worklist.empty()) {
554 for (
auto &
Op :
I->operands()) {
555 auto *AggrUndef = dyn_cast<UndefValue>(
Op);
556 if (!AggrUndef || !
Op->getType()->isAggregateType())
560 auto *IntrUndef =
B.CreateIntrinsic(Intrinsic::spv_undef, {}, {});
561 Worklist.push(IntrUndef);
562 I->replaceUsesOfWith(
Op, IntrUndef);
563 AggrConsts[IntrUndef] = AggrUndef;
564 AggrConstTypes[IntrUndef] = AggrUndef->getType();
569void SPIRVEmitIntrinsics::preprocessCompositeConstants(
IRBuilder<> &
B) {
570 std::queue<Instruction *> Worklist;
574 while (!Worklist.empty()) {
575 auto *
I = Worklist.front();
577 bool KeepInst =
false;
578 for (
const auto &
Op :
I->operands()) {
579 auto BuildCompositeIntrinsic =
582 bool &KeepInst, SPIRVEmitIntrinsics &SEI) {
585 B.CreateIntrinsic(Intrinsic::spv_const_composite, {}, {
Args});
587 I->replaceUsesOfWith(
Op, CCI);
589 SEI.AggrConsts[CCI] = AggrC;
590 SEI.AggrConstTypes[CCI] = SEI.deduceNestedTypeHelper(AggrC);
593 if (
auto *AggrC = dyn_cast<ConstantAggregate>(
Op)) {
595 BuildCompositeIntrinsic(AggrC, Args,
Op,
I,
B, Worklist, KeepInst,
597 }
else if (
auto *AggrC = dyn_cast<ConstantDataArray>(
Op)) {
599 for (
unsigned i = 0; i < AggrC->getNumElements(); ++i)
600 Args.push_back(AggrC->getElementAsConstant(i));
601 BuildCompositeIntrinsic(AggrC, Args,
Op,
I,
B, Worklist, KeepInst,
603 }
else if (isa<ConstantAggregateZero>(
Op) &&
604 !
Op->getType()->isVectorTy()) {
605 auto *AggrC = cast<ConstantAggregateZero>(
Op);
607 BuildCompositeIntrinsic(AggrC, Args,
Op,
I,
B, Worklist, KeepInst,
619 B.SetInsertPoint(&
I);
622 for (
auto &
Op :
I.operands()) {
623 if (
Op.get()->getType()->isSized()) {
625 }
else if (
BasicBlock *BB = dyn_cast<BasicBlock>(
Op.get())) {
632 CallInst *NewI =
B.CreateIntrinsic(Intrinsic::spv_switch,
636 I.replaceAllUsesWith(NewI);
640 B.SetInsertPoint(ParentBB);
651 B.SetInsertPoint(&
I);
654 Args.push_back(
B.getInt1(
I.isInBounds()));
655 for (
auto &
Op :
I.operands())
657 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_gep, {
Types}, {
Args});
658 I.replaceAllUsesWith(NewI);
665 B.SetInsertPoint(&
I);
673 I.replaceAllUsesWith(Source);
680 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_bitcast, {
Types}, {
Args});
681 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
682 I.replaceAllUsesWith(NewI);
688void SPIRVEmitIntrinsics::insertAssignTypeInstrForTargetExtTypes(
691 if (
V->getType() == AssignedType)
697 for (
auto User :
V->users()) {
698 auto *II = dyn_cast<IntrinsicInst>(
User);
699 if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_type)
704 dyn_cast<ConstantAsMetadata>(VMD->
getMetadata())->getType();
705 if (BuiltinType != AssignedType)
708 " for value " +
V->getName(),
714 buildIntrWithMD(Intrinsic::spv_assign_type, {
V->getType()},
Const,
V, {},
B);
717void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
722 while (
BitCastInst *BC = dyn_cast<BitCastInst>(Pointer))
726 Type *PointerElemTy = deduceElementTypeHelper(Pointer);
727 if (PointerElemTy == ExpectedElementType)
731 Constant *ExpectedElementTypeConst =
738 bool FirstPtrCastOrAssignPtrType =
true;
744 auto *II = dyn_cast<IntrinsicInst>(
User);
746 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
747 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
748 II->getOperand(0) != Pointer)
753 FirstPtrCastOrAssignPtrType =
false;
754 if (II->getOperand(1) != VMD ||
755 dyn_cast<ConstantInt>(II->getOperand(2))->getSExtValue() !=
761 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
766 if (II->getParent() !=
I->getParent())
769 I->setOperand(OperandToReplace, II);
780 if (FirstPtrCastOrAssignPtrType &&
781 (isa<Instruction>(Pointer) || isa<Argument>(Pointer))) {
783 Intrinsic::spv_assign_ptr_type, {
Pointer->getType()},
787 AssignPtrTypeInstr[
Pointer] = CI;
794 auto *PtrCastI =
B.CreateIntrinsic(Intrinsic::spv_ptrcast, {
Types},
Args);
795 I->setOperand(OperandToReplace, PtrCastI);
798void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(
Instruction *
I,
804 isa<Argument>(
SI->getValueOperand())) {
805 return replacePointerOperandWithPtrCast(
806 I,
SI->getValueOperand(), IntegerType::getInt8Ty(
F->getContext()), 0,
809 return replacePointerOperandWithPtrCast(
810 I,
SI->getPointerOperand(),
SI->getValueOperand()->getType(), 1,
B);
811 }
else if (
LoadInst *LI = dyn_cast<LoadInst>(
I)) {
812 return replacePointerOperandWithPtrCast(
I, LI->getPointerOperand(),
813 LI->getType(), 0,
B);
815 return replacePointerOperandWithPtrCast(
I, GEPI->getPointerOperand(),
816 GEPI->getSourceElementType(), 0,
B);
826 std::string DemangledName =
830 bool HaveTypes =
false;
831 for (
unsigned OpIdx = 0; OpIdx < CalledF->
arg_size(); ++OpIdx) {
837 CalledArgTys.
push_back(cast<TypedPointerType>(ArgType)->getElementType());
849 if (
Instruction *Inst = dyn_cast<Instruction>(U)) {
850 if ((ElemTy = deduceElementTypeHelper(Inst)) !=
nullptr)
856 HaveTypes |= ElemTy !=
nullptr;
861 if (DemangledName.empty() && !HaveTypes)
864 for (
unsigned OpIdx = 0; OpIdx < CI->
arg_size(); OpIdx++) {
866 if (!isa<PointerType>(ArgOperand->
getType()) &&
867 !isa<TypedPointerType>(ArgOperand->
getType()))
871 if (!isa<Instruction>(ArgOperand) && !isa<Argument>(ArgOperand)) {
881 OpIdx < CalledArgTys.
size() ? CalledArgTys[OpIdx] :
nullptr;
882 if (!ExpectedType && !DemangledName.empty())
884 DemangledName, OpIdx,
I->getContext());
889 insertAssignTypeInstrForTargetExtTypes(cast<TargetExtType>(ExpectedType),
892 replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx,
B);
898 I.getOperand(1)->getType(),
899 I.getOperand(2)->getType()};
901 B.SetInsertPoint(&
I);
903 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_insertelt, {
Types}, {
Args});
904 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
905 I.replaceAllUsesWith(NewI);
914 B.SetInsertPoint(&
I);
916 I.getIndexOperand()->getType()};
918 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_extractelt, {
Types}, {
Args});
919 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
920 I.replaceAllUsesWith(NewI);
928 B.SetInsertPoint(&
I);
931 for (
auto &
Op :
I.operands())
932 if (isa<UndefValue>(
Op))
936 for (
auto &
Op :
I.indices())
937 Args.push_back(
B.getInt32(
Op));
939 B.CreateIntrinsic(Intrinsic::spv_insertv, {
Types}, {
Args});
940 replaceMemInstrUses(&
I, NewI,
B);
946 B.SetInsertPoint(&
I);
948 for (
auto &
Op :
I.operands())
950 for (
auto &
Op :
I.indices())
951 Args.push_back(
B.getInt32(
Op));
953 B.CreateIntrinsic(Intrinsic::spv_extractv, {
I.getType()}, {
Args});
954 I.replaceAllUsesWith(NewI);
960 if (!
I.getType()->isAggregateType())
963 B.SetInsertPoint(&
I);
964 TrackConstants =
false;
965 const auto *TLI =
TM->getSubtargetImpl()->getTargetLowering();
967 TLI->getLoadMemOperandFlags(
I,
F->getParent()->getDataLayout());
969 B.CreateIntrinsic(Intrinsic::spv_load, {
I.getOperand(0)->
getType()},
970 {
I.getPointerOperand(),
B.getInt16(Flags),
971 B.getInt8(
I.getAlign().value())});
972 replaceMemInstrUses(&
I, NewI,
B);
980 B.SetInsertPoint(&
I);
981 TrackConstants =
false;
982 const auto *TLI =
TM->getSubtargetImpl()->getTargetLowering();
984 TLI->getStoreMemOperandFlags(
I,
F->getParent()->getDataLayout());
985 auto *PtrOp =
I.getPointerOperand();
986 auto *NewI =
B.CreateIntrinsic(
987 Intrinsic::spv_store, {
I.getValueOperand()->
getType(), PtrOp->getType()},
988 {
I.getValueOperand(), PtrOp,
B.getInt16(Flags),
989 B.getInt8(
I.getAlign().value())});
995 Value *ArraySize =
nullptr;
996 if (
I.isArrayAllocation()) {
999 SPIRV::Extension::SPV_INTEL_variable_length_array))
1001 "array allocation: this instruction requires the following "
1002 "SPIR-V extension: SPV_INTEL_variable_length_array",
1004 ArraySize =
I.getArraySize();
1007 B.SetInsertPoint(&
I);
1008 TrackConstants =
false;
1009 Type *PtrTy =
I.getType();
1011 ArraySize ?
B.CreateIntrinsic(Intrinsic::spv_alloca_array,
1012 {PtrTy, ArraySize->
getType()}, {ArraySize})
1013 :
B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
1014 std::string InstName =
I.hasName() ?
I.getName().str() :
"";
1015 I.replaceAllUsesWith(NewI);
1016 I.eraseFromParent();
1022 assert(
I.getType()->isAggregateType() &&
"Aggregate result is expected");
1024 B.SetInsertPoint(&
I);
1026 for (
auto &
Op :
I.operands())
1028 Args.push_back(
B.getInt32(
I.getSyncScopeID()));
1029 Args.push_back(
B.getInt32(
1031 Args.push_back(
B.getInt32(
1033 auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_cmpxchg,
1035 replaceMemInstrUses(&
I, NewI,
B);
1041 B.SetInsertPoint(&
I);
1042 B.CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
1049 if (GV.
getName() ==
"llvm.global.annotations")
1055 deduceElementTypeHelper(&GV);
1059 auto *InitInst =
B.CreateIntrinsic(Intrinsic::spv_init_global,
1061 InitInst->setArgOperand(1,
Init);
1065 B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.
getType(), &GV);
1068void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(
Instruction *
I,
1072 isa<BitCastInst>(
I))
1077 Type *ElemTy = deduceElementType(
I);
1080 CallInst *CI = buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {
I->getType()},
1083 AssignPtrTypeInstr[
I] = CI;
1086void SPIRVEmitIntrinsics::insertAssignTypeIntrs(
Instruction *
I,
1089 Type *Ty =
I->getType();
1092 Type *TypeToAssign = Ty;
1093 if (
auto *II = dyn_cast<IntrinsicInst>(
I)) {
1094 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
1095 II->getIntrinsicID() == Intrinsic::spv_undef) {
1096 auto It = AggrConstTypes.
find(II);
1097 if (It == AggrConstTypes.
end())
1099 TypeToAssign = It->second;
1103 buildIntrWithMD(Intrinsic::spv_assign_type, {Ty},
Const,
I, {},
B);
1105 for (
const auto &
Op :
I->operands()) {
1106 if (isa<ConstantPointerNull>(
Op) || isa<UndefValue>(
Op) ||
1108 (isa<ConstantExpr>(
Op) && isa<GEPOperator>(
Op))) {
1110 if (isa<UndefValue>(
Op) &&
Op->getType()->isAggregateType())
1111 buildIntrWithMD(Intrinsic::spv_assign_type, {
B.getInt32Ty()},
Op,
1113 else if (!isa<Instruction>(
Op))
1114 buildIntrWithMD(Intrinsic::spv_assign_type, {
Op->getType()},
Op,
Op, {},
1120void SPIRVEmitIntrinsics::insertSpirvDecorations(
Instruction *
I,
1122 if (
MDNode *MD =
I->getMetadata(
"spirv.Decorations")) {
1123 B.SetInsertPoint(
I->getNextNode());
1124 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {
I->getType()},
1129void SPIRVEmitIntrinsics::processInstrAfterVisit(
Instruction *
I,
1131 auto *II = dyn_cast<IntrinsicInst>(
I);
1132 if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
1134 B.SetInsertPoint(
I->getNextNode());
1135 Type *Ty =
B.getInt32Ty();
1136 auto t = AggrConsts.
find(
I);
1138 auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant, {Ty, Ty},
1139 t->second,
I, {},
B);
1140 I->replaceAllUsesWith(NewOp);
1141 NewOp->setArgOperand(0,
I);
1143 for (
const auto &
Op :
I->operands()) {
1144 if ((isa<ConstantAggregateZero>(
Op) &&
Op->getType()->isVectorTy()) ||
1145 isa<PHINode>(
I) || isa<SwitchInst>(
I))
1146 TrackConstants =
false;
1147 if ((isa<ConstantData>(
Op) || isa<ConstantExpr>(
Op)) && TrackConstants) {
1148 unsigned OpNo =
Op.getOperandNo();
1149 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
1150 (II->paramHasAttr(OpNo, Attribute::ImmArg))))
1152 B.SetInsertPoint(
I);
1154 if (
Op->getType()->isTargetExtTy())
1157 auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
1160 I->setOperand(OpNo, NewOp);
1166 std::vector<Value *>
Args = {
I};
1168 B.CreateIntrinsic(Intrinsic::spv_assign_name, {
I->getType()},
Args);
1172Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
Function *
F,
1174 std::unordered_set<Function *> FVisited;
1175 return deduceFunParamElementType(
F, OpIdx, FVisited);
1178Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
1179 Function *
F,
unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
1181 if (FVisited.find(
F) != FVisited.end())
1185 std::unordered_set<Value *> Visited;
1188 for (
User *U :
F->users()) {
1189 CallInst *CI = dyn_cast<CallInst>(U);
1190 if (!CI || OpIdx >= CI->
arg_size())
1200 if (
Type *Ty = deduceElementTypeHelper(OpArg, Visited))
1205 if (!Inst || Inst == CI)
1208 if (
Type *Ty = deduceElementTypeHelper(Inst, Visited))
1215 if (FVisited.find(OuterF) != FVisited.end())
1217 for (
unsigned i = 0; i < OuterF->
arg_size(); ++i) {
1218 if (OuterF->
getArg(i) == OpArg) {
1219 Lookup.push_back(std::make_pair(OuterF, i));
1226 for (
auto &Pair :
Lookup) {
1227 if (
Type *Ty = deduceFunParamElementType(Pair.first, Pair.second, FVisited))
1234void SPIRVEmitIntrinsics::processParamTypesByFunHeader(
Function *
F,
1236 B.SetInsertPointPastAllocas(
F);
1237 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
1244 buildAssignPtr(
B, ElemTy, Arg);
1249 B.SetInsertPointPastAllocas(
F);
1250 for (
unsigned OpIdx = 0; OpIdx <
F->arg_size(); ++OpIdx) {
1255 if (!ElemTy && (ElemTy = deduceFunParamElementType(
F, OpIdx)) !=
nullptr)
1256 buildAssignPtr(
B, ElemTy, Arg);
1260bool SPIRVEmitIntrinsics::runOnFunction(
Function &Func) {
1261 if (
Func.isDeclaration())
1265 GR =
ST.getSPIRVGlobalRegistry();
1270 AggrConstTypes.
clear();
1273 processParamTypesByFunHeader(
F,
B);
1281 Type *ElTy =
SI->getValueOperand()->getType();
1286 B.SetInsertPoint(&
Func.getEntryBlock(),
Func.getEntryBlock().begin());
1287 for (
auto &GV :
Func.getParent()->globals())
1288 processGlobalValue(GV,
B);
1290 preprocessUndefs(
B);
1291 preprocessCompositeConstants(
B);
1296 for (
auto &
I : Worklist) {
1297 insertAssignPtrTypeIntrs(
I,
B);
1298 insertAssignTypeIntrs(
I,
B);
1299 insertPtrCastOrAssignTypeInstr(
I,
B);
1304 deduceOperandElementType(&
I);
1306 for (
auto *
I : Worklist) {
1307 TrackConstants =
true;
1308 if (!
I->getType()->isVoidTy() || isa<StoreInst>(
I))
1309 B.SetInsertPoint(
I->getNextNode());
1315 processInstrAfterVisit(
I,
B);
1321bool SPIRVEmitIntrinsics::runOnModule(
Module &M) {
1322 bool Changed =
false;
1330 if (!
F.isDeclaration() && !
F.isIntrinsic()) {
1332 GR =
ST.getSPIRVGlobalRegistry();
1334 processParamTypes(&
F,
B);
1342 return new SPIRVEmitIntrinsics(
TM);
Expand Atomic instructions
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool runOnFunction(Function &F, bool PostInlining)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isMemInstrToReplace(Instruction *I)
static bool isAggrToReplace(const Value *V)
static void reportFatalOnTokenType(const Instruction *I)
static Type * getPointeeTypeByCallInst(StringRef DemangledName, Function *CalledF, unsigned OpIdx)
static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I)
static bool requireAssignType(Instruction *I)
static void insertSpirvDecorations(MachineFunction &MF, MachineIRBuilder MIB)
static SymbolRef::Type getType(const Symbol *Sym)
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
an instruction to allocate memory on the stack
Represent the analysis usage information of a pass.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An instruction that atomically checks whether a specified value is in a memory location,...
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVMContext & getContext() const
Get the context in which this basic block lives.
This class represents a no-op cast from one type to another.
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool isIndirectCall() const
Return true if the callsite is an indirect call.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This class represents an Operation in the Expression.
iterator find(const_arg_type_t< KeyT > Val)
Implements a dense probed hash-table based set.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
PointerType * getType() const
Global values are always pointers.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Indirect Branch Instruction.
void addDestination(BasicBlock *Dest)
Add a destination.
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
RetTy visitExtractElementInst(ExtractElementInst &I)
RetTy visitInsertValueInst(InsertValueInst &I)
RetTy visitUnreachableInst(UnreachableInst &I)
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
RetTy visitBitCastInst(BitCastInst &I)
RetTy visitSwitchInst(SwitchInst &I)
RetTy visitExtractValueInst(ExtractValueInst &I)
RetTy visitStoreInst(StoreInst &I)
RetTy visitInsertElementInst(InsertElementInst &I)
RetTy visitAllocaInst(AllocaInst &I)
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
void visitInstruction(Instruction &I)
RetTy visitLoadInst(LoadInst &I)
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Flags
Flags values. These may be or'd together.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Type * findDeducedCompositeType(const Value *Val)
void addDeducedElementType(Value *Val, Type *Ty)
unsigned getPointerSize() const
void addReturnType(const Function *ArgF, TypedPointerType *DerivedTy)
void addDeducedCompositeType(Value *Val, Type *Ty)
Type * findDeducedElementType(const Value *Val)
bool canUseExtension(SPIRV::Extension::Extension E) const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Class to represent target extensions types, which are generally unintrospectable from target-independ...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
StringRef getTargetExtName() const
bool isTargetExtTy() const
Return true if this is a target extension type.
bool isAggregateType() const
Return true if the type is an aggregate type.
bool isVoidTy() const
Return true if this is 'void'.
A few GPU targets, such as DXIL and SPIR-V, have typed pointers.
static TypedPointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
iterator_range< user_iterator > users()
unsigned getNumUses() const
This method computes the number of uses of this Value.
StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ SPIR_KERNEL
Used for SPIR kernel functions.
Type * parseBuiltinCallArgumentBaseType(const StringRef DemangledCall, unsigned ArgIdx, LLVMContext &Ctx)
Parses the provided ArgIdx argument base type in the DemangledCall skeleton.
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
void initializeSPIRVEmitIntrinsicsPass(PassRegistry &)
ModulePass * createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM)
unsigned getPointerAddressSpace(const Type *T)
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name)
bool isTypedPointerTy(const Type *T)
bool isPointerTy(const Type *T)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
@ Ref
The access may reference the value stored in memory.
DWARFExpression::Operation Op
Type * getPointeeTypeByAttr(Argument *Arg)
bool hasPointeeTypeAttr(Argument *Arg)
void addStringImm(const StringRef &Str, MCInst &Inst)
bool isUntypedPointerTy(const Type *T)
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)