LLVM 19.0.0git
SPIRVEmitIntrinsics.cpp
Go to the documentation of this file.
1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVMetadata.h"
16#include "SPIRVTargetMachine.h"
17#include "SPIRVUtils.h"
18#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/InstVisitor.h"
21#include "llvm/IR/IntrinsicsSPIRV.h"
22
23#include <queue>
24
25// This pass performs the following transformation on LLVM IR level required
26// for the following translation to SPIR-V:
27// - replaces direct usages of aggregate constants with target-specific
28// intrinsics;
29// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
30// with a target-specific intrinsics;
31// - emits intrinsics for the global variable initializers since IRTranslator
32// doesn't handle them and it's not very convenient to translate them
33// ourselves;
34// - emits intrinsics to keep track of the string names assigned to the values;
35// - emits intrinsics to keep track of constants (this is necessary to have an
36// LLVM IR constant after the IRTranslation is completed) for their further
37// deduplication;
38// - emits intrinsics to keep track of original LLVM types of the values
39// to be able to emit proper SPIR-V types eventually.
40//
41// TODO: consider removing spv.track.constant in favor of spv.assign.type.
42
43using namespace llvm;
44
45namespace llvm {
47} // namespace llvm
48
49namespace {
50class SPIRVEmitIntrinsics
51 : public FunctionPass,
52 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
53 SPIRVTargetMachine *TM = nullptr;
54 IRBuilder<> *IRB = nullptr;
55 Function *F = nullptr;
56 bool TrackConstants = true;
58 DenseSet<Instruction *> AggrStores;
59 void preprocessCompositeConstants();
60 void preprocessUndefs();
61 CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
62 Value *Arg, Value *Arg2,
65 MDTuple *TyMD = MDNode::get(F->getContext(), CM);
66 MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD);
68 Args.push_back(Arg2);
69 Args.push_back(VMD);
70 for (auto *Imm : Imms)
71 Args.push_back(Imm);
72 return IRB->CreateIntrinsic(IntrID, {Types}, Args);
73 }
74 void replaceMemInstrUses(Instruction *Old, Instruction *New);
75 void processInstrAfterVisit(Instruction *I);
76 void insertAssignPtrTypeIntrs(Instruction *I);
77 void insertAssignTypeIntrs(Instruction *I);
78 void insertPtrCastInstr(Instruction *I);
79 void processGlobalValue(GlobalVariable &GV);
80
81public:
82 static char ID;
83 SPIRVEmitIntrinsics() : FunctionPass(ID) {
85 }
86 SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : FunctionPass(ID), TM(_TM) {
88 }
102 bool runOnFunction(Function &F) override;
103};
104} // namespace
105
106char SPIRVEmitIntrinsics::ID = 0;
107
108INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
109 false, false)
110
111static inline bool isAssignTypeInstr(const Instruction *I) {
112 return isa<IntrinsicInst>(I) &&
113 cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::spv_assign_type;
114}
115
117 return isa<StoreInst>(I) || isa<LoadInst>(I) || isa<InsertValueInst>(I) ||
118 isa<ExtractValueInst>(I) || isa<AtomicCmpXchgInst>(I);
119}
120
121static bool isAggrToReplace(const Value *V) {
122 return isa<ConstantAggregate>(V) || isa<ConstantDataArray>(V) ||
123 (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
124}
125
127 if (isa<PHINode>(I))
128 B.SetInsertPoint(I->getParent(), I->getParent()->getFirstInsertionPt());
129 else
130 B.SetInsertPoint(I);
131}
132
134 if (isa<AllocaInst>(I) || isa<GetElementPtrInst>(I))
135 return true;
136
137 return false;
138}
139
141 IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(I);
142 if (Intr) {
143 switch (Intr->getIntrinsicID()) {
144 case Intrinsic::invariant_start:
145 case Intrinsic::invariant_end:
146 return false;
147 }
148 }
149 return true;
150}
151
152static inline void reportFatalOnTokenType(const Instruction *I) {
153 if (I->getType()->isTokenTy())
154 report_fatal_error("A token is encountered but SPIR-V without extensions "
155 "does not support token type",
156 false);
157}
158
159void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
160 Instruction *New) {
161 while (!Old->user_empty()) {
162 auto *U = Old->user_back();
163 if (isAssignTypeInstr(U)) {
164 IRB->SetInsertPoint(U);
165 SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
166 IRB->CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
167 U->eraseFromParent();
168 } else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
169 isa<CallInst>(U)) {
170 U->replaceUsesOfWith(Old, New);
171 } else {
172 llvm_unreachable("illegal aggregate intrinsic user");
173 }
174 }
175 Old->eraseFromParent();
176}
177
178void SPIRVEmitIntrinsics::preprocessUndefs() {
179 std::queue<Instruction *> Worklist;
180 for (auto &I : instructions(F))
181 Worklist.push(&I);
182
183 while (!Worklist.empty()) {
184 Instruction *I = Worklist.front();
185 Worklist.pop();
186
187 for (auto &Op : I->operands()) {
188 auto *AggrUndef = dyn_cast<UndefValue>(Op);
189 if (!AggrUndef || !Op->getType()->isAggregateType())
190 continue;
191
192 IRB->SetInsertPoint(I);
193 auto *IntrUndef = IRB->CreateIntrinsic(Intrinsic::spv_undef, {}, {});
194 Worklist.push(IntrUndef);
195 I->replaceUsesOfWith(Op, IntrUndef);
196 AggrConsts[IntrUndef] = AggrUndef;
197 }
198 }
199}
200
201void SPIRVEmitIntrinsics::preprocessCompositeConstants() {
202 std::queue<Instruction *> Worklist;
203 for (auto &I : instructions(F))
204 Worklist.push(&I);
205
206 while (!Worklist.empty()) {
207 auto *I = Worklist.front();
208 assert(I);
209 bool KeepInst = false;
210 for (const auto &Op : I->operands()) {
211 auto BuildCompositeIntrinsic = [&KeepInst, &Worklist, &I, &Op,
212 this](Constant *AggrC,
214 IRB->SetInsertPoint(I);
215 auto *CCI =
216 IRB->CreateIntrinsic(Intrinsic::spv_const_composite, {}, {Args});
217 Worklist.push(CCI);
218 I->replaceUsesOfWith(Op, CCI);
219 KeepInst = true;
220 AggrConsts[CCI] = AggrC;
221 };
222
223 if (auto *AggrC = dyn_cast<ConstantAggregate>(Op)) {
224 SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
225 BuildCompositeIntrinsic(AggrC, Args);
226 } else if (auto *AggrC = dyn_cast<ConstantDataArray>(Op)) {
228 for (unsigned i = 0; i < AggrC->getNumElements(); ++i)
229 Args.push_back(AggrC->getElementAsConstant(i));
230 BuildCompositeIntrinsic(AggrC, Args);
231 } else if (isa<ConstantAggregateZero>(Op) &&
232 !Op->getType()->isVectorTy()) {
233 auto *AggrC = cast<ConstantAggregateZero>(Op);
234 SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
235 BuildCompositeIntrinsic(AggrC, Args);
236 }
237 }
238 if (!KeepInst)
239 Worklist.pop();
240 }
241}
242
243Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
245 for (auto &Op : I.operands())
246 if (Op.get()->getType()->isSized())
247 Args.push_back(Op);
248 IRB->SetInsertPoint(&I);
249 IRB->CreateIntrinsic(Intrinsic::spv_switch, {I.getOperand(0)->getType()},
250 {Args});
251 return &I;
252}
253
254Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
255 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
257 Args.push_back(IRB->getInt1(I.isInBounds()));
258 for (auto &Op : I.operands())
259 Args.push_back(Op);
260 auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
261 I.replaceAllUsesWith(NewI);
262 I.eraseFromParent();
263 return NewI;
264}
265
266Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
267 Value *Source = I.getOperand(0);
268
269 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
270 // varying element types. In case of IR coming from older versions of LLVM
271 // such bitcasts do not provide sufficient information, should be just skipped
272 // here, and handled in insertPtrCastInstr.
273 if (I.getType()->isPointerTy()) {
274 I.replaceAllUsesWith(Source);
275 I.eraseFromParent();
276 return nullptr;
277 }
278
279 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
280 SmallVector<Value *> Args(I.op_begin(), I.op_end());
281 auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
282 std::string InstName = I.hasName() ? I.getName().str() : "";
283 I.replaceAllUsesWith(NewI);
284 I.eraseFromParent();
285 NewI->setName(InstName);
286 return NewI;
287}
288
289void SPIRVEmitIntrinsics::insertPtrCastInstr(Instruction *I) {
290 Value *Pointer;
291 Type *ExpectedElementType;
292 unsigned OperandToReplace;
293
294 StoreInst *SI = dyn_cast<StoreInst>(I);
295 if (SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
296 SI->getValueOperand()->getType()->isPointerTy() &&
297 isa<Argument>(SI->getValueOperand())) {
298 Pointer = SI->getValueOperand();
299 ExpectedElementType = IntegerType::getInt8Ty(F->getContext());
300 OperandToReplace = 0;
301 } else if (SI) {
302 Pointer = SI->getPointerOperand();
303 ExpectedElementType = SI->getValueOperand()->getType();
304 OperandToReplace = 1;
305 } else if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
306 Pointer = LI->getPointerOperand();
307 ExpectedElementType = LI->getType();
308 OperandToReplace = 0;
309 } else if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
310 Pointer = GEPI->getPointerOperand();
311 ExpectedElementType = GEPI->getSourceElementType();
312 OperandToReplace = 0;
313 } else {
314 return;
315 }
316
317 // If Pointer is the result of nop BitCastInst (ptr -> ptr), use the source
318 // pointer instead. The BitCastInst should be later removed when visited.
319 while (BitCastInst *BC = dyn_cast<BitCastInst>(Pointer))
320 Pointer = BC->getOperand(0);
321
322 // Do not emit spv_ptrcast if Pointer is a GlobalValue of expected type.
323 GlobalValue *GV = dyn_cast<GlobalValue>(Pointer);
324 if (GV && GV->getValueType() == ExpectedElementType)
325 return;
326
327 // Do not emit spv_ptrcast if Pointer is a result of alloca with expected
328 // type.
329 AllocaInst *A = dyn_cast<AllocaInst>(Pointer);
330 if (A && A->getAllocatedType() == ExpectedElementType)
331 return;
332
333 // Do not emit spv_ptrcast if Pointer is a result of GEP of expected type.
334 GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Pointer);
335 if (GEPI && GEPI->getResultElementType() == ExpectedElementType)
336 return;
337
339 Constant *ExpectedElementTypeConst =
340 Constant::getNullValue(ExpectedElementType);
342 ValueAsMetadata::getConstant(ExpectedElementTypeConst);
343 MDTuple *TyMD = MDNode::get(F->getContext(), CM);
344 MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD);
345 unsigned AddressSpace = Pointer->getType()->getPointerAddressSpace();
346 bool FirstPtrCastOrAssignPtrType = true;
347
348 // Do not emit new spv_ptrcast if equivalent one already exists or when
349 // spv_assign_ptr_type already targets this pointer with the same element
350 // type.
351 for (auto User : Pointer->users()) {
352 auto *II = dyn_cast<IntrinsicInst>(User);
353 if (!II ||
354 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
355 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
356 II->getOperand(0) != Pointer)
357 continue;
358
359 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
360 // pointer.
361 FirstPtrCastOrAssignPtrType = false;
362 if (II->getOperand(1) != VMD ||
363 dyn_cast<ConstantInt>(II->getOperand(2))->getSExtValue() !=
365 continue;
366
367 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the same
368 // element type and address space.
369 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
370 return;
371
372 // This must be a spv_ptrcast, do not emit new if this one has the same BB
373 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
374 if (II->getParent() != I->getParent())
375 continue;
376
377 I->setOperand(OperandToReplace, II);
378 return;
379 }
380
381 // Do not emit spv_ptrcast if it would cast to the default pointer element
382 // type (i8) of the same address space. In case of OpenCL kernels, make sure
383 // i8 is the pointer element type defined for the given kernel argument.
384 if (ExpectedElementType->isIntegerTy(8) &&
385 F->getCallingConv() != CallingConv::SPIR_KERNEL)
386 return;
387
388 Argument *Arg = dyn_cast<Argument>(Pointer);
389 if (ExpectedElementType->isIntegerTy(8) &&
390 F->getCallingConv() == CallingConv::SPIR_KERNEL && Arg) {
391 MDString *ArgType = getOCLKernelArgType(*Arg->getParent(), Arg->getArgNo());
392 if (ArgType && ArgType->getString().starts_with("uchar*"))
393 return;
394 }
395
396 // If this would be the first spv_ptrcast, the pointer's defining instruction
397 // requires spv_assign_ptr_type and does not already have one, do not emit
398 // spv_ptrcast and emit spv_assign_ptr_type instead.
399 Instruction *PointerDefInst = dyn_cast<Instruction>(Pointer);
400 if (FirstPtrCastOrAssignPtrType && PointerDefInst &&
401 requireAssignPtrType(PointerDefInst)) {
402 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Pointer->getType()},
403 ExpectedElementTypeConst, Pointer,
404 {IRB->getInt32(AddressSpace)});
405 return;
406 } else {
407 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
409 auto *PtrCastI =
410 IRB->CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
411 I->setOperand(OperandToReplace, PtrCastI);
412 return;
413 }
414}
415
416Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
417 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(0)->getType(),
418 I.getOperand(1)->getType(),
419 I.getOperand(2)->getType()};
420 SmallVector<Value *> Args(I.op_begin(), I.op_end());
421 auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
422 std::string InstName = I.hasName() ? I.getName().str() : "";
423 I.replaceAllUsesWith(NewI);
424 I.eraseFromParent();
425 NewI->setName(InstName);
426 return NewI;
427}
428
430SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
431 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
432 I.getIndexOperand()->getType()};
433 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
434 auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
435 std::string InstName = I.hasName() ? I.getName().str() : "";
436 I.replaceAllUsesWith(NewI);
437 I.eraseFromParent();
438 NewI->setName(InstName);
439 return NewI;
440}
441
442Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
443 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
445 for (auto &Op : I.operands())
446 if (isa<UndefValue>(Op))
447 Args.push_back(UndefValue::get(IRB->getInt32Ty()));
448 else
449 Args.push_back(Op);
450 for (auto &Op : I.indices())
451 Args.push_back(IRB->getInt32(Op));
452 Instruction *NewI =
453 IRB->CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
454 replaceMemInstrUses(&I, NewI);
455 return NewI;
456}
457
458Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
460 for (auto &Op : I.operands())
461 Args.push_back(Op);
462 for (auto &Op : I.indices())
463 Args.push_back(IRB->getInt32(Op));
464 auto *NewI =
465 IRB->CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
466 I.replaceAllUsesWith(NewI);
467 I.eraseFromParent();
468 return NewI;
469}
470
471Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
472 if (!I.getType()->isAggregateType())
473 return &I;
474 TrackConstants = false;
475 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
477 TLI->getLoadMemOperandFlags(I, F->getParent()->getDataLayout());
478 auto *NewI =
479 IRB->CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
480 {I.getPointerOperand(), IRB->getInt16(Flags),
481 IRB->getInt8(I.getAlign().value())});
482 replaceMemInstrUses(&I, NewI);
483 return NewI;
484}
485
486Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
487 if (!AggrStores.contains(&I))
488 return &I;
489 TrackConstants = false;
490 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
492 TLI->getStoreMemOperandFlags(I, F->getParent()->getDataLayout());
493 auto *PtrOp = I.getPointerOperand();
494 auto *NewI = IRB->CreateIntrinsic(
495 Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
496 {I.getValueOperand(), PtrOp, IRB->getInt16(Flags),
497 IRB->getInt8(I.getAlign().value())});
498 I.eraseFromParent();
499 return NewI;
500}
501
502Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
503 Value *ArraySize = nullptr;
504 if (I.isArrayAllocation()) {
505 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction());
506 if (!STI->canUseExtension(
507 SPIRV::Extension::SPV_INTEL_variable_length_array))
509 "array allocation: this instruction requires the following "
510 "SPIR-V extension: SPV_INTEL_variable_length_array",
511 false);
512 ArraySize = I.getArraySize();
513 }
514
515 TrackConstants = false;
516 Type *PtrTy = I.getType();
517 auto *NewI =
518 ArraySize
519 ? IRB->CreateIntrinsic(Intrinsic::spv_alloca_array,
520 {PtrTy, ArraySize->getType()}, {ArraySize})
521 : IRB->CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
522 std::string InstName = I.hasName() ? I.getName().str() : "";
523 I.replaceAllUsesWith(NewI);
524 I.eraseFromParent();
525 NewI->setName(InstName);
526 return NewI;
527}
528
529Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
530 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
532 for (auto &Op : I.operands())
533 Args.push_back(Op);
534 Args.push_back(IRB->getInt32(I.getSyncScopeID()));
535 Args.push_back(IRB->getInt32(
536 static_cast<uint32_t>(getMemSemantics(I.getSuccessOrdering()))));
537 Args.push_back(IRB->getInt32(
538 static_cast<uint32_t>(getMemSemantics(I.getFailureOrdering()))));
539 auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_cmpxchg,
540 {I.getPointerOperand()->getType()}, {Args});
541 replaceMemInstrUses(&I, NewI);
542 return NewI;
543}
544
545Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
546 IRB->SetInsertPoint(&I);
547 IRB->CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
548 return &I;
549}
550
551void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV) {
552 // Skip special artifical variable llvm.global.annotations.
553 if (GV.getName() == "llvm.global.annotations")
554 return;
555 if (GV.hasInitializer() && !isa<UndefValue>(GV.getInitializer())) {
557 Type *Ty = isAggrToReplace(Init) ? IRB->getInt32Ty() : Init->getType();
559 auto *InitInst = IRB->CreateIntrinsic(Intrinsic::spv_init_global,
560 {GV.getType(), Ty}, {&GV, Const});
561 InitInst->setArgOperand(1, Init);
562 }
563 if ((!GV.hasInitializer() || isa<UndefValue>(GV.getInitializer())) &&
564 GV.getNumUses() == 0)
565 IRB->CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV);
566}
567
568void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I) {
570 if (I->getType()->isVoidTy() || !requireAssignPtrType(I))
571 return;
572
573 setInsertPointSkippingPhis(*IRB, I->getNextNode());
574
575 Constant *EltTyConst;
576 unsigned AddressSpace = 0;
577 if (auto *AI = dyn_cast<AllocaInst>(I)) {
578 EltTyConst = UndefValue::get(AI->getAllocatedType());
579 AddressSpace = AI->getAddressSpace();
580 } else if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
581 EltTyConst = UndefValue::get(GEP->getResultElementType());
582 AddressSpace = GEP->getPointerAddressSpace();
583 } else {
584 llvm_unreachable("Unexpected instruction!");
585 }
586
587 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()}, EltTyConst, I,
588 {IRB->getInt32(AddressSpace)});
589}
590
591void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I) {
593 Type *Ty = I->getType();
594 if (!Ty->isVoidTy() && requireAssignType(I) && !requireAssignPtrType(I)) {
595 setInsertPointSkippingPhis(*IRB, I->getNextNode());
596 Type *TypeToAssign = Ty;
597 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
598 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
599 II->getIntrinsicID() == Intrinsic::spv_undef) {
600 auto t = AggrConsts.find(II);
601 assert(t != AggrConsts.end());
602 TypeToAssign = t->second->getType();
603 }
604 }
605 Constant *Const = UndefValue::get(TypeToAssign);
606 buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I, {});
607 }
608 for (const auto &Op : I->operands()) {
609 if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
610 // Check GetElementPtrConstantExpr case.
611 (isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
613 if (isa<UndefValue>(Op) && Op->getType()->isAggregateType())
614 buildIntrWithMD(Intrinsic::spv_assign_type, {IRB->getInt32Ty()}, Op,
615 UndefValue::get(IRB->getInt32Ty()), {});
616 else
617 buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op,
618 {});
619 }
620 }
621}
622
623void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I) {
624 auto *II = dyn_cast<IntrinsicInst>(I);
625 if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
626 TrackConstants) {
627 IRB->SetInsertPoint(I->getNextNode());
628 Type *Ty = IRB->getInt32Ty();
629 auto t = AggrConsts.find(I);
630 assert(t != AggrConsts.end());
631 auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant, {Ty, Ty},
632 t->second, I, {});
633 I->replaceAllUsesWith(NewOp);
634 NewOp->setArgOperand(0, I);
635 }
636 for (const auto &Op : I->operands()) {
637 if ((isa<ConstantAggregateZero>(Op) && Op->getType()->isVectorTy()) ||
638 isa<PHINode>(I) || isa<SwitchInst>(I))
639 TrackConstants = false;
640 if ((isa<ConstantData>(Op) || isa<ConstantExpr>(Op)) && TrackConstants) {
641 unsigned OpNo = Op.getOperandNo();
642 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
643 (II->paramHasAttr(OpNo, Attribute::ImmArg))))
644 continue;
645 IRB->SetInsertPoint(I);
646 auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
647 {Op->getType(), Op->getType()}, Op, Op, {});
648 I->setOperand(OpNo, NewOp);
649 }
650 }
651 if (I->hasName()) {
653 setInsertPointSkippingPhis(*IRB, I->getNextNode());
654 std::vector<Value *> Args = {I};
655 addStringImm(I->getName(), *IRB, Args);
656 IRB->CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
657 }
658}
659
660bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
661 if (Func.isDeclaration())
662 return false;
663 F = &Func;
664 IRB = new IRBuilder<>(Func.getContext());
665 AggrConsts.clear();
666 AggrStores.clear();
667
668 // StoreInst's operand type can be changed during the next transformations,
669 // so we need to store it in the set. Also store already transformed types.
670 for (auto &I : instructions(Func)) {
671 StoreInst *SI = dyn_cast<StoreInst>(&I);
672 if (!SI)
673 continue;
674 Type *ElTy = SI->getValueOperand()->getType();
675 if (ElTy->isAggregateType() || ElTy->isVectorTy())
676 AggrStores.insert(&I);
677 }
678
679 IRB->SetInsertPoint(&Func.getEntryBlock(), Func.getEntryBlock().begin());
680 for (auto &GV : Func.getParent()->globals())
681 processGlobalValue(GV);
682
683 preprocessUndefs();
684 preprocessCompositeConstants();
686 for (auto &I : instructions(Func))
687 Worklist.push_back(&I);
688
689 for (auto &I : Worklist) {
690 insertAssignPtrTypeIntrs(I);
691 insertAssignTypeIntrs(I);
692 insertPtrCastInstr(I);
693 }
694
695 for (auto *I : Worklist) {
696 TrackConstants = true;
697 if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
698 IRB->SetInsertPoint(I->getNextNode());
699 // Visitors return either the original/newly created instruction for further
700 // processing, nullptr otherwise.
701 I = visit(*I);
702 if (!I)
703 continue;
704 processInstrAfterVisit(I);
705 }
706 return true;
707}
708
710 return new SPIRVEmitIntrinsics(TM);
711}
aarch64 promote const
unsigned Intr
always inline
Expand Atomic instructions
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Hexagon Common GEP
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isMemInstrToReplace(Instruction *I)
static bool isAggrToReplace(const Value *V)
static void reportFatalOnTokenType(const Instruction *I)
static bool requireAssignPtrType(Instruction *I)
static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I)
static bool requireAssignType(Instruction *I)
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
an instruction to allocate memory on the stack
Definition: Instructions.h:59
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
const Function * getParent() const
Definition: Argument.h:40
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:46
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
This class represents a no-op cast from one type to another.
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1653
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
This class represents an Operation in the Expression.
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
This instruction extracts a single (scalar) element from a VectorType value.
This instruction extracts a struct member or array element value from an aggregate value.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
virtual bool runOnFunction(Function &F)=0
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
Type * getResultElementType() const
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:294
Type * getValueType() const
Definition: GlobalValue.h:296
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Definition: IRBuilder.h:455
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:930
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:520
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Definition: IRBuilder.h:470
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:480
ConstantInt * getInt16(uint16_t C)
Get a constant 16-bit value.
Definition: IRBuilder.h:475
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:180
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
Definition: InstVisitor.h:78
RetTy visitExtractElementInst(ExtractElementInst &I)
Definition: InstVisitor.h:191
RetTy visitInsertValueInst(InsertValueInst &I)
Definition: InstVisitor.h:195
RetTy visitUnreachableInst(UnreachableInst &I)
Definition: InstVisitor.h:241
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
Definition: InstVisitor.h:171
RetTy visitBitCastInst(BitCastInst &I)
Definition: InstVisitor.h:187
RetTy visitSwitchInst(SwitchInst &I)
Definition: InstVisitor.h:232
RetTy visitExtractValueInst(ExtractValueInst &I)
Definition: InstVisitor.h:194
RetTy visitStoreInst(StoreInst &I)
Definition: InstVisitor.h:170
RetTy visitInsertElementInst(InsertElementInst &I)
Definition: InstVisitor.h:192
RetTy visitAllocaInst(AllocaInst &I)
Definition: InstVisitor.h:168
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
Definition: InstVisitor.h:174
void visitInstruction(Instruction &I)
Definition: InstVisitor.h:280
RetTy visitLoadInst(LoadInst &I)
Definition: InstVisitor.h:169
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:147
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
An instruction for reading from memory.
Definition: Instructions.h:184
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
A single uniqued string.
Definition: Metadata.h:720
StringRef getString() const
Definition: Metadata.cpp:607
Tuple of metadata.
Definition: Metadata.h:1470
Flags
Flags values. These may be or'd together.
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:176
static MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition: Metadata.cpp:103
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:37
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
bool canUseExtension(SPIRV::Extension::Extension E) const
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:317
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:257
Multiway switch.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:295
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
This function has undefined behavior.
static ConstantAsMetadata * getConstant(Value *C)
Definition: Metadata.h:472
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
unsigned getNumUses() const
This method computes the number of uses of this Value.
Definition: Value.cpp:255
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
bool user_empty() const
Definition: Value.h:385
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ SPIR_KERNEL
Used for SPIR kernel functions.
Definition: CallingConv.h:144
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void initializeSPIRVEmitIntrinsicsPass(PassRegistry &)
AddressSpace
Definition: NVPTXBaseInfo.h:21
FunctionPass * createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
MDString * getOCLKernelArgType(const Function &F, unsigned ArgIdx)
DWARFExpression::Operation Op
void addStringImm(const StringRef &Str, MCInst &Inst)
Definition: SPIRVUtils.cpp:51
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
Definition: SPIRVUtils.cpp:208