LLVM 20.0.0git
SPIRVInstructionSelector.cpp
Go to the documentation of this file.
1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
21#include "SPIRVRegisterInfo.h"
22#include "SPIRVTargetMachine.h"
23#include "SPIRVUtils.h"
24#include "llvm/ADT/APFloat.h"
34#include "llvm/IR/IntrinsicsSPIRV.h"
35#include "llvm/Support/Debug.h"
37
38#define DEBUG_TYPE "spirv-isel"
39
40using namespace llvm;
41namespace CL = SPIRV::OpenCLExtInst;
42namespace GL = SPIRV::GLSLExtInst;
43
45 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
46
47namespace {
48
49llvm::SPIRV::SelectionControl::SelectionControl
50getSelectionOperandForImm(int Imm) {
51 if (Imm == 2)
52 return SPIRV::SelectionControl::Flatten;
53 if (Imm == 1)
54 return SPIRV::SelectionControl::DontFlatten;
55 if (Imm == 0)
56 return SPIRV::SelectionControl::None;
57 llvm_unreachable("Invalid immediate");
58}
59
60#define GET_GLOBALISEL_PREDICATE_BITSET
61#include "SPIRVGenGlobalISel.inc"
62#undef GET_GLOBALISEL_PREDICATE_BITSET
63
64class SPIRVInstructionSelector : public InstructionSelector {
65 const SPIRVSubtarget &STI;
66 const SPIRVInstrInfo &TII;
68 const RegisterBankInfo &RBI;
71 MachineFunction *HasVRegsReset = nullptr;
72
73 /// We need to keep track of the number we give to anonymous global values to
74 /// generate the same name every time when this is needed.
75 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
77
78public:
79 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
80 const SPIRVSubtarget &ST,
81 const RegisterBankInfo &RBI);
82 void setupMF(MachineFunction &MF, GISelKnownBits *KB,
83 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
84 BlockFrequencyInfo *BFI) override;
85 // Common selection code. Instruction-specific selection occurs in spvSelect.
86 bool select(MachineInstr &I) override;
87 static const char *getName() { return DEBUG_TYPE; }
88
89#define GET_GLOBALISEL_PREDICATES_DECL
90#include "SPIRVGenGlobalISel.inc"
91#undef GET_GLOBALISEL_PREDICATES_DECL
92
93#define GET_GLOBALISEL_TEMPORARIES_DECL
94#include "SPIRVGenGlobalISel.inc"
95#undef GET_GLOBALISEL_TEMPORARIES_DECL
96
97private:
98 void resetVRegsType(MachineFunction &MF);
99
100 // tblgen-erated 'select' implementation, used as the initial selector for
101 // the patterns that don't require complex C++.
102 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
103
104 // All instruction-specific selection that didn't happen in "select()".
105 // Is basically a large Switch/Case delegating to all other select method.
106 bool spvSelect(Register ResVReg, const SPIRVType *ResType,
107 MachineInstr &I) const;
108
109 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType,
110 MachineInstr &I, bool IsSigned) const;
111
112 bool selectFirstBitLow(Register ResVReg, const SPIRVType *ResType,
113 MachineInstr &I) const;
114
115 bool selectFirstBitSet16(Register ResVReg, const SPIRVType *ResType,
116 MachineInstr &I, unsigned ExtendOpcode,
117 unsigned BitSetOpcode) const;
118
119 bool selectFirstBitSet32(Register ResVReg, const SPIRVType *ResType,
120 MachineInstr &I, Register SrcReg,
121 unsigned BitSetOpcode) const;
122
123 bool selectFirstBitSet64(Register ResVReg, const SPIRVType *ResType,
124 MachineInstr &I, Register SrcReg,
125 unsigned BitSetOpcode, bool SwapPrimarySide) const;
126
127 bool selectFirstBitSet64Overflow(Register ResVReg, const SPIRVType *ResType,
128 MachineInstr &I, Register SrcReg,
129 unsigned BitSetOpcode,
130 bool SwapPrimarySide) const;
131
132 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
133 const MachineInstr *Init = nullptr) const;
134
135 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType,
136 MachineInstr &I, std::vector<Register> SrcRegs,
137 unsigned Opcode) const;
138
139 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
140 unsigned Opcode) const;
141
142 bool selectBitcast(Register ResVReg, const SPIRVType *ResType,
143 MachineInstr &I) const;
144
145 bool selectLoad(Register ResVReg, const SPIRVType *ResType,
146 MachineInstr &I) const;
147 bool selectStore(MachineInstr &I) const;
148
149 bool selectStackSave(Register ResVReg, const SPIRVType *ResType,
150 MachineInstr &I) const;
151 bool selectStackRestore(MachineInstr &I) const;
152
153 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
154
155 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType,
156 MachineInstr &I, unsigned NewOpcode,
157 unsigned NegateOpcode = 0) const;
158
159 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType,
160 MachineInstr &I) const;
161
162 bool selectFence(MachineInstr &I) const;
163
164 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
165 MachineInstr &I) const;
166
167 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType,
168 MachineInstr &I, unsigned OpType) const;
169
170 bool selectAll(Register ResVReg, const SPIRVType *ResType,
171 MachineInstr &I) const;
172
173 bool selectAny(Register ResVReg, const SPIRVType *ResType,
174 MachineInstr &I) const;
175
176 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
177 MachineInstr &I) const;
178
179 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType,
180 MachineInstr &I) const;
181 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType,
182 MachineInstr &I) const;
183
184 bool selectCmp(Register ResVReg, const SPIRVType *ResType,
185 unsigned comparisonOpcode, MachineInstr &I) const;
186 bool selectCross(Register ResVReg, const SPIRVType *ResType,
187 MachineInstr &I) const;
188 bool selectDiscard(Register ResVReg, const SPIRVType *ResType,
189 MachineInstr &I) const;
190
191 bool selectICmp(Register ResVReg, const SPIRVType *ResType,
192 MachineInstr &I) const;
193 bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
194 MachineInstr &I) const;
195
196 bool selectSign(Register ResVReg, const SPIRVType *ResType,
197 MachineInstr &I) const;
198
199 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType,
200 MachineInstr &I) const;
201
202 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType,
203 MachineInstr &I, unsigned Opcode) const;
204
205 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType,
206 MachineInstr &I, bool Signed) const;
207
208 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType,
209 MachineInstr &I) const;
210
211 template <bool Signed>
212 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType,
213 MachineInstr &I) const;
214 template <bool Signed>
215 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType,
216 MachineInstr &I) const;
217
218 bool selectWaveReduceMax(Register ResVReg, const SPIRVType *ResType,
219 MachineInstr &I, bool IsUnsigned) const;
220
221 bool selectWaveReduceSum(Register ResVReg, const SPIRVType *ResType,
222 MachineInstr &I) const;
223
224 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
225 int OpIdx) const;
226 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
227 int OpIdx) const;
228
229 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm,
230 MachineInstr &I) const;
231
232 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
233 bool IsSigned) const;
234 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
235 bool IsSigned, unsigned Opcode) const;
236 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
237 bool IsSigned) const;
238
239 bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
240 MachineInstr &I) const;
241
242 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
243 bool IsSigned) const;
244
245 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
246 const SPIRVType *intTy, const SPIRVType *boolTy) const;
247
248 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
249 MachineInstr &I) const;
250 bool selectFreeze(Register ResVReg, const SPIRVType *ResType,
251 MachineInstr &I) const;
252 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType,
253 MachineInstr &I) const;
254 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType,
255 MachineInstr &I) const;
256 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType,
257 MachineInstr &I) const;
258 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType,
259 MachineInstr &I) const;
260 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType,
261 MachineInstr &I) const;
262 bool selectGEP(Register ResVReg, const SPIRVType *ResType,
263 MachineInstr &I) const;
264
265 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType,
266 MachineInstr &I) const;
267 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType,
268 MachineInstr &I) const;
269
270 bool selectBranch(MachineInstr &I) const;
271 bool selectBranchCond(MachineInstr &I) const;
272
273 bool selectPhi(Register ResVReg, const SPIRVType *ResType,
274 MachineInstr &I) const;
275
276 [[maybe_unused]] bool selectExtInst(Register ResVReg,
277 const SPIRVType *RestType,
279 GL::GLSLExtInst GLInst) const;
280 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
281 MachineInstr &I, CL::OpenCLExtInst CLInst) const;
282 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
283 MachineInstr &I, CL::OpenCLExtInst CLInst,
284 GL::GLSLExtInst GLInst) const;
285 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
286 MachineInstr &I, const ExtInstList &ExtInsts) const;
287
288 bool selectLog10(Register ResVReg, const SPIRVType *ResType,
289 MachineInstr &I) const;
290
291 bool selectSaturate(Register ResVReg, const SPIRVType *ResType,
292 MachineInstr &I) const;
293
294 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType,
295 MachineInstr &I, unsigned Opcode) const;
296
297 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType,
298 MachineInstr &I) const;
299
301
302 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType,
303 MachineInstr &I) const;
304
305 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType,
306 MachineInstr &I) const;
307 bool selectImageWriteIntrinsic(MachineInstr &I) const;
308 bool selectResourceGetPointer(Register &ResVReg, const SPIRVType *ResType,
309 MachineInstr &I) const;
310
311 // Utilities
312 std::pair<Register, bool>
313 buildI32Constant(uint32_t Val, MachineInstr &I,
314 const SPIRVType *ResType = nullptr) const;
315
316 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
317 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const;
318 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
319 MachineInstr &I) const;
320 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const;
321
322 bool wrapIntoSpecConstantOp(MachineInstr &I,
323 SmallVector<Register> &CompositeArgs) const;
324
325 Register getUcharPtrTypeReg(MachineInstr &I,
326 SPIRV::StorageClass::StorageClass SC) const;
327 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
328 Register Src, Register DestType,
329 uint32_t Opcode) const;
330 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
331 SPIRVType *SrcPtrTy) const;
332 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set,
333 uint32_t Binding, uint32_t ArraySize,
334 Register IndexReg, bool IsNonUniform,
335 MachineIRBuilder MIRBuilder) const;
336 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const;
337 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType,
338 Register &ReadReg, MachineInstr &InsertionPoint) const;
339 bool generateImageRead(Register &ResVReg, const SPIRVType *ResType,
340 Register ImageReg, Register IdxReg, DebugLoc Loc,
341 MachineInstr &Pos) const;
342 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
343 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
344 Register ResVReg, const SPIRVType *ResType,
345 MachineInstr &I) const;
346 bool loadHandleBeforePosition(Register &HandleReg, const SPIRVType *ResType,
347 GIntrinsic &HandleDef, MachineInstr &Pos) const;
348};
349
350} // end anonymous namespace
351
352#define GET_GLOBALISEL_IMPL
353#include "SPIRVGenGlobalISel.inc"
354#undef GET_GLOBALISEL_IMPL
355
356SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
357 const SPIRVSubtarget &ST,
358 const RegisterBankInfo &RBI)
359 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
360 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
362#include "SPIRVGenGlobalISel.inc"
365#include "SPIRVGenGlobalISel.inc"
367{
368}
369
370void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
371 CodeGenCoverage *CoverageInfo,
373 BlockFrequencyInfo *BFI) {
374 MRI = &MF.getRegInfo();
375 GR.setCurrentFunc(MF);
376 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
377}
378
379// Ensure that register classes correspond to pattern matching rules.
380void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
381 if (HasVRegsReset == &MF)
382 return;
383 HasVRegsReset = &MF;
384
386 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
388 LLT RegType = MRI.getType(Reg);
389 if (RegType.isScalar())
390 MRI.setType(Reg, LLT::scalar(64));
391 else if (RegType.isPointer())
392 MRI.setType(Reg, LLT::pointer(0, 64));
393 else if (RegType.isVector())
394 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64)));
395 }
396 for (const auto &MBB : MF) {
397 for (const auto &MI : MBB) {
398 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
399 continue;
400 Register DstReg = MI.getOperand(0).getReg();
401 LLT DstType = MRI.getType(DstReg);
402 Register SrcReg = MI.getOperand(1).getReg();
403 LLT SrcType = MRI.getType(SrcReg);
404 if (DstType != SrcType)
405 MRI.setType(DstReg, MRI.getType(SrcReg));
406
407 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg);
408 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg);
409 if (DstRC != SrcRC && SrcRC)
410 MRI.setRegClass(DstReg, SrcRC);
411 }
412 }
413}
414
415static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI);
416
417// Defined in SPIRVLegalizerInfo.cpp.
418extern bool isTypeFoldingSupported(unsigned Opcode);
419
421 for (const auto &MO : MI.all_defs()) {
422 Register Reg = MO.getReg();
423 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg))
424 return false;
425 }
426 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
427 MI.isLifetimeMarker())
428 return false;
429 if (MI.isPHI())
430 return true;
431 if (MI.mayStore() || MI.isCall() ||
432 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
433 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo())
434 return false;
435 return true;
436}
437
438bool SPIRVInstructionSelector::select(MachineInstr &I) {
439 resetVRegsType(*I.getParent()->getParent());
440
441 assert(I.getParent() && "Instruction should be in a basic block!");
442 assert(I.getParent()->getParent() && "Instruction should be in a function!");
443
444 Register Opcode = I.getOpcode();
445 // If it's not a GMIR instruction, we've selected it already.
446 if (!isPreISelGenericOpcode(Opcode)) {
447 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
448 Register DstReg = I.getOperand(0).getReg();
449 Register SrcReg = I.getOperand(1).getReg();
450 auto *Def = MRI->getVRegDef(SrcReg);
451 if (isTypeFoldingSupported(Def->getOpcode())) {
452 bool Res = selectImpl(I, *CoverageInfo);
453 LLVM_DEBUG({
454 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
455 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
456 I.print(dbgs());
457 }
458 });
459 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
460 if (Res) {
461 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI))
462 DeadMIs.insert(Def);
463 return Res;
464 }
465 }
466 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg));
467 MRI->replaceRegWith(SrcReg, DstReg);
468 GR.invalidateMachineInstr(&I);
469 I.removeFromParent();
470 return true;
471 } else if (I.getNumDefs() == 1) {
472 // Make all vregs 64 bits (for SPIR-V IDs).
473 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64));
474 }
476 }
477
478 if (DeadMIs.contains(&I)) {
479 // if the instruction has been already made dead by folding it away
480 // erase it
481 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
483 GR.invalidateMachineInstr(&I);
484 I.eraseFromParent();
485 return true;
486 }
487
488 if (I.getNumOperands() != I.getNumExplicitOperands()) {
489 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
490 return false;
491 }
492
493 // Common code for getting return reg+type, and removing selected instr
494 // from parent occurs here. Instr-specific selection happens in spvSelect().
495 bool HasDefs = I.getNumDefs() > 0;
496 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
497 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
498 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
499 if (spvSelect(ResVReg, ResType, I)) {
500 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
501 for (unsigned i = 0; i < I.getNumDefs(); ++i)
502 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64));
503 GR.invalidateMachineInstr(&I);
504 I.removeFromParent();
505 return true;
506 }
507 return false;
508}
509
510static bool mayApplyGenericSelection(unsigned Opcode) {
511 switch (Opcode) {
512 case TargetOpcode::G_CONSTANT:
513 return false;
514 case TargetOpcode::G_SADDO:
515 case TargetOpcode::G_SSUBO:
516 return true;
517 }
518 return isTypeFoldingSupported(Opcode);
519}
520
521bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
522 MachineInstr &I) const {
523 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg);
524 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg);
525 if (DstRC != SrcRC && SrcRC)
526 MRI->setRegClass(DestReg, SrcRC);
527 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
528 TII.get(TargetOpcode::COPY))
529 .addDef(DestReg)
530 .addUse(SrcReg)
531 .constrainAllUses(TII, TRI, RBI);
532}
533
534bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
535 const SPIRVType *ResType,
536 MachineInstr &I) const {
537 const unsigned Opcode = I.getOpcode();
538 if (mayApplyGenericSelection(Opcode))
539 return selectImpl(I, *CoverageInfo);
540 switch (Opcode) {
541 case TargetOpcode::G_CONSTANT:
542 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(),
543 I);
544 case TargetOpcode::G_GLOBAL_VALUE:
545 return selectGlobalValue(ResVReg, I);
546 case TargetOpcode::G_IMPLICIT_DEF:
547 return selectOpUndef(ResVReg, ResType, I);
548 case TargetOpcode::G_FREEZE:
549 return selectFreeze(ResVReg, ResType, I);
550
551 case TargetOpcode::G_INTRINSIC:
552 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
553 case TargetOpcode::G_INTRINSIC_CONVERGENT:
554 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
555 return selectIntrinsic(ResVReg, ResType, I);
556 case TargetOpcode::G_BITREVERSE:
557 return selectBitreverse(ResVReg, ResType, I);
558
559 case TargetOpcode::G_BUILD_VECTOR:
560 return selectBuildVector(ResVReg, ResType, I);
561 case TargetOpcode::G_SPLAT_VECTOR:
562 return selectSplatVector(ResVReg, ResType, I);
563
564 case TargetOpcode::G_SHUFFLE_VECTOR: {
565 MachineBasicBlock &BB = *I.getParent();
566 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
567 .addDef(ResVReg)
568 .addUse(GR.getSPIRVTypeID(ResType))
569 .addUse(I.getOperand(1).getReg())
570 .addUse(I.getOperand(2).getReg());
571 for (auto V : I.getOperand(3).getShuffleMask())
572 MIB.addImm(V);
573 return MIB.constrainAllUses(TII, TRI, RBI);
574 }
575 case TargetOpcode::G_MEMMOVE:
576 case TargetOpcode::G_MEMCPY:
577 case TargetOpcode::G_MEMSET:
578 return selectMemOperation(ResVReg, I);
579
580 case TargetOpcode::G_ICMP:
581 return selectICmp(ResVReg, ResType, I);
582 case TargetOpcode::G_FCMP:
583 return selectFCmp(ResVReg, ResType, I);
584
585 case TargetOpcode::G_FRAME_INDEX:
586 return selectFrameIndex(ResVReg, ResType, I);
587
588 case TargetOpcode::G_LOAD:
589 return selectLoad(ResVReg, ResType, I);
590 case TargetOpcode::G_STORE:
591 return selectStore(I);
592
593 case TargetOpcode::G_BR:
594 return selectBranch(I);
595 case TargetOpcode::G_BRCOND:
596 return selectBranchCond(I);
597
598 case TargetOpcode::G_PHI:
599 return selectPhi(ResVReg, ResType, I);
600
601 case TargetOpcode::G_FPTOSI:
602 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
603 case TargetOpcode::G_FPTOUI:
604 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
605
606 case TargetOpcode::G_SITOFP:
607 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
608 case TargetOpcode::G_UITOFP:
609 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
610
611 case TargetOpcode::G_CTPOP:
612 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount);
613 case TargetOpcode::G_SMIN:
614 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
615 case TargetOpcode::G_UMIN:
616 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
617
618 case TargetOpcode::G_SMAX:
619 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
620 case TargetOpcode::G_UMAX:
621 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
622
623 case TargetOpcode::G_SCMP:
624 return selectSUCmp(ResVReg, ResType, I, true);
625 case TargetOpcode::G_UCMP:
626 return selectSUCmp(ResVReg, ResType, I, false);
627
628 case TargetOpcode::G_STRICT_FMA:
629 case TargetOpcode::G_FMA:
630 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
631
632 case TargetOpcode::G_STRICT_FLDEXP:
633 return selectExtInst(ResVReg, ResType, I, CL::ldexp);
634
635 case TargetOpcode::G_FPOW:
636 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
637 case TargetOpcode::G_FPOWI:
638 return selectExtInst(ResVReg, ResType, I, CL::pown);
639
640 case TargetOpcode::G_FEXP:
641 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
642 case TargetOpcode::G_FEXP2:
643 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
644
645 case TargetOpcode::G_FLOG:
646 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
647 case TargetOpcode::G_FLOG2:
648 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
649 case TargetOpcode::G_FLOG10:
650 return selectLog10(ResVReg, ResType, I);
651
652 case TargetOpcode::G_FABS:
653 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
654 case TargetOpcode::G_ABS:
655 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
656
657 case TargetOpcode::G_FMINNUM:
658 case TargetOpcode::G_FMINIMUM:
659 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin);
660 case TargetOpcode::G_FMAXNUM:
661 case TargetOpcode::G_FMAXIMUM:
662 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax);
663
664 case TargetOpcode::G_FCOPYSIGN:
665 return selectExtInst(ResVReg, ResType, I, CL::copysign);
666
667 case TargetOpcode::G_FCEIL:
668 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
669 case TargetOpcode::G_FFLOOR:
670 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
671
672 case TargetOpcode::G_FCOS:
673 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
674 case TargetOpcode::G_FSIN:
675 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
676 case TargetOpcode::G_FTAN:
677 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan);
678 case TargetOpcode::G_FACOS:
679 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos);
680 case TargetOpcode::G_FASIN:
681 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin);
682 case TargetOpcode::G_FATAN:
683 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan);
684 case TargetOpcode::G_FATAN2:
685 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2);
686 case TargetOpcode::G_FCOSH:
687 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh);
688 case TargetOpcode::G_FSINH:
689 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh);
690 case TargetOpcode::G_FTANH:
691 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh);
692
693 case TargetOpcode::G_STRICT_FSQRT:
694 case TargetOpcode::G_FSQRT:
695 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
696
697 case TargetOpcode::G_CTTZ:
698 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
699 return selectExtInst(ResVReg, ResType, I, CL::ctz);
700 case TargetOpcode::G_CTLZ:
701 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
702 return selectExtInst(ResVReg, ResType, I, CL::clz);
703
704 case TargetOpcode::G_INTRINSIC_ROUND:
705 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
706 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
707 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
708 case TargetOpcode::G_INTRINSIC_TRUNC:
709 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
710 case TargetOpcode::G_FRINT:
711 case TargetOpcode::G_FNEARBYINT:
712 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
713
714 case TargetOpcode::G_SMULH:
715 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
716 case TargetOpcode::G_UMULH:
717 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
718
719 case TargetOpcode::G_SADDSAT:
720 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat);
721 case TargetOpcode::G_UADDSAT:
722 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat);
723 case TargetOpcode::G_SSUBSAT:
724 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat);
725 case TargetOpcode::G_USUBSAT:
726 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
727
728 case TargetOpcode::G_UADDO:
729 return selectOverflowArith(ResVReg, ResType, I,
730 ResType->getOpcode() == SPIRV::OpTypeVector
731 ? SPIRV::OpIAddCarryV
732 : SPIRV::OpIAddCarryS);
733 case TargetOpcode::G_USUBO:
734 return selectOverflowArith(ResVReg, ResType, I,
735 ResType->getOpcode() == SPIRV::OpTypeVector
736 ? SPIRV::OpISubBorrowV
737 : SPIRV::OpISubBorrowS);
738 case TargetOpcode::G_UMULO:
739 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended);
740 case TargetOpcode::G_SMULO:
741 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended);
742
743 case TargetOpcode::G_SEXT:
744 return selectExt(ResVReg, ResType, I, true);
745 case TargetOpcode::G_ANYEXT:
746 case TargetOpcode::G_ZEXT:
747 return selectExt(ResVReg, ResType, I, false);
748 case TargetOpcode::G_TRUNC:
749 return selectTrunc(ResVReg, ResType, I);
750 case TargetOpcode::G_FPTRUNC:
751 case TargetOpcode::G_FPEXT:
752 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
753
754 case TargetOpcode::G_PTRTOINT:
755 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
756 case TargetOpcode::G_INTTOPTR:
757 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
758 case TargetOpcode::G_BITCAST:
759 return selectBitcast(ResVReg, ResType, I);
760 case TargetOpcode::G_ADDRSPACE_CAST:
761 return selectAddrSpaceCast(ResVReg, ResType, I);
762 case TargetOpcode::G_PTR_ADD: {
763 // Currently, we get G_PTR_ADD only applied to global variables.
764 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
765 Register GV = I.getOperand(1).getReg();
766 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
767 (void)II;
768 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
769 (*II).getOpcode() == TargetOpcode::COPY ||
770 (*II).getOpcode() == SPIRV::OpVariable) &&
771 isImm(I.getOperand(2), MRI));
772 // It may be the initialization of a global variable.
773 bool IsGVInit = false;
775 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()),
776 UseEnd = MRI->use_instr_end();
777 UseIt != UseEnd; UseIt = std::next(UseIt)) {
778 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
779 (*UseIt).getOpcode() == SPIRV::OpVariable) {
780 IsGVInit = true;
781 break;
782 }
783 }
784 MachineBasicBlock &BB = *I.getParent();
785 if (!IsGVInit) {
786 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV);
787 SPIRVType *GVPointeeType = GR.getPointeeType(GVType);
788 SPIRVType *ResPointeeType = GR.getPointeeType(ResType);
789 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
790 // Build a new virtual register that is associated with the required
791 // data type.
792 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV));
793 MRI->setRegClass(NewVReg, MRI->getRegClass(GV));
794 // Having a correctly typed base we are ready to build the actually
795 // required GEP. It may not be a constant though, because all Operands
796 // of OpSpecConstantOp is to originate from other const instructions,
797 // and only the AccessChain named opcodes accept a global OpVariable
798 // instruction. We can't use an AccessChain opcode because of the type
799 // mismatch between result and base types.
800 if (!GR.isBitcastCompatible(ResType, GVType))
802 "incompatible result and operand types in a bitcast");
803 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
805 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
806 .addDef(NewVReg)
807 .addUse(ResTypeReg)
808 .addUse(GV);
809 return MIB.constrainAllUses(TII, TRI, RBI) &&
810 BuildMI(BB, I, I.getDebugLoc(),
811 TII.get(STI.isVulkanEnv()
812 ? SPIRV::OpInBoundsAccessChain
813 : SPIRV::OpInBoundsPtrAccessChain))
814 .addDef(ResVReg)
815 .addUse(ResTypeReg)
816 .addUse(NewVReg)
817 .addUse(I.getOperand(2).getReg())
818 .constrainAllUses(TII, TRI, RBI);
819 } else {
820 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
821 .addDef(ResVReg)
822 .addUse(GR.getSPIRVTypeID(ResType))
823 .addImm(
824 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
825 .addUse(GV)
826 .addUse(I.getOperand(2).getReg())
827 .constrainAllUses(TII, TRI, RBI);
828 }
829 }
830 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
831 // initialize a global variable with a constant expression (e.g., the test
832 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
833 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
834 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
835 .addDef(ResVReg)
836 .addUse(GR.getSPIRVTypeID(ResType))
837 .addImm(static_cast<uint32_t>(
838 SPIRV::Opcode::InBoundsPtrAccessChain))
839 .addUse(GV)
840 .addUse(Idx)
841 .addUse(I.getOperand(2).getReg());
842 return MIB.constrainAllUses(TII, TRI, RBI);
843 }
844
845 case TargetOpcode::G_ATOMICRMW_OR:
846 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
847 case TargetOpcode::G_ATOMICRMW_ADD:
848 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
849 case TargetOpcode::G_ATOMICRMW_AND:
850 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
851 case TargetOpcode::G_ATOMICRMW_MAX:
852 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
853 case TargetOpcode::G_ATOMICRMW_MIN:
854 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
855 case TargetOpcode::G_ATOMICRMW_SUB:
856 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
857 case TargetOpcode::G_ATOMICRMW_XOR:
858 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
859 case TargetOpcode::G_ATOMICRMW_UMAX:
860 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
861 case TargetOpcode::G_ATOMICRMW_UMIN:
862 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
863 case TargetOpcode::G_ATOMICRMW_XCHG:
864 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
865 case TargetOpcode::G_ATOMIC_CMPXCHG:
866 return selectAtomicCmpXchg(ResVReg, ResType, I);
867
868 case TargetOpcode::G_ATOMICRMW_FADD:
869 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT);
870 case TargetOpcode::G_ATOMICRMW_FSUB:
871 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
872 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT,
873 SPIRV::OpFNegate);
874 case TargetOpcode::G_ATOMICRMW_FMIN:
875 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT);
876 case TargetOpcode::G_ATOMICRMW_FMAX:
877 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT);
878
879 case TargetOpcode::G_FENCE:
880 return selectFence(I);
881
882 case TargetOpcode::G_STACKSAVE:
883 return selectStackSave(ResVReg, ResType, I);
884 case TargetOpcode::G_STACKRESTORE:
885 return selectStackRestore(I);
886
887 case TargetOpcode::G_UNMERGE_VALUES:
888 return selectUnmergeValues(I);
889
890 // Discard gen opcodes for intrinsics which we do not expect to actually
891 // represent code after lowering or intrinsics which are not implemented but
892 // should not crash when found in a customer's LLVM IR input.
893 case TargetOpcode::G_TRAP:
894 case TargetOpcode::G_DEBUGTRAP:
895 case TargetOpcode::G_UBSANTRAP:
896 case TargetOpcode::DBG_LABEL:
897 return true;
898
899 default:
900 return false;
901 }
902}
903
904bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
905 const SPIRVType *ResType,
907 GL::GLSLExtInst GLInst) const {
908 return selectExtInst(ResVReg, ResType, I,
909 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}});
910}
911
912bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
913 const SPIRVType *ResType,
915 CL::OpenCLExtInst CLInst) const {
916 return selectExtInst(ResVReg, ResType, I,
917 {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
918}
919
920bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
921 const SPIRVType *ResType,
923 CL::OpenCLExtInst CLInst,
924 GL::GLSLExtInst GLInst) const {
925 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
926 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
927 return selectExtInst(ResVReg, ResType, I, ExtInsts);
928}
929
930bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
931 const SPIRVType *ResType,
933 const ExtInstList &Insts) const {
934
935 for (const auto &Ex : Insts) {
936 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
937 uint32_t Opcode = Ex.second;
938 if (STI.canUseExtInstSet(Set)) {
939 MachineBasicBlock &BB = *I.getParent();
940 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
941 .addDef(ResVReg)
942 .addUse(GR.getSPIRVTypeID(ResType))
943 .addImm(static_cast<uint32_t>(Set))
944 .addImm(Opcode);
945 const unsigned NumOps = I.getNumOperands();
946 unsigned Index = 1;
947 if (Index < NumOps &&
948 I.getOperand(Index).getType() ==
949 MachineOperand::MachineOperandType::MO_IntrinsicID)
950 Index = 2;
951 for (; Index < NumOps; ++Index)
952 MIB.add(I.getOperand(Index));
953 return MIB.constrainAllUses(TII, TRI, RBI);
954 }
955 }
956 return false;
957}
958
959bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
960 const SPIRVType *ResType,
962 std::vector<Register> Srcs,
963 unsigned Opcode) const {
964 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
965 .addDef(ResVReg)
966 .addUse(GR.getSPIRVTypeID(ResType));
967 for (Register SReg : Srcs) {
968 MIB.addUse(SReg);
969 }
970 return MIB.constrainAllUses(TII, TRI, RBI);
971}
972
973bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
974 const SPIRVType *ResType,
976 unsigned Opcode) const {
977 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) {
978 Register SrcReg = I.getOperand(1).getReg();
979 bool IsGV = false;
981 MRI->def_instr_begin(SrcReg);
982 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) {
983 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
984 IsGV = true;
985 break;
986 }
987 }
988 if (IsGV) {
989 uint32_t SpecOpcode = 0;
990 switch (Opcode) {
991 case SPIRV::OpConvertPtrToU:
992 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
993 break;
994 case SPIRV::OpConvertUToPtr:
995 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
996 break;
997 }
998 if (SpecOpcode)
999 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1000 TII.get(SPIRV::OpSpecConstantOp))
1001 .addDef(ResVReg)
1002 .addUse(GR.getSPIRVTypeID(ResType))
1003 .addImm(SpecOpcode)
1004 .addUse(SrcReg)
1005 .constrainAllUses(TII, TRI, RBI);
1006 }
1007 }
1008 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()},
1009 Opcode);
1010}
1011
1012bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
1013 const SPIRVType *ResType,
1014 MachineInstr &I) const {
1015 Register OpReg = I.getOperand(1).getReg();
1016 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
1017 if (!GR.isBitcastCompatible(ResType, OpType))
1018 report_fatal_error("incompatible result and operand types in a bitcast");
1019 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
1020}
1021
1023 MachineInstrBuilder &MIB) {
1024 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1025 if (MemOp->isVolatile())
1026 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1027 if (MemOp->isNonTemporal())
1028 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1029 if (MemOp->getAlign().value())
1030 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
1031
1032 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
1033 MIB.addImm(SpvMemOp);
1034 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
1035 MIB.addImm(MemOp->getAlign().value());
1036 }
1037}
1038
1040 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
1041 if (Flags & MachineMemOperand::Flags::MOVolatile)
1042 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
1043 if (Flags & MachineMemOperand::Flags::MONonTemporal)
1044 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
1045
1046 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
1047 MIB.addImm(SpvMemOp);
1048}
1049
1050bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
1051 const SPIRVType *ResType,
1052 MachineInstr &I) const {
1053 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
1054 Register Ptr = I.getOperand(1 + OpOffset).getReg();
1055
1056 auto *PtrDef = getVRegDef(*MRI, Ptr);
1057 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
1058 if (IntPtrDef &&
1059 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1060 Register ImageReg = IntPtrDef->getOperand(2).getReg();
1061 Register NewImageReg =
1062 MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
1063 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
1064 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
1065 *ImageDef, I)) {
1066 return false;
1067 }
1068
1069 Register IdxReg = IntPtrDef->getOperand(3).getReg();
1070 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg,
1071 I.getDebugLoc(), I);
1072 }
1073
1074 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1075 .addDef(ResVReg)
1076 .addUse(GR.getSPIRVTypeID(ResType))
1077 .addUse(Ptr);
1078 if (!I.getNumMemOperands()) {
1079 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1080 I.getOpcode() ==
1081 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1082 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
1083 } else {
1084 addMemoryOperands(*I.memoperands_begin(), MIB);
1085 }
1086 return MIB.constrainAllUses(TII, TRI, RBI);
1087}
1088
1089bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
1090 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
1091 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
1092 Register Ptr = I.getOperand(1 + OpOffset).getReg();
1093
1094 auto *PtrDef = getVRegDef(*MRI, Ptr);
1095 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
1096 if (IntPtrDef &&
1097 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1098 Register ImageReg = IntPtrDef->getOperand(2).getReg();
1099 Register NewImageReg =
1100 MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
1101 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
1102 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
1103 *ImageDef, I)) {
1104 return false;
1105 }
1106
1107 Register IdxReg = IntPtrDef->getOperand(3).getReg();
1108 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1109 TII.get(SPIRV::OpImageWrite))
1110 .addUse(NewImageReg)
1111 .addUse(IdxReg)
1112 .addUse(StoreVal)
1113 .constrainAllUses(TII, TRI, RBI);
1114 }
1115
1116 MachineBasicBlock &BB = *I.getParent();
1117 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
1118 .addUse(Ptr)
1119 .addUse(StoreVal);
1120 if (!I.getNumMemOperands()) {
1121 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
1122 I.getOpcode() ==
1123 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
1124 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
1125 } else {
1126 addMemoryOperands(*I.memoperands_begin(), MIB);
1127 }
1128 return MIB.constrainAllUses(TII, TRI, RBI);
1129}
1130
1131bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
1132 const SPIRVType *ResType,
1133 MachineInstr &I) const {
1134 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
1136 "llvm.stacksave intrinsic: this instruction requires the following "
1137 "SPIR-V extension: SPV_INTEL_variable_length_array",
1138 false);
1139 MachineBasicBlock &BB = *I.getParent();
1140 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL))
1141 .addDef(ResVReg)
1142 .addUse(GR.getSPIRVTypeID(ResType))
1143 .constrainAllUses(TII, TRI, RBI);
1144}
1145
1146bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
1147 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
1149 "llvm.stackrestore intrinsic: this instruction requires the following "
1150 "SPIR-V extension: SPV_INTEL_variable_length_array",
1151 false);
1152 if (!I.getOperand(0).isReg())
1153 return false;
1154 MachineBasicBlock &BB = *I.getParent();
1155 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL))
1156 .addUse(I.getOperand(0).getReg())
1157 .constrainAllUses(TII, TRI, RBI);
1158}
1159
1160bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
1161 MachineInstr &I) const {
1162 MachineBasicBlock &BB = *I.getParent();
1163 Register SrcReg = I.getOperand(1).getReg();
1164 bool Result = true;
1165 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
1166 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1167 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
1168 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
1169 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
1170 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII);
1171 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII);
1172 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
1173 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
1174 // TODO: check if we have such GV, add init, use buildGlobalVariable.
1175 Function &CurFunction = GR.CurMF->getFunction();
1176 Type *LLVMArrTy =
1177 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num);
1178 // Module takes ownership of the global var.
1179 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
1181 Constant::getNullValue(LLVMArrTy));
1182 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
1183 GR.add(GV, GR.CurMF, VarReg);
1184 GR.addGlobalObject(GV, GR.CurMF, VarReg);
1185
1186 Result &=
1187 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
1188 .addDef(VarReg)
1189 .addUse(GR.getSPIRVTypeID(VarTy))
1190 .addImm(SPIRV::StorageClass::UniformConstant)
1191 .addUse(Const)
1192 .constrainAllUses(TII, TRI, RBI);
1193 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
1194 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
1195 ValTy, I, TII, SPIRV::StorageClass::UniformConstant);
1196 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
1197 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast);
1198 }
1199 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
1200 .addUse(I.getOperand(0).getReg())
1201 .addUse(SrcReg)
1202 .addUse(I.getOperand(2).getReg());
1203 if (I.getNumMemOperands())
1204 addMemoryOperands(*I.memoperands_begin(), MIB);
1205 Result &= MIB.constrainAllUses(TII, TRI, RBI);
1206 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg())
1207 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I);
1208 return Result;
1209}
1210
1211bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
1212 const SPIRVType *ResType,
1213 MachineInstr &I,
1214 unsigned NewOpcode,
1215 unsigned NegateOpcode) const {
1216 bool Result = true;
1217 assert(I.hasOneMemOperand());
1218 const MachineMemOperand *MemOp = *I.memoperands_begin();
1219 uint32_t Scope = static_cast<uint32_t>(getMemScope(
1220 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID()));
1221 auto ScopeConstant = buildI32Constant(Scope, I);
1222 Register ScopeReg = ScopeConstant.first;
1223 Result &= ScopeConstant.second;
1224
1225 Register Ptr = I.getOperand(1).getReg();
1226 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
1227 // auto ScSem =
1228 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
1229 AtomicOrdering AO = MemOp->getSuccessOrdering();
1230 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
1231 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I);
1232 Register MemSemReg = MemSemConstant.first;
1233 Result &= MemSemConstant.second;
1234
1235 Register ValueReg = I.getOperand(2).getReg();
1236 if (NegateOpcode != 0) {
1237 // Translation with negative value operand is requested
1238 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
1239 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode);
1240 ValueReg = TmpReg;
1241 }
1242
1243 return Result &&
1244 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
1245 .addDef(ResVReg)
1246 .addUse(GR.getSPIRVTypeID(ResType))
1247 .addUse(Ptr)
1248 .addUse(ScopeReg)
1249 .addUse(MemSemReg)
1250 .addUse(ValueReg)
1251 .constrainAllUses(TII, TRI, RBI);
1252}
1253
1254bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
1255 unsigned ArgI = I.getNumOperands() - 1;
1256 Register SrcReg =
1257 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0);
1258 SPIRVType *DefType =
1259 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr;
1260 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector)
1262 "cannot select G_UNMERGE_VALUES with a non-vector argument");
1263
1264 SPIRVType *ScalarType =
1265 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg());
1266 MachineBasicBlock &BB = *I.getParent();
1267 bool Res = false;
1268 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
1269 Register ResVReg = I.getOperand(i).getReg();
1270 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg);
1271 if (!ResType) {
1272 // There was no "assign type" actions, let's fix this now
1273 ResType = ScalarType;
1274 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
1275 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType)));
1276 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
1277 }
1278 auto MIB =
1279 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1280 .addDef(ResVReg)
1281 .addUse(GR.getSPIRVTypeID(ResType))
1282 .addUse(SrcReg)
1283 .addImm(static_cast<int64_t>(i));
1284 Res |= MIB.constrainAllUses(TII, TRI, RBI);
1285 }
1286 return Res;
1287}
1288
1289bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
1290 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
1291 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
1292 auto MemSemConstant = buildI32Constant(MemSem, I);
1293 Register MemSemReg = MemSemConstant.first;
1294 bool Result = MemSemConstant.second;
1295 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
1296 uint32_t Scope = static_cast<uint32_t>(
1297 getMemScope(GR.CurMF->getFunction().getContext(), Ord));
1298 auto ScopeConstant = buildI32Constant(Scope, I);
1299 Register ScopeReg = ScopeConstant.first;
1300 Result &= ScopeConstant.second;
1301 MachineBasicBlock &BB = *I.getParent();
1302 return Result &&
1303 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
1304 .addUse(ScopeReg)
1305 .addUse(MemSemReg)
1306 .constrainAllUses(TII, TRI, RBI);
1307}
1308
1309bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
1310 const SPIRVType *ResType,
1311 MachineInstr &I,
1312 unsigned Opcode) const {
1313 Type *ResTy = nullptr;
1314 StringRef ResName;
1315 if (!GR.findValueAttrs(&I, ResTy, ResName))
1317 "Not enough info to select the arithmetic with overflow instruction");
1318 if (!ResTy || !ResTy->isStructTy())
1319 report_fatal_error("Expect struct type result for the arithmetic "
1320 "with overflow instruction");
1321 // "Result Type must be from OpTypeStruct. The struct must have two members,
1322 // and the two members must be the same type."
1323 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
1324 ResTy = StructType::get(ResElemTy, ResElemTy);
1325 // Build SPIR-V types and constant(s) if needed.
1326 MachineIRBuilder MIRBuilder(I);
1327 SPIRVType *StructType = GR.getOrCreateSPIRVType(
1328 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
1329 assert(I.getNumDefs() > 1 && "Not enought operands");
1330 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
1331 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
1332 if (N > 1)
1333 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
1334 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
1335 Register ZeroReg = buildZerosVal(ResType, I);
1336 // A new virtual register to store the result struct.
1337 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
1338 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
1339 // Build the result name if needed.
1340 if (ResName.size() > 0)
1341 buildOpName(StructVReg, ResName, MIRBuilder);
1342 // Build the arithmetic with overflow instruction.
1343 MachineBasicBlock &BB = *I.getParent();
1344 auto MIB =
1345 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode))
1346 .addDef(StructVReg)
1347 .addUse(GR.getSPIRVTypeID(StructType));
1348 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
1349 MIB.addUse(I.getOperand(i).getReg());
1350 bool Result = MIB.constrainAllUses(TII, TRI, RBI);
1351 // Build instructions to extract fields of the instruction's result.
1352 // A new virtual register to store the higher part of the result struct.
1353 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
1354 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass);
1355 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
1356 auto MIB =
1357 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1358 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg())
1359 .addUse(GR.getSPIRVTypeID(ResType))
1360 .addUse(StructVReg)
1361 .addImm(i);
1362 Result &= MIB.constrainAllUses(TII, TRI, RBI);
1363 }
1364 // Build boolean value from the higher part.
1365 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
1366 .addDef(I.getOperand(1).getReg())
1367 .addUse(BoolTypeReg)
1368 .addUse(HigherVReg)
1369 .addUse(ZeroReg)
1370 .constrainAllUses(TII, TRI, RBI);
1371}
1372
1373bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
1374 const SPIRVType *ResType,
1375 MachineInstr &I) const {
1376 bool Result = true;
1377 Register ScopeReg;
1378 Register MemSemEqReg;
1379 Register MemSemNeqReg;
1380 Register Ptr = I.getOperand(2).getReg();
1381 if (!isa<GIntrinsic>(I)) {
1382 assert(I.hasOneMemOperand());
1383 const MachineMemOperand *MemOp = *I.memoperands_begin();
1384 unsigned Scope = static_cast<uint32_t>(getMemScope(
1385 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID()));
1386 auto ScopeConstant = buildI32Constant(Scope, I);
1387 ScopeReg = ScopeConstant.first;
1388 Result &= ScopeConstant.second;
1389
1390 unsigned ScSem = static_cast<uint32_t>(
1391 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)));
1392 AtomicOrdering AO = MemOp->getSuccessOrdering();
1393 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem;
1394 auto MemSemEqConstant = buildI32Constant(MemSemEq, I);
1395 MemSemEqReg = MemSemEqConstant.first;
1396 Result &= MemSemEqConstant.second;
1397 AtomicOrdering FO = MemOp->getFailureOrdering();
1398 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem;
1399 if (MemSemEq == MemSemNeq)
1400 MemSemNeqReg = MemSemEqReg;
1401 else {
1402 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I);
1403 MemSemNeqReg = MemSemNeqConstant.first;
1404 Result &= MemSemNeqConstant.second;
1405 }
1406 } else {
1407 ScopeReg = I.getOperand(5).getReg();
1408 MemSemEqReg = I.getOperand(6).getReg();
1409 MemSemNeqReg = I.getOperand(7).getReg();
1410 }
1411
1412 Register Cmp = I.getOperand(3).getReg();
1413 Register Val = I.getOperand(4).getReg();
1414 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val);
1415 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
1416 const DebugLoc &DL = I.getDebugLoc();
1417 Result &=
1418 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
1419 .addDef(ACmpRes)
1420 .addUse(GR.getSPIRVTypeID(SpvValTy))
1421 .addUse(Ptr)
1422 .addUse(ScopeReg)
1423 .addUse(MemSemEqReg)
1424 .addUse(MemSemNeqReg)
1425 .addUse(Val)
1426 .addUse(Cmp)
1427 .constrainAllUses(TII, TRI, RBI);
1428 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
1429 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
1430 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
1431 .addDef(CmpSuccReg)
1432 .addUse(GR.getSPIRVTypeID(BoolTy))
1433 .addUse(ACmpRes)
1434 .addUse(Cmp)
1435 .constrainAllUses(TII, TRI, RBI);
1436 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
1437 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
1438 .addDef(TmpReg)
1439 .addUse(GR.getSPIRVTypeID(ResType))
1440 .addUse(ACmpRes)
1441 .addUse(GR.getOrCreateUndef(I, ResType, TII))
1442 .addImm(0)
1443 .constrainAllUses(TII, TRI, RBI);
1444 return Result &&
1445 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
1446 .addDef(ResVReg)
1447 .addUse(GR.getSPIRVTypeID(ResType))
1448 .addUse(CmpSuccReg)
1449 .addUse(TmpReg)
1450 .addImm(1)
1451 .constrainAllUses(TII, TRI, RBI);
1452}
1453
1454static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) {
1455 switch (SC) {
1456 case SPIRV::StorageClass::Workgroup:
1457 case SPIRV::StorageClass::CrossWorkgroup:
1458 case SPIRV::StorageClass::Function:
1459 return true;
1460 default:
1461 return false;
1462 }
1463}
1464
1465static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
1466 switch (SC) {
1467 case SPIRV::StorageClass::DeviceOnlyINTEL:
1468 case SPIRV::StorageClass::HostOnlyINTEL:
1469 return true;
1470 default:
1471 return false;
1472 }
1473}
1474
1475// Returns true ResVReg is referred only from global vars and OpName's.
1477 bool IsGRef = false;
1478 bool IsAllowedRefs =
1479 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(),
1480 [&IsGRef](auto const &It) {
1481 unsigned Opcode = It.getOpcode();
1482 if (Opcode == SPIRV::OpConstantComposite ||
1483 Opcode == SPIRV::OpVariable ||
1484 isSpvIntrinsic(It, Intrinsic::spv_init_global))
1485 return IsGRef = true;
1486 return Opcode == SPIRV::OpName;
1487 });
1488 return IsAllowedRefs && IsGRef;
1489}
1490
1491Register SPIRVInstructionSelector::getUcharPtrTypeReg(
1492 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
1493 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType(
1494 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC));
1495}
1496
1498SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
1499 Register Src, Register DestType,
1500 uint32_t Opcode) const {
1501 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1502 TII.get(SPIRV::OpSpecConstantOp))
1503 .addDef(Dest)
1504 .addUse(DestType)
1505 .addImm(Opcode)
1506 .addUse(Src);
1507}
1508
1510SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
1511 SPIRVType *SrcPtrTy) const {
1512 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType(
1513 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic);
1514 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass);
1516 SPIRV::StorageClass::Generic),
1517 GR.getPointerSize()));
1518 MachineFunction *MF = I.getParent()->getParent();
1519 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF);
1520 MachineInstrBuilder MIB = buildSpecConstantOp(
1521 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy),
1522 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
1523 GR.add(MIB.getInstr(), MF, Tmp);
1524 return MIB;
1525}
1526
1527// In SPIR-V address space casting can only happen to and from the Generic
1528// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
1529// pointers to and from Generic pointers. As such, we can convert e.g. from
1530// Workgroup to Function by going via a Generic pointer as an intermediary. All
1531// other combinations can only be done by a bitcast, and are probably not safe.
1532bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
1533 const SPIRVType *ResType,
1534 MachineInstr &I) const {
1535 MachineBasicBlock &BB = *I.getParent();
1536 const DebugLoc &DL = I.getDebugLoc();
1537
1538 Register SrcPtr = I.getOperand(1).getReg();
1539 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
1540
1541 // don't generate a cast for a null that may be represented by OpTypeInt
1542 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer ||
1543 ResType->getOpcode() != SPIRV::OpTypePointer)
1544 return BuildCOPY(ResVReg, SrcPtr, I);
1545
1546 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy);
1547 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType);
1548
1549 if (isASCastInGVar(MRI, ResVReg)) {
1550 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
1551 // are expressed by OpSpecConstantOp with an Opcode.
1552 // TODO: maybe insert a check whether the Kernel capability was declared and
1553 // so PtrCastToGeneric/GenericCastToPtr are available.
1554 unsigned SpecOpcode =
1555 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)
1556 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)
1557 : (SrcSC == SPIRV::StorageClass::Generic &&
1559 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)
1560 : 0);
1561 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
1562 // correct value of ResType and use general i8* instead. Maybe this should
1563 // be addressed in the emit-intrinsic step to infer a correct
1564 // OpConstantComposite type.
1565 if (SpecOpcode) {
1566 return buildSpecConstantOp(I, ResVReg, SrcPtr,
1567 getUcharPtrTypeReg(I, DstSC), SpecOpcode)
1568 .constrainAllUses(TII, TRI, RBI);
1569 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
1570 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
1571 return MIB.constrainAllUses(TII, TRI, RBI) &&
1572 buildSpecConstantOp(
1573 I, ResVReg, MIB->getOperand(0).getReg(),
1574 getUcharPtrTypeReg(I, DstSC),
1575 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
1576 .constrainAllUses(TII, TRI, RBI);
1577 }
1578 }
1579
1580 // don't generate a cast between identical storage classes
1581 if (SrcSC == DstSC)
1582 return BuildCOPY(ResVReg, SrcPtr, I);
1583
1584 if ((SrcSC == SPIRV::StorageClass::Function &&
1585 DstSC == SPIRV::StorageClass::Private) ||
1586 (DstSC == SPIRV::StorageClass::Function &&
1587 SrcSC == SPIRV::StorageClass::Private))
1588 return BuildCOPY(ResVReg, SrcPtr, I);
1589
1590 // Casting from an eligible pointer to Generic.
1591 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
1592 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
1593 // Casting from Generic to an eligible pointer.
1594 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
1595 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
1596 // Casting between 2 eligible pointers using Generic as an intermediary.
1597 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
1598 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
1599 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType(
1600 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic);
1601 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
1602 .addDef(Tmp)
1603 .addUse(GR.getSPIRVTypeID(GenericPtrTy))
1604 .addUse(SrcPtr)
1605 .constrainAllUses(TII, TRI, RBI);
1606 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
1607 .addDef(ResVReg)
1608 .addUse(GR.getSPIRVTypeID(ResType))
1609 .addUse(Tmp)
1610 .constrainAllUses(TII, TRI, RBI);
1611 }
1612
1613 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
1614 // be applied
1615 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
1616 return selectUnOp(ResVReg, ResType, I,
1617 SPIRV::OpPtrCastToCrossWorkgroupINTEL);
1618 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC))
1619 return selectUnOp(ResVReg, ResType, I,
1620 SPIRV::OpCrossWorkgroupCastToPtrINTEL);
1621 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic)
1622 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
1623 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC))
1624 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
1625
1626 // Bitcast for pointers requires that the address spaces must match
1627 return false;
1628}
1629
1630static unsigned getFCmpOpcode(unsigned PredNum) {
1631 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1632 switch (Pred) {
1633 case CmpInst::FCMP_OEQ:
1634 return SPIRV::OpFOrdEqual;
1635 case CmpInst::FCMP_OGE:
1636 return SPIRV::OpFOrdGreaterThanEqual;
1637 case CmpInst::FCMP_OGT:
1638 return SPIRV::OpFOrdGreaterThan;
1639 case CmpInst::FCMP_OLE:
1640 return SPIRV::OpFOrdLessThanEqual;
1641 case CmpInst::FCMP_OLT:
1642 return SPIRV::OpFOrdLessThan;
1643 case CmpInst::FCMP_ONE:
1644 return SPIRV::OpFOrdNotEqual;
1645 case CmpInst::FCMP_ORD:
1646 return SPIRV::OpOrdered;
1647 case CmpInst::FCMP_UEQ:
1648 return SPIRV::OpFUnordEqual;
1649 case CmpInst::FCMP_UGE:
1650 return SPIRV::OpFUnordGreaterThanEqual;
1651 case CmpInst::FCMP_UGT:
1652 return SPIRV::OpFUnordGreaterThan;
1653 case CmpInst::FCMP_ULE:
1654 return SPIRV::OpFUnordLessThanEqual;
1655 case CmpInst::FCMP_ULT:
1656 return SPIRV::OpFUnordLessThan;
1657 case CmpInst::FCMP_UNE:
1658 return SPIRV::OpFUnordNotEqual;
1659 case CmpInst::FCMP_UNO:
1660 return SPIRV::OpUnordered;
1661 default:
1662 llvm_unreachable("Unknown predicate type for FCmp");
1663 }
1664}
1665
1666static unsigned getICmpOpcode(unsigned PredNum) {
1667 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1668 switch (Pred) {
1669 case CmpInst::ICMP_EQ:
1670 return SPIRV::OpIEqual;
1671 case CmpInst::ICMP_NE:
1672 return SPIRV::OpINotEqual;
1673 case CmpInst::ICMP_SGE:
1674 return SPIRV::OpSGreaterThanEqual;
1675 case CmpInst::ICMP_SGT:
1676 return SPIRV::OpSGreaterThan;
1677 case CmpInst::ICMP_SLE:
1678 return SPIRV::OpSLessThanEqual;
1679 case CmpInst::ICMP_SLT:
1680 return SPIRV::OpSLessThan;
1681 case CmpInst::ICMP_UGE:
1682 return SPIRV::OpUGreaterThanEqual;
1683 case CmpInst::ICMP_UGT:
1684 return SPIRV::OpUGreaterThan;
1685 case CmpInst::ICMP_ULE:
1686 return SPIRV::OpULessThanEqual;
1687 case CmpInst::ICMP_ULT:
1688 return SPIRV::OpULessThan;
1689 default:
1690 llvm_unreachable("Unknown predicate type for ICmp");
1691 }
1692}
1693
1694static unsigned getPtrCmpOpcode(unsigned Pred) {
1695 switch (static_cast<CmpInst::Predicate>(Pred)) {
1696 case CmpInst::ICMP_EQ:
1697 return SPIRV::OpPtrEqual;
1698 case CmpInst::ICMP_NE:
1699 return SPIRV::OpPtrNotEqual;
1700 default:
1701 llvm_unreachable("Unknown predicate type for pointer comparison");
1702 }
1703}
1704
1705// Return the logical operation, or abort if none exists.
1706static unsigned getBoolCmpOpcode(unsigned PredNum) {
1707 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1708 switch (Pred) {
1709 case CmpInst::ICMP_EQ:
1710 return SPIRV::OpLogicalEqual;
1711 case CmpInst::ICMP_NE:
1712 return SPIRV::OpLogicalNotEqual;
1713 default:
1714 llvm_unreachable("Unknown predicate type for Bool comparison");
1715 }
1716}
1717
1718static APFloat getZeroFP(const Type *LLVMFloatTy) {
1719 if (!LLVMFloatTy)
1720 return APFloat::getZero(APFloat::IEEEsingle());
1721 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
1722 case Type::HalfTyID:
1723 return APFloat::getZero(APFloat::IEEEhalf());
1724 default:
1725 case Type::FloatTyID:
1726 return APFloat::getZero(APFloat::IEEEsingle());
1727 case Type::DoubleTyID:
1728 return APFloat::getZero(APFloat::IEEEdouble());
1729 }
1730}
1731
1732static APFloat getOneFP(const Type *LLVMFloatTy) {
1733 if (!LLVMFloatTy)
1734 return APFloat::getOne(APFloat::IEEEsingle());
1735 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
1736 case Type::HalfTyID:
1737 return APFloat::getOne(APFloat::IEEEhalf());
1738 default:
1739 case Type::FloatTyID:
1740 return APFloat::getOne(APFloat::IEEEsingle());
1741 case Type::DoubleTyID:
1742 return APFloat::getOne(APFloat::IEEEdouble());
1743 }
1744}
1745
1746bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
1747 const SPIRVType *ResType,
1748 MachineInstr &I,
1749 unsigned OpAnyOrAll) const {
1750 assert(I.getNumOperands() == 3);
1751 assert(I.getOperand(2).isReg());
1752 MachineBasicBlock &BB = *I.getParent();
1753 Register InputRegister = I.getOperand(2).getReg();
1754 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister);
1755
1756 if (!InputType)
1757 report_fatal_error("Input Type could not be determined.");
1758
1759 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool);
1760 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
1761 if (IsBoolTy && !IsVectorTy) {
1762 assert(ResVReg == I.getOperand(0).getReg());
1763 return BuildCOPY(ResVReg, InputRegister, I);
1764 }
1765
1766 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
1767 unsigned SpirvNotEqualId =
1768 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
1769 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
1770 SPIRVType *SpvBoolTy = SpvBoolScalarTy;
1771 Register NotEqualReg = ResVReg;
1772
1773 if (IsVectorTy) {
1774 NotEqualReg = IsBoolTy ? InputRegister
1775 : MRI->createVirtualRegister(&SPIRV::iIDRegClass);
1776 const unsigned NumElts = InputType->getOperand(2).getImm();
1777 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII);
1778 }
1779
1780 bool Result = true;
1781 if (!IsBoolTy) {
1782 Register ConstZeroReg =
1783 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I);
1784
1785 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId))
1786 .addDef(NotEqualReg)
1787 .addUse(GR.getSPIRVTypeID(SpvBoolTy))
1788 .addUse(InputRegister)
1789 .addUse(ConstZeroReg)
1790 .constrainAllUses(TII, TRI, RBI);
1791 }
1792
1793 if (!IsVectorTy)
1794 return Result;
1795
1796 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
1797 .addDef(ResVReg)
1798 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy))
1799 .addUse(NotEqualReg)
1800 .constrainAllUses(TII, TRI, RBI);
1801}
1802
1803bool SPIRVInstructionSelector::selectAll(Register ResVReg,
1804 const SPIRVType *ResType,
1805 MachineInstr &I) const {
1806 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
1807}
1808
1809bool SPIRVInstructionSelector::selectAny(Register ResVReg,
1810 const SPIRVType *ResType,
1811 MachineInstr &I) const {
1812 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
1813}
1814
1815// Select the OpDot instruction for the given float dot
1816bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
1817 const SPIRVType *ResType,
1818 MachineInstr &I) const {
1819 assert(I.getNumOperands() == 4);
1820 assert(I.getOperand(2).isReg());
1821 assert(I.getOperand(3).isReg());
1822
1823 [[maybe_unused]] SPIRVType *VecType =
1824 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg());
1825
1826 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
1827 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
1828 "dot product requires a vector of at least 2 components");
1829
1830 [[maybe_unused]] SPIRVType *EltType =
1831 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg());
1832
1833 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
1834
1835 MachineBasicBlock &BB = *I.getParent();
1836 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot))
1837 .addDef(ResVReg)
1838 .addUse(GR.getSPIRVTypeID(ResType))
1839 .addUse(I.getOperand(2).getReg())
1840 .addUse(I.getOperand(3).getReg())
1841 .constrainAllUses(TII, TRI, RBI);
1842}
1843
1844bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
1845 const SPIRVType *ResType,
1846 MachineInstr &I,
1847 bool Signed) const {
1848 assert(I.getNumOperands() == 4);
1849 assert(I.getOperand(2).isReg());
1850 assert(I.getOperand(3).isReg());
1851 MachineBasicBlock &BB = *I.getParent();
1852
1853 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
1854 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
1855 .addDef(ResVReg)
1856 .addUse(GR.getSPIRVTypeID(ResType))
1857 .addUse(I.getOperand(2).getReg())
1858 .addUse(I.getOperand(3).getReg())
1859 .constrainAllUses(TII, TRI, RBI);
1860}
1861
1862// Since pre-1.6 SPIRV has no integer dot implementation,
1863// expand by piecewise multiplying and adding the results
1864bool SPIRVInstructionSelector::selectIntegerDotExpansion(
1865 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
1866 assert(I.getNumOperands() == 4);
1867 assert(I.getOperand(2).isReg());
1868 assert(I.getOperand(3).isReg());
1869 MachineBasicBlock &BB = *I.getParent();
1870
1871 // Multiply the vectors, then sum the results
1872 Register Vec0 = I.getOperand(2).getReg();
1873 Register Vec1 = I.getOperand(3).getReg();
1874 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType));
1875 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0);
1876
1877 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV))
1878 .addDef(TmpVec)
1879 .addUse(GR.getSPIRVTypeID(VecType))
1880 .addUse(Vec0)
1881 .addUse(Vec1)
1882 .constrainAllUses(TII, TRI, RBI);
1883
1884 assert(VecType->getOpcode() == SPIRV::OpTypeVector &&
1885 GR.getScalarOrVectorComponentCount(VecType) > 1 &&
1886 "dot product requires a vector of at least 2 components");
1887
1888 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType));
1889 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1890 .addDef(Res)
1891 .addUse(GR.getSPIRVTypeID(ResType))
1892 .addUse(TmpVec)
1893 .addImm(0)
1894 .constrainAllUses(TII, TRI, RBI);
1895
1896 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) {
1897 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType));
1898
1899 Result &=
1900 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1901 .addDef(Elt)
1902 .addUse(GR.getSPIRVTypeID(ResType))
1903 .addUse(TmpVec)
1904 .addImm(i)
1905 .constrainAllUses(TII, TRI, RBI);
1906
1907 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1
1908 ? MRI->createVirtualRegister(GR.getRegClass(ResType))
1909 : ResVReg;
1910
1911 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
1912 .addDef(Sum)
1913 .addUse(GR.getSPIRVTypeID(ResType))
1914 .addUse(Res)
1915 .addUse(Elt)
1916 .constrainAllUses(TII, TRI, RBI);
1917 Res = Sum;
1918 }
1919
1920 return Result;
1921}
1922
1923template <bool Signed>
1924bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
1925 const SPIRVType *ResType,
1926 MachineInstr &I) const {
1927 assert(I.getNumOperands() == 5);
1928 assert(I.getOperand(2).isReg());
1929 assert(I.getOperand(3).isReg());
1930 assert(I.getOperand(4).isReg());
1931 MachineBasicBlock &BB = *I.getParent();
1932
1933 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
1934 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType));
1935 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
1936 .addDef(Dot)
1937 .addUse(GR.getSPIRVTypeID(ResType))
1938 .addUse(I.getOperand(2).getReg())
1939 .addUse(I.getOperand(3).getReg())
1940 .constrainAllUses(TII, TRI, RBI);
1941
1942 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
1943 .addDef(ResVReg)
1944 .addUse(GR.getSPIRVTypeID(ResType))
1945 .addUse(Dot)
1946 .addUse(I.getOperand(4).getReg())
1947 .constrainAllUses(TII, TRI, RBI);
1948}
1949
1950// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
1951// extract the elements of the packed inputs, multiply them and add the result
1952// to the accumulator.
1953template <bool Signed>
1954bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
1955 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
1956 assert(I.getNumOperands() == 5);
1957 assert(I.getOperand(2).isReg());
1958 assert(I.getOperand(3).isReg());
1959 assert(I.getOperand(4).isReg());
1960 MachineBasicBlock &BB = *I.getParent();
1961
1962 bool Result = true;
1963
1964 // Acc = C
1965 Register Acc = I.getOperand(4).getReg();
1966 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII);
1967 auto ExtractOp =
1968 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
1969
1970 // Extract the i8 element, multiply and add it to the accumulator
1971 for (unsigned i = 0; i < 4; i++) {
1972 // A[i]
1973 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1974 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
1975 .addDef(AElt)
1976 .addUse(GR.getSPIRVTypeID(ResType))
1977 .addUse(I.getOperand(2).getReg())
1978 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII))
1979 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII))
1980 .constrainAllUses(TII, TRI, RBI);
1981
1982 // B[i]
1983 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1984 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
1985 .addDef(BElt)
1986 .addUse(GR.getSPIRVTypeID(ResType))
1987 .addUse(I.getOperand(3).getReg())
1988 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII))
1989 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII))
1990 .constrainAllUses(TII, TRI, RBI);
1991
1992 // A[i] * B[i]
1993 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1994 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS))
1995 .addDef(Mul)
1996 .addUse(GR.getSPIRVTypeID(ResType))
1997 .addUse(AElt)
1998 .addUse(BElt)
1999 .constrainAllUses(TII, TRI, RBI);
2000
2001 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
2002 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2003 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
2004 .addDef(MaskMul)
2005 .addUse(GR.getSPIRVTypeID(ResType))
2006 .addUse(Mul)
2007 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII))
2008 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII))
2009 .constrainAllUses(TII, TRI, RBI);
2010
2011 // Acc = Acc + A[i] * B[i]
2012 Register Sum =
2013 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg;
2014 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
2015 .addDef(Sum)
2016 .addUse(GR.getSPIRVTypeID(ResType))
2017 .addUse(Acc)
2018 .addUse(MaskMul)
2019 .constrainAllUses(TII, TRI, RBI);
2020
2021 Acc = Sum;
2022 }
2023
2024 return Result;
2025}
2026
2027/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
2028/// does not have a saturate builtin.
2029bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
2030 const SPIRVType *ResType,
2031 MachineInstr &I) const {
2032 assert(I.getNumOperands() == 3);
2033 assert(I.getOperand(2).isReg());
2034 MachineBasicBlock &BB = *I.getParent();
2035 Register VZero = buildZerosValF(ResType, I);
2036 Register VOne = buildOnesValF(ResType, I);
2037
2038 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
2039 .addDef(ResVReg)
2040 .addUse(GR.getSPIRVTypeID(ResType))
2041 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2042 .addImm(GL::FClamp)
2043 .addUse(I.getOperand(2).getReg())
2044 .addUse(VZero)
2045 .addUse(VOne)
2046 .constrainAllUses(TII, TRI, RBI);
2047}
2048
2049bool SPIRVInstructionSelector::selectSign(Register ResVReg,
2050 const SPIRVType *ResType,
2051 MachineInstr &I) const {
2052 assert(I.getNumOperands() == 3);
2053 assert(I.getOperand(2).isReg());
2054 MachineBasicBlock &BB = *I.getParent();
2055 Register InputRegister = I.getOperand(2).getReg();
2056 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister);
2057 auto &DL = I.getDebugLoc();
2058
2059 if (!InputType)
2060 report_fatal_error("Input Type could not be determined.");
2061
2062 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
2063
2064 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType);
2065 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType);
2066
2067 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
2068
2069 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
2070 Register SignReg = NeedsConversion
2071 ? MRI->createVirtualRegister(&SPIRV::IDRegClass)
2072 : ResVReg;
2073
2074 bool Result =
2075 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst))
2076 .addDef(SignReg)
2077 .addUse(GR.getSPIRVTypeID(InputType))
2078 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2079 .addImm(SignOpcode)
2080 .addUse(InputRegister)
2081 .constrainAllUses(TII, TRI, RBI);
2082
2083 if (NeedsConversion) {
2084 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
2085 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode))
2086 .addDef(ResVReg)
2087 .addUse(GR.getSPIRVTypeID(ResType))
2088 .addUse(SignReg)
2089 .constrainAllUses(TII, TRI, RBI);
2090 }
2091
2092 return Result;
2093}
2094
2095bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
2096 const SPIRVType *ResType,
2097 MachineInstr &I,
2098 unsigned Opcode) const {
2099 MachineBasicBlock &BB = *I.getParent();
2100 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
2101
2102 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2103 .addDef(ResVReg)
2104 .addUse(GR.getSPIRVTypeID(ResType))
2105 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I,
2106 IntTy, TII));
2107
2108 for (unsigned J = 2; J < I.getNumOperands(); J++) {
2109 BMI.addUse(I.getOperand(J).getReg());
2110 }
2111
2112 return BMI.constrainAllUses(TII, TRI, RBI);
2113}
2114
2115bool SPIRVInstructionSelector::selectWaveActiveCountBits(
2116 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
2117
2118 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
2119 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII);
2120 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType));
2121 bool Result = selectWaveOpInst(BallotReg, BallotType, I,
2122 SPIRV::OpGroupNonUniformBallot);
2123
2124 MachineBasicBlock &BB = *I.getParent();
2125 Result &=
2126 BuildMI(BB, I, I.getDebugLoc(),
2127 TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
2128 .addDef(ResVReg)
2129 .addUse(GR.getSPIRVTypeID(ResType))
2130 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII))
2131 .addImm(SPIRV::GroupOperation::Reduce)
2132 .addUse(BallotReg)
2133 .constrainAllUses(TII, TRI, RBI);
2134
2135 return Result;
2136}
2137
2138bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
2139 const SPIRVType *ResType,
2140 MachineInstr &I,
2141 bool IsUnsigned) const {
2142 assert(I.getNumOperands() == 3);
2143 assert(I.getOperand(2).isReg());
2144 MachineBasicBlock &BB = *I.getParent();
2145 Register InputRegister = I.getOperand(2).getReg();
2146 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister);
2147
2148 if (!InputType)
2149 report_fatal_error("Input Type could not be determined.");
2150
2151 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
2152 // Retreive the operation to use based on input type
2153 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
2154 auto IntegerOpcodeType =
2155 IsUnsigned ? SPIRV::OpGroupNonUniformUMax : SPIRV::OpGroupNonUniformSMax;
2156 auto Opcode = IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntegerOpcodeType;
2157 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2158 .addDef(ResVReg)
2159 .addUse(GR.getSPIRVTypeID(ResType))
2160 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII))
2161 .addImm(SPIRV::GroupOperation::Reduce)
2162 .addUse(I.getOperand(2).getReg())
2163 .constrainAllUses(TII, TRI, RBI);
2164}
2165
2166bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
2167 const SPIRVType *ResType,
2168 MachineInstr &I) const {
2169 assert(I.getNumOperands() == 3);
2170 assert(I.getOperand(2).isReg());
2171 MachineBasicBlock &BB = *I.getParent();
2172 Register InputRegister = I.getOperand(2).getReg();
2173 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister);
2174
2175 if (!InputType)
2176 report_fatal_error("Input Type could not be determined.");
2177
2178 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
2179 // Retreive the operation to use based on input type
2180 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
2181 auto Opcode =
2182 IsFloatTy ? SPIRV::OpGroupNonUniformFAdd : SPIRV::OpGroupNonUniformIAdd;
2183 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2184 .addDef(ResVReg)
2185 .addUse(GR.getSPIRVTypeID(ResType))
2186 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII))
2187 .addImm(SPIRV::GroupOperation::Reduce)
2188 .addUse(I.getOperand(2).getReg());
2189}
2190
2191bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
2192 const SPIRVType *ResType,
2193 MachineInstr &I) const {
2194 MachineBasicBlock &BB = *I.getParent();
2195 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
2196 .addDef(ResVReg)
2197 .addUse(GR.getSPIRVTypeID(ResType))
2198 .addUse(I.getOperand(1).getReg())
2199 .constrainAllUses(TII, TRI, RBI);
2200}
2201
2202bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
2203 const SPIRVType *ResType,
2204 MachineInstr &I) const {
2205 // There is no way to implement `freeze` correctly without support on SPIR-V
2206 // standard side, but we may at least address a simple (static) case when
2207 // undef/poison value presence is obvious. The main benefit of even
2208 // incomplete `freeze` support is preventing of translation from crashing due
2209 // to lack of support on legalization and instruction selection steps.
2210 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg())
2211 return false;
2212 Register OpReg = I.getOperand(1).getReg();
2213 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) {
2214 Register Reg;
2215 switch (Def->getOpcode()) {
2216 case SPIRV::ASSIGN_TYPE:
2217 if (MachineInstr *AssignToDef =
2218 MRI->getVRegDef(Def->getOperand(1).getReg())) {
2219 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
2220 Reg = Def->getOperand(2).getReg();
2221 }
2222 break;
2223 case SPIRV::OpUndef:
2224 Reg = Def->getOperand(1).getReg();
2225 break;
2226 }
2227 unsigned DestOpCode;
2228 if (Reg.isValid()) {
2229 DestOpCode = SPIRV::OpConstantNull;
2230 } else {
2231 DestOpCode = TargetOpcode::COPY;
2232 Reg = OpReg;
2233 }
2234 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode))
2235 .addDef(I.getOperand(0).getReg())
2236 .addUse(Reg)
2237 .constrainAllUses(TII, TRI, RBI);
2238 }
2239 return false;
2240}
2241
2243 const SPIRVType *ResType) {
2244 Register OpReg = ResType->getOperand(2).getReg();
2245 SPIRVType *OpDef = MRI->getVRegDef(OpReg);
2246 if (!OpDef)
2247 return 0;
2248 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
2249 OpDef->getOperand(1).isReg()) {
2250 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
2251 OpDef = RefDef;
2252 }
2253 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT
2254 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue()
2255 : 0;
2256 return N;
2257}
2258
2259// Return true if the type represents a constant register
2261 SmallPtrSet<SPIRVType *, 4> &Visited) {
2262 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
2263 OpDef->getOperand(1).isReg()) {
2264 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
2265 OpDef = RefDef;
2266 }
2267
2268 if (Visited.contains(OpDef))
2269 return true;
2270 Visited.insert(OpDef);
2271
2272 unsigned Opcode = OpDef->getOpcode();
2273 switch (Opcode) {
2274 case TargetOpcode::G_CONSTANT:
2275 case TargetOpcode::G_FCONSTANT:
2276 return true;
2277 case TargetOpcode::G_INTRINSIC:
2278 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
2279 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
2280 return cast<GIntrinsic>(*OpDef).getIntrinsicID() ==
2281 Intrinsic::spv_const_composite;
2282 case TargetOpcode::G_BUILD_VECTOR:
2283 case TargetOpcode::G_SPLAT_VECTOR: {
2284 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands();
2285 i++) {
2286 SPIRVType *OpNestedDef =
2287 OpDef->getOperand(i).isReg()
2288 ? MRI->getVRegDef(OpDef->getOperand(i).getReg())
2289 : nullptr;
2290 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited))
2291 return false;
2292 }
2293 return true;
2294 }
2295 }
2296 return false;
2297}
2298
2299// Return true if the virtual register represents a constant
2302 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg))
2303 return isConstReg(MRI, OpDef, Visited);
2304 return false;
2305}
2306
2307bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
2308 const SPIRVType *ResType,
2309 MachineInstr &I) const {
2310 unsigned N = 0;
2311 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2312 N = GR.getScalarOrVectorComponentCount(ResType);
2313 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
2314 N = getArrayComponentCount(MRI, ResType);
2315 else
2316 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result");
2317 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
2318 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent");
2319
2320 // check if we may construct a constant vector
2321 bool IsConst = true;
2322 for (unsigned i = I.getNumExplicitDefs();
2323 i < I.getNumExplicitOperands() && IsConst; ++i)
2324 if (!isConstReg(MRI, I.getOperand(i).getReg()))
2325 IsConst = false;
2326
2327 if (!IsConst && N < 2)
2329 "There must be at least two constituent operands in a vector");
2330
2331 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
2332 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
2333 TII.get(IsConst ? SPIRV::OpConstantComposite
2334 : SPIRV::OpCompositeConstruct))
2335 .addDef(ResVReg)
2336 .addUse(GR.getSPIRVTypeID(ResType));
2337 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
2338 MIB.addUse(I.getOperand(i).getReg());
2339 return MIB.constrainAllUses(TII, TRI, RBI);
2340}
2341
2342bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
2343 const SPIRVType *ResType,
2344 MachineInstr &I) const {
2345 unsigned N = 0;
2346 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2347 N = GR.getScalarOrVectorComponentCount(ResType);
2348 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
2349 N = getArrayComponentCount(MRI, ResType);
2350 else
2351 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result");
2352
2353 unsigned OpIdx = I.getNumExplicitDefs();
2354 if (!I.getOperand(OpIdx).isReg())
2355 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR");
2356
2357 // check if we may construct a constant vector
2358 Register OpReg = I.getOperand(OpIdx).getReg();
2359 bool IsConst = isConstReg(MRI, OpReg);
2360
2361 if (!IsConst && N < 2)
2363 "There must be at least two constituent operands in a vector");
2364
2365 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
2366 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
2367 TII.get(IsConst ? SPIRV::OpConstantComposite
2368 : SPIRV::OpCompositeConstruct))
2369 .addDef(ResVReg)
2370 .addUse(GR.getSPIRVTypeID(ResType));
2371 for (unsigned i = 0; i < N; ++i)
2372 MIB.addUse(OpReg);
2373 return MIB.constrainAllUses(TII, TRI, RBI);
2374}
2375
2376bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
2377 const SPIRVType *ResType,
2378 MachineInstr &I) const {
2379
2380 unsigned Opcode;
2381
2382 if (STI.canUseExtension(
2383 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
2384 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) {
2385 Opcode = SPIRV::OpDemoteToHelperInvocation;
2386 } else {
2387 Opcode = SPIRV::OpKill;
2388 // OpKill must be the last operation of any basic block.
2389 if (MachineInstr *NextI = I.getNextNode()) {
2390 GR.invalidateMachineInstr(NextI);
2391 NextI->removeFromParent();
2392 }
2393 }
2394
2395 MachineBasicBlock &BB = *I.getParent();
2396 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2397 .constrainAllUses(TII, TRI, RBI);
2398}
2399
2400bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
2401 const SPIRVType *ResType,
2402 unsigned CmpOpc,
2403 MachineInstr &I) const {
2404 Register Cmp0 = I.getOperand(2).getReg();
2405 Register Cmp1 = I.getOperand(3).getReg();
2406 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
2407 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
2408 "CMP operands should have the same type");
2409 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
2410 .addDef(ResVReg)
2411 .addUse(GR.getSPIRVTypeID(ResType))
2412 .addUse(Cmp0)
2413 .addUse(Cmp1)
2414 .constrainAllUses(TII, TRI, RBI);
2415}
2416
2417bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
2418 const SPIRVType *ResType,
2419 MachineInstr &I) const {
2420 auto Pred = I.getOperand(1).getPredicate();
2421 unsigned CmpOpc;
2422
2423 Register CmpOperand = I.getOperand(2).getReg();
2424 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer))
2425 CmpOpc = getPtrCmpOpcode(Pred);
2426 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
2427 CmpOpc = getBoolCmpOpcode(Pred);
2428 else
2429 CmpOpc = getICmpOpcode(Pred);
2430 return selectCmp(ResVReg, ResType, CmpOpc, I);
2431}
2432
2433void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB,
2434 const MachineInstr &I,
2435 int OpIdx) const {
2436 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
2437 "Expected G_FCONSTANT");
2438 const ConstantFP *FPImm = I.getOperand(1).getFPImm();
2439 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB);
2440}
2441
2442void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB,
2443 const MachineInstr &I,
2444 int OpIdx) const {
2445 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
2446 "Expected G_CONSTANT");
2447 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB);
2448}
2449
2450std::pair<Register, bool>
2451SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
2452 const SPIRVType *ResType) const {
2453 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
2454 const SPIRVType *SpvI32Ty =
2455 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
2456 // Find a constant in DT or build a new one.
2457 auto ConstInt = ConstantInt::get(LLVMTy, Val);
2458 Register NewReg = GR.find(ConstInt, GR.CurMF);
2459 bool Result = true;
2460 if (!NewReg.isValid()) {
2461 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2462 GR.add(ConstInt, GR.CurMF, NewReg);
2464 MachineBasicBlock &BB = *I.getParent();
2465 if (Val == 0) {
2466 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
2467 .addDef(NewReg)
2468 .addUse(GR.getSPIRVTypeID(SpvI32Ty));
2469 } else {
2470 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
2471 .addDef(NewReg)
2472 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
2473 .addImm(APInt(32, Val).getZExtValue());
2474 }
2476 }
2477 return {NewReg, Result};
2478}
2479
2480bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
2481 const SPIRVType *ResType,
2482 MachineInstr &I) const {
2483 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
2484 return selectCmp(ResVReg, ResType, CmpOp, I);
2485}
2486
2487Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
2488 MachineInstr &I) const {
2489 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
2490 bool ZeroAsNull = STI.isOpenCLEnv();
2491 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2492 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
2493 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
2494}
2495
2496Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType,
2497 MachineInstr &I) const {
2498 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
2499 bool ZeroAsNull = STI.isOpenCLEnv();
2500 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType));
2501 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2502 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
2503 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull);
2504}
2505
2506Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType,
2507 MachineInstr &I) const {
2508 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
2509 bool ZeroAsNull = STI.isOpenCLEnv();
2510 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType));
2511 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2512 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull);
2513 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull);
2514}
2515
2516Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
2517 const SPIRVType *ResType,
2518 MachineInstr &I) const {
2519 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
2520 APInt One =
2521 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
2522 if (ResType->getOpcode() == SPIRV::OpTypeVector)
2523 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII);
2524 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII);
2525}
2526
2527bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
2528 const SPIRVType *ResType,
2529 MachineInstr &I,
2530 bool IsSigned) const {
2531 // To extend a bool, we need to use OpSelect between constants.
2532 Register ZeroReg = buildZerosVal(ResType, I);
2533 Register OneReg = buildOnesVal(IsSigned, ResType, I);
2534 bool IsScalarBool =
2535 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
2536 unsigned Opcode =
2537 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
2538 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
2539 .addDef(ResVReg)
2540 .addUse(GR.getSPIRVTypeID(ResType))
2541 .addUse(I.getOperand(1).getReg())
2542 .addUse(OneReg)
2543 .addUse(ZeroReg)
2544 .constrainAllUses(TII, TRI, RBI);
2545}
2546
2547bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
2548 const SPIRVType *ResType,
2549 MachineInstr &I, bool IsSigned,
2550 unsigned Opcode) const {
2551 Register SrcReg = I.getOperand(1).getReg();
2552 // We can convert bool value directly to float type without OpConvert*ToF,
2553 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
2554 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
2555 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
2556 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
2557 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
2558 const unsigned NumElts = ResType->getOperand(2).getImm();
2559 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
2560 }
2561 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
2562 selectSelect(SrcReg, TmpType, I, false);
2563 }
2564 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
2565}
2566
2567bool SPIRVInstructionSelector::selectExt(Register ResVReg,
2568 const SPIRVType *ResType,
2569 MachineInstr &I, bool IsSigned) const {
2570 Register SrcReg = I.getOperand(1).getReg();
2571 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool))
2572 return selectSelect(ResVReg, ResType, I, IsSigned);
2573
2574 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg);
2575 if (SrcType == ResType)
2576 return BuildCOPY(ResVReg, SrcReg, I);
2577
2578 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
2579 return selectUnOp(ResVReg, ResType, I, Opcode);
2580}
2581
2582bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
2583 const SPIRVType *ResType,
2584 MachineInstr &I,
2585 bool IsSigned) const {
2586 MachineIRBuilder MIRBuilder(I);
2587 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2588 MachineBasicBlock &BB = *I.getParent();
2589 // Ensure we have bool.
2590 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
2591 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
2592 if (N > 1)
2593 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
2594 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
2595 // Build less-than-equal and less-than.
2596 // TODO: replace with one-liner createVirtualRegister() from
2597 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged.
2598 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
2599 MRI->setType(IsLessEqReg, LLT::scalar(64));
2600 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF());
2601 bool Result = BuildMI(BB, I, I.getDebugLoc(),
2602 TII.get(IsSigned ? SPIRV::OpSLessThanEqual
2603 : SPIRV::OpULessThanEqual))
2604 .addDef(IsLessEqReg)
2605 .addUse(BoolTypeReg)
2606 .addUse(I.getOperand(1).getReg())
2607 .addUse(I.getOperand(2).getReg())
2608 .constrainAllUses(TII, TRI, RBI);
2609 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
2610 MRI->setType(IsLessReg, LLT::scalar(64));
2611 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF());
2612 Result &= BuildMI(BB, I, I.getDebugLoc(),
2613 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
2614 .addDef(IsLessReg)
2615 .addUse(BoolTypeReg)
2616 .addUse(I.getOperand(1).getReg())
2617 .addUse(I.getOperand(2).getReg())
2618 .constrainAllUses(TII, TRI, RBI);
2619 // Build selects.
2620 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
2621 Register NegOneOrZeroReg =
2622 MRI->createVirtualRegister(GR.getRegClass(ResType));
2623 MRI->setType(NegOneOrZeroReg, LLT::scalar(64));
2624 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF());
2625 unsigned SelectOpcode =
2626 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond;
2627 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
2628 .addDef(NegOneOrZeroReg)
2629 .addUse(ResTypeReg)
2630 .addUse(IsLessReg)
2631 .addUse(buildOnesVal(true, ResType, I)) // -1
2632 .addUse(buildZerosVal(ResType, I))
2633 .constrainAllUses(TII, TRI, RBI);
2634 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
2635 .addDef(ResVReg)
2636 .addUse(ResTypeReg)
2637 .addUse(IsLessEqReg)
2638 .addUse(NegOneOrZeroReg) // -1 or 0
2639 .addUse(buildOnesVal(false, ResType, I))
2640 .constrainAllUses(TII, TRI, RBI);
2641}
2642
2643bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
2644 Register ResVReg,
2645 MachineInstr &I,
2646 const SPIRVType *IntTy,
2647 const SPIRVType *BoolTy) const {
2648 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
2649 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
2650 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
2651 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
2652 Register Zero = buildZerosVal(IntTy, I);
2653 Register One = buildOnesVal(false, IntTy, I);
2654 MachineBasicBlock &BB = *I.getParent();
2655 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2656 .addDef(BitIntReg)
2657 .addUse(GR.getSPIRVTypeID(IntTy))
2658 .addUse(IntReg)
2659 .addUse(One)
2660 .constrainAllUses(TII, TRI, RBI);
2661 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
2662 .addDef(ResVReg)
2663 .addUse(GR.getSPIRVTypeID(BoolTy))
2664 .addUse(BitIntReg)
2665 .addUse(Zero)
2666 .constrainAllUses(TII, TRI, RBI);
2667}
2668
2669bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
2670 const SPIRVType *ResType,
2671 MachineInstr &I) const {
2672 Register IntReg = I.getOperand(1).getReg();
2673 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg);
2674 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool))
2675 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
2676 if (ArgType == ResType)
2677 return BuildCOPY(ResVReg, IntReg, I);
2678 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
2679 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
2680 return selectUnOp(ResVReg, ResType, I, Opcode);
2681}
2682
2683bool SPIRVInstructionSelector::selectConst(Register ResVReg,
2684 const SPIRVType *ResType,
2685 const APInt &Imm,
2686 MachineInstr &I) const {
2687 unsigned TyOpcode = ResType->getOpcode();
2688 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero());
2689 MachineBasicBlock &BB = *I.getParent();
2690 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) &&
2691 Imm.isZero())
2692 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
2693 .addDef(ResVReg)
2694 .addUse(GR.getSPIRVTypeID(ResType))
2695 .constrainAllUses(TII, TRI, RBI);
2696 if (TyOpcode == SPIRV::OpTypeInt) {
2697 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!");
2698 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII);
2699 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I);
2700 }
2701 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
2702 .addDef(ResVReg)
2703 .addUse(GR.getSPIRVTypeID(ResType));
2704 // <=32-bit integers should be caught by the sdag pattern.
2705 assert(Imm.getBitWidth() > 32);
2706 addNumImm(Imm, MIB);
2707 return MIB.constrainAllUses(TII, TRI, RBI);
2708}
2709
2710bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
2711 const SPIRVType *ResType,
2712 MachineInstr &I) const {
2713 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
2714 .addDef(ResVReg)
2715 .addUse(GR.getSPIRVTypeID(ResType))
2716 .constrainAllUses(TII, TRI, RBI);
2717}
2718
2720 assert(MO.isReg());
2721 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
2722 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) {
2723 assert(TypeInst->getOperand(1).isReg());
2724 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
2725 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT;
2726 }
2727 return TypeInst->getOpcode() == SPIRV::OpConstantI;
2728}
2729
2730static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
2731 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
2732 if (TypeInst->getOpcode() == SPIRV::OpConstantI)
2733 return TypeInst->getOperand(2).getImm();
2734 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
2735 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT);
2736 return ImmInst->getOperand(1).getCImm()->getZExtValue();
2737}
2738
2739bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
2740 const SPIRVType *ResType,
2741 MachineInstr &I) const {
2742 MachineBasicBlock &BB = *I.getParent();
2743 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
2744 .addDef(ResVReg)
2745 .addUse(GR.getSPIRVTypeID(ResType))
2746 // object to insert
2747 .addUse(I.getOperand(3).getReg())
2748 // composite to insert into
2749 .addUse(I.getOperand(2).getReg());
2750 for (unsigned i = 4; i < I.getNumOperands(); i++)
2751 MIB.addImm(foldImm(I.getOperand(i), MRI));
2752 return MIB.constrainAllUses(TII, TRI, RBI);
2753}
2754
2755bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
2756 const SPIRVType *ResType,
2757 MachineInstr &I) const {
2758 MachineBasicBlock &BB = *I.getParent();
2759 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2760 .addDef(ResVReg)
2761 .addUse(GR.getSPIRVTypeID(ResType))
2762 .addUse(I.getOperand(2).getReg());
2763 for (unsigned i = 3; i < I.getNumOperands(); i++)
2764 MIB.addImm(foldImm(I.getOperand(i), MRI));
2765 return MIB.constrainAllUses(TII, TRI, RBI);
2766}
2767
2768bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
2769 const SPIRVType *ResType,
2770 MachineInstr &I) const {
2771 if (isImm(I.getOperand(4), MRI))
2772 return selectInsertVal(ResVReg, ResType, I);
2773 MachineBasicBlock &BB = *I.getParent();
2774 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
2775 .addDef(ResVReg)
2776 .addUse(GR.getSPIRVTypeID(ResType))
2777 .addUse(I.getOperand(2).getReg())
2778 .addUse(I.getOperand(3).getReg())
2779 .addUse(I.getOperand(4).getReg())
2780 .constrainAllUses(TII, TRI, RBI);
2781}
2782
2783bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
2784 const SPIRVType *ResType,
2785 MachineInstr &I) const {
2786 if (isImm(I.getOperand(3), MRI))
2787 return selectExtractVal(ResVReg, ResType, I);
2788 MachineBasicBlock &BB = *I.getParent();
2789 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
2790 .addDef(ResVReg)
2791 .addUse(GR.getSPIRVTypeID(ResType))
2792 .addUse(I.getOperand(2).getReg())
2793 .addUse(I.getOperand(3).getReg())
2794 .constrainAllUses(TII, TRI, RBI);
2795}
2796
2797bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
2798 const SPIRVType *ResType,
2799 MachineInstr &I) const {
2800 const bool IsGEPInBounds = I.getOperand(2).getImm();
2801
2802 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
2803 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
2804 // we have to use Op[InBounds]AccessChain.
2805 const unsigned Opcode = STI.isVulkanEnv()
2806 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
2807 : SPIRV::OpAccessChain)
2808 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
2809 : SPIRV::OpPtrAccessChain);
2810
2811 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
2812 .addDef(ResVReg)
2813 .addUse(GR.getSPIRVTypeID(ResType))
2814 // Object to get a pointer to.
2815 .addUse(I.getOperand(3).getReg());
2816 // Adding indices.
2817 const unsigned StartingIndex =
2818 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
2819 ? 5
2820 : 4;
2821 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
2822 Res.addUse(I.getOperand(i).getReg());
2823 return Res.constrainAllUses(TII, TRI, RBI);
2824}
2825
2826// Maybe wrap a value into OpSpecConstantOp
2827bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
2828 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
2829 bool Result = true;
2830 unsigned Lim = I.getNumExplicitOperands();
2831 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
2832 Register OpReg = I.getOperand(i).getReg();
2833 SPIRVType *OpDefine = MRI->getVRegDef(OpReg);
2834 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg);
2836 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) ||
2837 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
2838 GR.isAggregateType(OpType)) {
2839 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
2840 // by selectAddrSpaceCast()
2841 CompositeArgs.push_back(OpReg);
2842 continue;
2843 }
2844 MachineFunction *MF = I.getMF();
2845 Register WrapReg = GR.find(OpDefine, MF);
2846 if (WrapReg.isValid()) {
2847 CompositeArgs.push_back(WrapReg);
2848 continue;
2849 }
2850 // Create a new register for the wrapper
2851 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType));
2852 GR.add(OpDefine, MF, WrapReg);
2853 CompositeArgs.push_back(WrapReg);
2854 // Decorate the wrapper register and generate a new instruction
2855 MRI->setType(WrapReg, LLT::pointer(0, 64));
2856 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF);
2857 MachineBasicBlock &BB = *I.getParent();
2858 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
2859 .addDef(WrapReg)
2860 .addUse(GR.getSPIRVTypeID(OpType))
2861 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
2862 .addUse(OpReg)
2863 .constrainAllUses(TII, TRI, RBI);
2864 if (!Result)
2865 break;
2866 }
2867 return Result;
2868}
2869
2870bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
2871 const SPIRVType *ResType,
2872 MachineInstr &I) const {
2873 MachineBasicBlock &BB = *I.getParent();
2874 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID();
2875 switch (IID) {
2876 case Intrinsic::spv_load:
2877 return selectLoad(ResVReg, ResType, I);
2878 case Intrinsic::spv_store:
2879 return selectStore(I);
2880 case Intrinsic::spv_extractv:
2881 return selectExtractVal(ResVReg, ResType, I);
2882 case Intrinsic::spv_insertv:
2883 return selectInsertVal(ResVReg, ResType, I);
2884 case Intrinsic::spv_extractelt:
2885 return selectExtractElt(ResVReg, ResType, I);
2886 case Intrinsic::spv_insertelt:
2887 return selectInsertElt(ResVReg, ResType, I);
2888 case Intrinsic::spv_gep:
2889 return selectGEP(ResVReg, ResType, I);
2890 case Intrinsic::spv_unref_global:
2891 case Intrinsic::spv_init_global: {
2892 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
2893 MachineInstr *Init = I.getNumExplicitOperands() > 2
2894 ? MRI->getVRegDef(I.getOperand(2).getReg())
2895 : nullptr;
2896 assert(MI);
2897 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init);
2898 }
2899 case Intrinsic::spv_undef: {
2900 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
2901 .addDef(ResVReg)
2902 .addUse(GR.getSPIRVTypeID(ResType));
2903 return MIB.constrainAllUses(TII, TRI, RBI);
2904 }
2905 case Intrinsic::spv_const_composite: {
2906 // If no values are attached, the composite is null constant.
2907 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
2908 // Select a proper instruction.
2909 unsigned Opcode = SPIRV::OpConstantNull;
2910 SmallVector<Register> CompositeArgs;
2911 if (!IsNull) {
2912 Opcode = SPIRV::OpConstantComposite;
2913 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
2914 return false;
2915 }
2916 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
2917 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2918 .addDef(ResVReg)
2919 .addUse(GR.getSPIRVTypeID(ResType));
2920 // skip type MD node we already used when generated assign.type for this
2921 if (!IsNull) {
2922 for (Register OpReg : CompositeArgs)
2923 MIB.addUse(OpReg);
2924 }
2925 return MIB.constrainAllUses(TII, TRI, RBI);
2926 }
2927 case Intrinsic::spv_assign_name: {
2928 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
2929 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
2930 for (unsigned i = I.getNumExplicitDefs() + 2;
2931 i < I.getNumExplicitOperands(); ++i) {
2932 MIB.addImm(I.getOperand(i).getImm());
2933 }
2934 return MIB.constrainAllUses(TII, TRI, RBI);
2935 }
2936 case Intrinsic::spv_switch: {
2937 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
2938 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
2939 if (I.getOperand(i).isReg())
2940 MIB.addReg(I.getOperand(i).getReg());
2941 else if (I.getOperand(i).isCImm())
2942 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
2943 else if (I.getOperand(i).isMBB())
2944 MIB.addMBB(I.getOperand(i).getMBB());
2945 else
2946 llvm_unreachable("Unexpected OpSwitch operand");
2947 }
2948 return MIB.constrainAllUses(TII, TRI, RBI);
2949 }
2950 case Intrinsic::spv_loop_merge: {
2951 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge));
2952 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
2953 assert(I.getOperand(i).isMBB());
2954 MIB.addMBB(I.getOperand(i).getMBB());
2955 }
2956 MIB.addImm(SPIRV::SelectionControl::None);
2957 return MIB.constrainAllUses(TII, TRI, RBI);
2958 }
2959 case Intrinsic::spv_selection_merge: {
2960 auto MIB =
2961 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge));
2962 assert(I.getOperand(1).isMBB() &&
2963 "operand 1 to spv_selection_merge must be a basic block");
2964 MIB.addMBB(I.getOperand(1).getMBB());
2965 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm()));
2966 return MIB.constrainAllUses(TII, TRI, RBI);
2967 }
2968 case Intrinsic::spv_cmpxchg:
2969 return selectAtomicCmpXchg(ResVReg, ResType, I);
2970 case Intrinsic::spv_unreachable:
2971 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable))
2972 .constrainAllUses(TII, TRI, RBI);
2973 case Intrinsic::spv_alloca:
2974 return selectFrameIndex(ResVReg, ResType, I);
2975 case Intrinsic::spv_alloca_array:
2976 return selectAllocaArray(ResVReg, ResType, I);
2977 case Intrinsic::spv_assume:
2978 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
2979 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
2980 .addUse(I.getOperand(1).getReg())
2981 .constrainAllUses(TII, TRI, RBI);
2982 break;
2983 case Intrinsic::spv_expect:
2984 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
2985 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
2986 .addDef(ResVReg)
2987 .addUse(GR.getSPIRVTypeID(ResType))
2988 .addUse(I.getOperand(2).getReg())
2989 .addUse(I.getOperand(3).getReg())
2990 .constrainAllUses(TII, TRI, RBI);
2991 break;
2992 case Intrinsic::arithmetic_fence:
2993 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence))
2994 return BuildMI(BB, I, I.getDebugLoc(),
2995 TII.get(SPIRV::OpArithmeticFenceEXT))
2996 .addDef(ResVReg)
2997 .addUse(GR.getSPIRVTypeID(ResType))
2998 .addUse(I.getOperand(2).getReg())
2999 .constrainAllUses(TII, TRI, RBI);
3000 else
3001 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I);
3002 break;
3003 case Intrinsic::spv_thread_id:
3004 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
3005 // intrinsic in LLVM IR for SPIR-V backend.
3006 //
3007 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
3008 // `GlobalInvocationId` builtin variable
3009 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
3010 ResType, I);
3011 case Intrinsic::spv_thread_id_in_group:
3012 // The HLSL SV_GroupThreadId semantic is lowered to
3013 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
3014 //
3015 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
3016 // translated to a `LocalInvocationId` builtin variable
3017 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg,
3018 ResType, I);
3019 case Intrinsic::spv_group_id:
3020 // The HLSL SV_GroupId semantic is lowered to
3021 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
3022 //
3023 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
3024 // builtin variable
3025 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
3026 I);
3027 case Intrinsic::spv_fdot:
3028 return selectFloatDot(ResVReg, ResType, I);
3029 case Intrinsic::spv_udot:
3030 case Intrinsic::spv_sdot:
3031 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3032 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
3033 return selectIntegerDot(ResVReg, ResType, I,
3034 /*Signed=*/IID == Intrinsic::spv_sdot);
3035 return selectIntegerDotExpansion(ResVReg, ResType, I);
3036 case Intrinsic::spv_dot4add_i8packed:
3037 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3038 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
3039 return selectDot4AddPacked<true>(ResVReg, ResType, I);
3040 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
3041 case Intrinsic::spv_dot4add_u8packed:
3042 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
3043 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
3044 return selectDot4AddPacked<false>(ResVReg, ResType, I);
3045 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
3046 case Intrinsic::spv_all:
3047 return selectAll(ResVReg, ResType, I);
3048 case Intrinsic::spv_any:
3049 return selectAny(ResVReg, ResType, I);
3050 case Intrinsic::spv_cross:
3051 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross);
3052 case Intrinsic::spv_distance:
3053 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance);
3054 case Intrinsic::spv_lerp:
3055 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix);
3056 case Intrinsic::spv_length:
3057 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length);
3058 case Intrinsic::spv_degrees:
3059 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees);
3060 case Intrinsic::spv_frac:
3061 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract);
3062 case Intrinsic::spv_normalize:
3063 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize);
3064 case Intrinsic::spv_rsqrt:
3065 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt);
3066 case Intrinsic::spv_sign:
3067 return selectSign(ResVReg, ResType, I);
3068 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
3069 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
3070 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
3071 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
3072 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
3073 return selectFirstBitLow(ResVReg, ResType, I);
3074 case Intrinsic::spv_group_memory_barrier_with_group_sync: {
3075 bool Result = true;
3076 auto MemSemConstant =
3077 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I);
3078 Register MemSemReg = MemSemConstant.first;
3079 Result &= MemSemConstant.second;
3080 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I);
3081 Register ScopeReg = ScopeConstant.first;
3082 Result &= ScopeConstant.second;
3083 MachineBasicBlock &BB = *I.getParent();
3084 return Result &&
3085 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier))
3086 .addUse(ScopeReg)
3087 .addUse(ScopeReg)
3088 .addUse(MemSemReg)
3089 .constrainAllUses(TII, TRI, RBI);
3090 }
3091 case Intrinsic::spv_lifetime_start:
3092 case Intrinsic::spv_lifetime_end: {
3093 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
3094 : SPIRV::OpLifetimeStop;
3095 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm();
3096 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg();
3097 if (Size == -1)
3098 Size = 0;
3099 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op))
3100 .addUse(PtrReg)
3101 .addImm(Size)
3102 .constrainAllUses(TII, TRI, RBI);
3103 }
3104 case Intrinsic::spv_saturate:
3105 return selectSaturate(ResVReg, ResType, I);
3106 case Intrinsic::spv_nclamp:
3107 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp);
3108 case Intrinsic::spv_uclamp:
3109 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp);
3110 case Intrinsic::spv_sclamp:
3111 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp);
3112 case Intrinsic::spv_wave_active_countbits:
3113 return selectWaveActiveCountBits(ResVReg, ResType, I);
3114 case Intrinsic::spv_wave_all:
3115 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll);
3116 case Intrinsic::spv_wave_any:
3117 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
3118 case Intrinsic::spv_wave_is_first_lane:
3119 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect);
3120 case Intrinsic::spv_wave_reduce_umax:
3121 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
3122 case Intrinsic::spv_wave_reduce_max:
3123 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
3124 case Intrinsic::spv_wave_reduce_sum:
3125 return selectWaveReduceSum(ResVReg, ResType, I);
3126 case Intrinsic::spv_wave_readlane:
3127 return selectWaveOpInst(ResVReg, ResType, I,
3128 SPIRV::OpGroupNonUniformShuffle);
3129 case Intrinsic::spv_step:
3130 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step);
3131 case Intrinsic::spv_radians:
3132 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians);
3133 // Discard intrinsics which we do not expect to actually represent code after
3134 // lowering or intrinsics which are not implemented but should not crash when
3135 // found in a customer's LLVM IR input.
3136 case Intrinsic::instrprof_increment:
3137 case Intrinsic::instrprof_increment_step:
3138 case Intrinsic::instrprof_value_profile:
3139 break;
3140 // Discard internal intrinsics.
3141 case Intrinsic::spv_value_md:
3142 break;
3143 case Intrinsic::spv_resource_handlefrombinding: {
3144 return selectHandleFromBinding(ResVReg, ResType, I);
3145 }
3146 case Intrinsic::spv_resource_store_typedbuffer: {
3147 return selectImageWriteIntrinsic(I);
3148 }
3149 case Intrinsic::spv_resource_load_typedbuffer: {
3150 return selectReadImageIntrinsic(ResVReg, ResType, I);
3151 }
3152 case Intrinsic::spv_resource_getpointer: {
3153 return selectResourceGetPointer(ResVReg, ResType, I);
3154 }
3155 case Intrinsic::spv_discard: {
3156 return selectDiscard(ResVReg, ResType, I);
3157 }
3158 default: {
3159 std::string DiagMsg;
3160 raw_string_ostream OS(DiagMsg);
3161 I.print(OS);
3162 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
3163 report_fatal_error(DiagMsg.c_str(), false);
3164 }
3165 }
3166 return true;
3167}
3168
3169bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
3170 const SPIRVType *ResType,
3171 MachineInstr &I) const {
3172 return true;
3173}
3174
3175bool SPIRVInstructionSelector::selectReadImageIntrinsic(
3176 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
3177
3178 // If the load of the image is in a different basic block, then
3179 // this will generate invalid code. A proper solution is to move
3180 // the OpLoad from selectHandleFromBinding here. However, to do
3181 // that we will need to change the return type of the intrinsic.
3182 // We will do that when we can, but for now trying to move forward with other
3183 // issues.
3184 Register ImageReg = I.getOperand(2).getReg();
3185 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
3186 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
3187 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
3188 *ImageDef, I)) {
3189 return false;
3190 }
3191
3192 Register IdxReg = I.getOperand(3).getReg();
3193 DebugLoc Loc = I.getDebugLoc();
3194 MachineInstr &Pos = I;
3195
3196 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg, Loc, Pos);
3197}
3198
3199bool SPIRVInstructionSelector::generateImageRead(Register &ResVReg,
3200 const SPIRVType *ResType,
3201 Register ImageReg,
3202 Register IdxReg, DebugLoc Loc,
3203 MachineInstr &Pos) const {
3204 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
3205 if (ResultSize == 4) {
3206 return BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
3207 .addDef(ResVReg)
3208 .addUse(GR.getSPIRVTypeID(ResType))
3209 .addUse(ImageReg)
3210 .addUse(IdxReg)
3211 .constrainAllUses(TII, TRI, RBI);
3212 }
3213
3214 SPIRVType *ReadType = widenTypeToVec4(ResType, Pos);
3215 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType));
3216 bool Succeed =
3217 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
3218 .addDef(ReadReg)
3219 .addUse(GR.getSPIRVTypeID(ReadType))
3220 .addUse(ImageReg)
3221 .addUse(IdxReg)
3222 .constrainAllUses(TII, TRI, RBI);
3223 if (!Succeed)
3224 return false;
3225
3226 if (ResultSize == 1) {
3227 return BuildMI(*Pos.getParent(), Pos, Loc,
3228 TII.get(SPIRV::OpCompositeExtract))
3229 .addDef(ResVReg)
3230 .addUse(GR.getSPIRVTypeID(ResType))
3231 .addUse(ReadReg)
3232 .addImm(0)
3233 .constrainAllUses(TII, TRI, RBI);
3234 }
3235 return extractSubvector(ResVReg, ResType, ReadReg, Pos);
3236}
3237
3238bool SPIRVInstructionSelector::selectResourceGetPointer(
3239 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const {
3240#ifdef ASSERT
3241 // For now, the operand is an image. This will change once we start handling
3242 // more resource types.
3243 Register ResourcePtr = I.getOperand(2).getReg();
3244 SPIRVType *RegType = GR.getResultType(ResourcePtr);
3245 assert(RegType->getOpcode() == SPIRV::OpTypeImage &&
3246 "Can only handle texel buffers for now.");
3247#endif
3248
3249 // For texel buffers, the index into the image is part of the OpImageRead or
3250 // OpImageWrite instructions. So we will do nothing in this case. This
3251 // intrinsic will be combined with the load or store when selecting the load
3252 // or store.
3253 return true;
3254}
3255
3256bool SPIRVInstructionSelector::extractSubvector(
3257 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg,
3259 SPIRVType *InputType = GR.getResultType(ReadReg);
3260 [[maybe_unused]] uint64_t InputSize =
3261 GR.getScalarOrVectorComponentCount(InputType);
3262 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
3263 assert(InputSize > 1 && "The input must be a vector.");
3264 assert(ResultSize > 1 && "The result must be a vector.");
3265 assert(ResultSize < InputSize &&
3266 "Cannot extract more element than there are in the input.");
3267 SmallVector<Register> ComponentRegisters;
3268 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType);
3269 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType);
3270 for (uint64_t I = 0; I < ResultSize; I++) {
3271 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass);
3272 bool Succeed = BuildMI(*InsertionPoint.getParent(), InsertionPoint,
3273 InsertionPoint.getDebugLoc(),
3274 TII.get(SPIRV::OpCompositeExtract))
3275 .addDef(ComponentReg)
3276 .addUse(ScalarType->getOperand(0).getReg())
3277 .addUse(ReadReg)
3278 .addImm(I)
3279 .constrainAllUses(TII, TRI, RBI);
3280 if (!Succeed)
3281 return false;
3282 ComponentRegisters.emplace_back(ComponentReg);
3283 }
3284
3286 InsertionPoint.getDebugLoc(),
3287 TII.get(SPIRV::OpCompositeConstruct))
3288 .addDef(ResVReg)
3289 .addUse(GR.getSPIRVTypeID(ResType));
3290
3291 for (Register ComponentReg : ComponentRegisters)
3292 MIB.addUse(ComponentReg);
3293 return MIB.constrainAllUses(TII, TRI, RBI);
3294}
3295
3296bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
3297 MachineInstr &I) const {
3298 // If the load of the image is in a different basic block, then
3299 // this will generate invalid code. A proper solution is to move
3300 // the OpLoad from selectHandleFromBinding here. However, to do
3301 // that we will need to change the return type of the intrinsic.
3302 // We will do that when we can, but for now trying to move forward with other
3303 // issues.
3304 Register ImageReg = I.getOperand(1).getReg();
3305 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
3306 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
3307 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
3308 *ImageDef, I)) {
3309 return false;
3310 }
3311
3312 Register CoordinateReg = I.getOperand(2).getReg();
3313 Register DataReg = I.getOperand(3).getReg();
3314 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector);
3315 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4);
3316 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
3317 TII.get(SPIRV::OpImageWrite))
3318 .addUse(NewImageReg)
3319 .addUse(CoordinateReg)
3320 .addUse(DataReg)
3321 .constrainAllUses(TII, TRI, RBI);
3322}
3323
3324Register SPIRVInstructionSelector::buildPointerToResource(
3325 const SPIRVType *ResType, uint32_t Set, uint32_t Binding,
3326 uint32_t ArraySize, Register IndexReg, bool IsNonUniform,
3327 MachineIRBuilder MIRBuilder) const {
3328 if (ArraySize == 1)
3329 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding,
3330 MIRBuilder);
3331
3332 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType(
3333 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII);
3334 Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
3335 VarType, Set, Binding, MIRBuilder);
3336
3337 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType(
3338 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant);
3339
3340 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
3341 if (IsNonUniform) {
3342 // It is unclear which value needs to be marked an non-uniform, so both
3343 // the index and the access changed are decorated as non-uniform.
3344 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {});
3345 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {});
3346 }
3347
3348 MIRBuilder.buildInstr(SPIRV::OpAccessChain)
3349 .addDef(AcReg)
3350 .addUse(GR.getSPIRVTypeID(ResPointerType))
3351 .addUse(VarReg)
3352 .addUse(IndexReg);
3353
3354 return AcReg;
3355}
3356
3357bool SPIRVInstructionSelector::selectFirstBitSet16(
3358 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3359 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
3360 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
3361 bool Result = selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()},
3362 ExtendOpcode);
3363
3364 return Result &&
3365 selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode);
3366}
3367
3368bool SPIRVInstructionSelector::selectFirstBitSet32(
3369 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3370 Register SrcReg, unsigned BitSetOpcode) const {
3371 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
3372 .addDef(ResVReg)
3373 .addUse(GR.getSPIRVTypeID(ResType))
3374 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3375 .addImm(BitSetOpcode)
3376 .addUse(SrcReg)
3377 .constrainAllUses(TII, TRI, RBI);
3378}
3379
3380bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
3381 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3382 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const {
3383
3384 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors
3385 // requires creating a param register and return register with an invalid
3386 // vector size. If that is resolved, then this function can be used for
3387 // vectors of any component size.
3388 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
3389 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
3390
3391 MachineIRBuilder MIRBuilder(I);
3392 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType);
3393 SPIRVType *I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder);
3394 SPIRVType *I64x2Type = GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder);
3395 SPIRVType *Vec2ResType =
3396 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder);
3397
3398 std::vector<Register> PartialRegs;
3399
3400 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd
3401 unsigned CurrentComponent = 0;
3402 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
3403 // This register holds the firstbitX result for each of the i64x2 vectors
3404 // extracted from SrcReg
3405 Register BitSetResult =
3406 MRI->createVirtualRegister(GR.getRegClass(I64x2Type));
3407
3408 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
3409 TII.get(SPIRV::OpVectorShuffle))
3410 .addDef(BitSetResult)
3411 .addUse(GR.getSPIRVTypeID(I64x2Type))
3412 .addUse(SrcReg)
3413 .addUse(SrcReg)
3414 .addImm(CurrentComponent)
3415 .addImm(CurrentComponent + 1);
3416
3417 if (!MIB.constrainAllUses(TII, TRI, RBI))
3418 return false;
3419
3420 Register SubVecBitSetReg =
3421 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType));
3422
3423 if (!selectFirstBitSet64(SubVecBitSetReg, Vec2ResType, I, BitSetResult,
3424 BitSetOpcode, SwapPrimarySide))
3425 return false;
3426
3427 PartialRegs.push_back(SubVecBitSetReg);
3428 }
3429
3430 // On odd component counts we need to handle one more component
3431 if (CurrentComponent != ComponentCount) {
3432 bool ZeroAsNull = STI.isOpenCLEnv();
3433 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type));
3434 Register ConstIntLastIdx = GR.getOrCreateConstInt(
3435 ComponentCount - 1, I, BaseType, TII, ZeroAsNull);
3436
3437 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx},
3438 SPIRV::OpVectorExtractDynamic))
3439 return false;
3440
3441 Register FinalElemBitSetReg =
3442 MRI->createVirtualRegister(GR.getRegClass(BaseType));
3443
3444 if (!selectFirstBitSet64(FinalElemBitSetReg, BaseType, I, FinalElemReg,
3445 BitSetOpcode, SwapPrimarySide))
3446 return false;
3447
3448 PartialRegs.push_back(FinalElemBitSetReg);
3449 }
3450
3451 // Join all the resulting registers back into the return type in order
3452 // (ie i32x2, i32x2, i32x1 -> i32x5)
3453 return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs,
3454 SPIRV::OpCompositeConstruct);
3455}
3456
3457bool SPIRVInstructionSelector::selectFirstBitSet64(
3458 Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
3459 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const {
3460 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
3461 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType);
3462 bool ZeroAsNull = STI.isOpenCLEnv();
3463 Register ConstIntZero =
3464 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull);
3465 Register ConstIntOne =
3466 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull);
3467
3468 // SPIRV doesn't support vectors with more than 4 components. Since the
3469 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
3470 // operate on vectors with 2 or less components. When largers vectors are
3471 // seen. Split them, recurse, then recombine them.
3472 if (ComponentCount > 2) {
3473 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg,
3474 BitSetOpcode, SwapPrimarySide);
3475 }
3476
3477 // 1. Split int64 into 2 pieces using a bitcast
3478 MachineIRBuilder MIRBuilder(I);
3479 SPIRVType *PostCastType =
3480 GR.getOrCreateSPIRVVectorType(BaseType, 2 * ComponentCount, MIRBuilder);
3481 Register BitcastReg =
3482 MRI->createVirtualRegister(GR.getRegClass(PostCastType));
3483
3484 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg},
3485 SPIRV::OpBitcast))
3486 return false;
3487
3488 // 2. Find the first set bit from the primary side for all the pieces in #1
3489 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType));
3490 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode))
3491 return false;
3492
3493 // 3. Split result vector into high bits and low bits
3494 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
3495 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
3496
3497 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector;
3498 if (IsScalarRes) {
3499 // if scalar do a vector extract
3500 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntZero},
3501 SPIRV::OpVectorExtractDynamic))
3502 return false;
3503 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntOne},
3504 SPIRV::OpVectorExtractDynamic))
3505 return false;
3506 } else {
3507 // if vector do a shufflevector
3508 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
3509 TII.get(SPIRV::OpVectorShuffle))
3510 .addDef(HighReg)
3511 .addUse(GR.getSPIRVTypeID(ResType))
3512 .addUse(FBSReg)
3513 // Per the spec, repeat the vector if only one vec is needed
3514 .addUse(FBSReg);
3515
3516 // high bits are stored in even indexes. Extract them from FBSReg
3517 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
3518 MIB.addImm(J);
3519 }
3520
3521 if (!MIB.constrainAllUses(TII, TRI, RBI))
3522 return false;
3523
3524 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
3525 TII.get(SPIRV::OpVectorShuffle))
3526 .addDef(LowReg)
3527 .addUse(GR.getSPIRVTypeID(ResType))
3528 .addUse(FBSReg)
3529 // Per the spec, repeat the vector if only one vec is needed
3530 .addUse(FBSReg);
3531
3532 // low bits are stored in odd indexes. Extract them from FBSReg
3533 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
3534 MIB.addImm(J);
3535 }
3536 if (!MIB.constrainAllUses(TII, TRI, RBI))
3537 return false;
3538 }
3539
3540 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
3541 // primary
3542 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
3543 Register NegOneReg;
3544 Register Reg0;
3545 Register Reg32;
3546 unsigned SelectOp;
3547 unsigned AddOp;
3548
3549 if (IsScalarRes) {
3550 NegOneReg =
3551 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull);
3552 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
3553 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull);
3554 SelectOp = SPIRV::OpSelectSISCond;
3555 AddOp = SPIRV::OpIAddS;
3556 } else {
3557 BoolType =
3558 GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount, MIRBuilder);
3559 NegOneReg =
3560 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull);
3561 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull);
3562 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull);
3563 SelectOp = SPIRV::OpSelectVIVCond;
3564 AddOp = SPIRV::OpIAddV;
3565 }
3566
3567 Register PrimaryReg = HighReg;
3568 Register SecondaryReg = LowReg;
3569 Register PrimaryShiftReg = Reg32;
3570 Register SecondaryShiftReg = Reg0;
3571
3572 // By default the emitted opcodes check for the set bit from the MSB side.
3573 // Setting SwapPrimarySide checks the set bit from the LSB side
3574 if (SwapPrimarySide) {
3575 PrimaryReg = LowReg;
3576 SecondaryReg = HighReg;
3577 PrimaryShiftReg = Reg0;
3578 SecondaryShiftReg = Reg32;
3579 }
3580
3581 // Check if the primary bits are == -1
3582 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType));
3583 if (!selectOpWithSrcs(BReg, BoolType, I, {PrimaryReg, NegOneReg},
3584 SPIRV::OpIEqual))
3585 return false;
3586
3587 // Select secondary bits if true in BReg, otherwise primary bits
3588 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
3589 if (!selectOpWithSrcs(TmpReg, ResType, I, {BReg, SecondaryReg, PrimaryReg},
3590 SelectOp))
3591 return false;
3592
3593 // 5. Add 32 when high bits are used, otherwise 0 for low bits
3594 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
3595 if (!selectOpWithSrcs(ValReg, ResType, I,
3596 {BReg, SecondaryShiftReg, PrimaryShiftReg}, SelectOp))
3597 return false;
3598
3599 return selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, AddOp);
3600}
3601
3602bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
3603 const SPIRVType *ResType,
3604 MachineInstr &I,
3605 bool IsSigned) const {
3606 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
3607 Register OpReg = I.getOperand(2).getReg();
3608 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg);
3609 // zero or sign extend
3610 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3611 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
3612
3613 switch (GR.getScalarOrVectorBitWidth(OpType)) {
3614 case 16:
3615 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
3616 case 32:
3617 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
3618 case 64:
3619 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
3620 /*SwapPrimarySide=*/false);
3621 default:
3623 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
3624 }
3625}
3626
3627bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
3628 const SPIRVType *ResType,
3629 MachineInstr &I) const {
3630 // FindILsb intrinsic only supports 32 bit integers
3631 Register OpReg = I.getOperand(2).getReg();
3632 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg);
3633 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
3634 // to an unsigned i32. As this leaves all the least significant bits unchanged
3635 // so the first set bit from the LSB side doesn't change.
3636 unsigned ExtendOpcode = SPIRV::OpUConvert;
3637 unsigned BitSetOpcode = GL::FindILsb;
3638
3639 switch (GR.getScalarOrVectorBitWidth(OpType)) {
3640 case 16:
3641 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
3642 case 32:
3643 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
3644 case 64:
3645 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
3646 /*SwapPrimarySide=*/true);
3647 default:
3648 report_fatal_error("spv_firstbitlow only supports 16,32,64 bits.");
3649 }
3650}
3651
3652bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
3653 const SPIRVType *ResType,
3654 MachineInstr &I) const {
3655 // there was an allocation size parameter to the allocation instruction
3656 // that is not 1
3657 MachineBasicBlock &BB = *I.getParent();
3658 bool Res = BuildMI(BB, I, I.getDebugLoc(),
3659 TII.get(SPIRV::OpVariableLengthArrayINTEL))
3660 .addDef(ResVReg)
3661 .addUse(GR.getSPIRVTypeID(ResType))
3662 .addUse(I.getOperand(2).getReg())
3663 .constrainAllUses(TII, TRI, RBI);
3664 if (!STI.isVulkanEnv()) {
3665 unsigned Alignment = I.getOperand(3).getImm();
3666 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment});
3667 }
3668 return Res;
3669}
3670
3671bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
3672 const SPIRVType *ResType,
3673 MachineInstr &I) const {
3674 // Change order of instructions if needed: all OpVariable instructions in a
3675 // function must be the first instructions in the first block
3676 auto It = getOpVariableMBBIt(I);
3677 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(),
3678 TII.get(SPIRV::OpVariable))
3679 .addDef(ResVReg)
3680 .addUse(GR.getSPIRVTypeID(ResType))
3681 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
3682 .constrainAllUses(TII, TRI, RBI);
3683 if (!STI.isVulkanEnv()) {
3684 unsigned Alignment = I.getOperand(2).getImm();
3685 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment,
3686 {Alignment});
3687 }
3688 return Res;
3689}
3690
3691bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
3692 // InstructionSelector walks backwards through the instructions. We can use
3693 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
3694 // first, so can generate an OpBranchConditional here. If there is no
3695 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
3696 const MachineInstr *PrevI = I.getPrevNode();
3697 MachineBasicBlock &MBB = *I.getParent();
3698 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
3699 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
3700 .addUse(PrevI->getOperand(0).getReg())
3701 .addMBB(PrevI->getOperand(1).getMBB())
3702 .addMBB(I.getOperand(0).getMBB())
3703 .constrainAllUses(TII, TRI, RBI);
3704 }
3705 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
3706 .addMBB(I.getOperand(0).getMBB())
3707 .constrainAllUses(TII, TRI, RBI);
3708}
3709
3710bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
3711 // InstructionSelector walks backwards through the instructions. For an
3712 // explicit conditional branch with no fallthrough, we use both a G_BR and a
3713 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
3714 // generate the OpBranchConditional in selectBranch above.
3715 //
3716 // If an OpBranchConditional has been generated, we simply return, as the work
3717 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
3718 // implicit fallthrough to the next basic block, so we need to create an
3719 // OpBranchConditional with an explicit "false" argument pointing to the next
3720 // basic block that LLVM would fall through to.
3721 const MachineInstr *NextI = I.getNextNode();
3722 // Check if this has already been successfully selected.
3723 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
3724 return true;
3725 // Must be relying on implicit block fallthrough, so generate an
3726 // OpBranchConditional with the "next" basic block as the "false" target.
3727 MachineBasicBlock &MBB = *I.getParent();
3728 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
3729 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
3730 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
3731 .addUse(I.getOperand(0).getReg())
3732 .addMBB(I.getOperand(1).getMBB())
3733 .addMBB(NextMBB)
3734 .constrainAllUses(TII, TRI, RBI);
3735}
3736
3737bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
3738 const SPIRVType *ResType,
3739 MachineInstr &I) const {
3740 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi))
3741 .addDef(ResVReg)
3742 .addUse(GR.getSPIRVTypeID(ResType));
3743 const unsigned NumOps = I.getNumOperands();
3744 for (unsigned i = 1; i < NumOps; i += 2) {
3745 MIB.addUse(I.getOperand(i + 0).getReg());
3746 MIB.addMBB(I.getOperand(i + 1).getMBB());
3747 }
3748 bool Res = MIB.constrainAllUses(TII, TRI, RBI);
3749 MIB->setDesc(TII.get(TargetOpcode::PHI));
3750 MIB->removeOperand(1);
3751 return Res;
3752}
3753
3754bool SPIRVInstructionSelector::selectGlobalValue(
3755 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
3756 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
3757 MachineIRBuilder MIRBuilder(I);
3758 const GlobalValue *GV = I.getOperand(1).getGlobal();
3759 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV));
3760 SPIRVType *PointerBaseType;
3761 if (GVType->isArrayTy()) {
3762 SPIRVType *ArrayElementType =
3763 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder,
3764 SPIRV::AccessQualifier::ReadWrite, false);
3765 PointerBaseType = GR.getOrCreateSPIRVArrayType(
3766 ArrayElementType, GVType->getArrayNumElements(), I, TII);
3767 } else {
3768 PointerBaseType = GR.getOrCreateSPIRVType(
3769 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
3770 }
3771
3772 std::string GlobalIdent;
3773 if (!GV->hasName()) {
3774 unsigned &ID = UnnamedGlobalIDs[GV];
3775 if (ID == 0)
3776 ID = UnnamedGlobalIDs.size();
3777 GlobalIdent = "__unnamed_" + Twine(ID).str();
3778 } else {
3779 GlobalIdent = GV->getName();
3780 }
3781
3782 // Behaviour of functions as operands depends on availability of the
3783 // corresponding extension (SPV_INTEL_function_pointers):
3784 // - If there is an extension to operate with functions as operands:
3785 // We create a proper constant operand and evaluate a correct type for a
3786 // function pointer.
3787 // - Without the required extension:
3788 // We have functions as operands in tests with blocks of instruction e.g. in
3789 // transcoding/global_block.ll. These operands are not used and should be
3790 // substituted by zero constants. Their type is expected to be always
3791 // OpTypePointer Function %uchar.
3792 if (isa<Function>(GV)) {
3793 const Constant *ConstVal = GV;
3794 MachineBasicBlock &BB = *I.getParent();
3795 Register NewReg = GR.find(ConstVal, GR.CurMF);
3796 if (!NewReg.isValid()) {
3797 Register NewReg = ResVReg;
3798 GR.add(ConstVal, GR.CurMF, NewReg);
3799 const Function *GVFun =
3800 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)
3801 ? dyn_cast<Function>(GV)
3802 : nullptr;
3803 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
3804 PointerBaseType, I, TII,
3805 GVFun ? SPIRV::StorageClass::CodeSectionINTEL
3807 if (GVFun) {
3808 // References to a function via function pointers generate virtual
3809 // registers without a definition. We will resolve it later, during
3810 // module analysis stage.
3811 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
3812 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3813 Register FuncVReg =
3814 MRI->createGenericVirtualRegister(GR.getRegType(ResType));
3815 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass);
3816 MachineInstrBuilder MIB1 =
3817 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
3818 .addDef(FuncVReg)
3819 .addUse(ResTypeReg);
3820 MachineInstrBuilder MIB2 =
3821 BuildMI(BB, I, I.getDebugLoc(),
3822 TII.get(SPIRV::OpConstantFunctionPointerINTEL))
3823 .addDef(NewReg)
3824 .addUse(ResTypeReg)
3825 .addUse(FuncVReg);
3826 // mapping the function pointer to the used Function
3827 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun);
3828 return MIB1.constrainAllUses(TII, TRI, RBI) &&
3829 MIB2.constrainAllUses(TII, TRI, RBI);
3830 }
3831 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
3832 .addDef(NewReg)
3833 .addUse(GR.getSPIRVTypeID(ResType))
3834 .constrainAllUses(TII, TRI, RBI);
3835 }
3836 assert(NewReg != ResVReg);
3837 return BuildCOPY(ResVReg, NewReg, I);
3838 }
3839 auto GlobalVar = cast<GlobalVariable>(GV);
3840 assert(GlobalVar->getName() != "llvm.global.annotations");
3841
3842 // Skip empty declaration for GVs with initializers till we get the decl with
3843 // passed initializer.
3844 if (hasInitializer(GlobalVar) && !Init)
3845 return true;
3846
3847 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage();
3848 SPIRV::LinkageType::LinkageType LnkType =
3850 ? SPIRV::LinkageType::Import
3851 : (GV->hasLinkOnceODRLinkage() &&
3852 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr)
3853 ? SPIRV::LinkageType::LinkOnceODR
3854 : SPIRV::LinkageType::Export);
3855
3856 const unsigned AddrSpace = GV->getAddressSpace();
3857 SPIRV::StorageClass::StorageClass StorageClass =
3858 addressSpaceToStorageClass(AddrSpace, STI);
3859 SPIRVType *ResType =
3860 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass);
3861 Register Reg = GR.buildGlobalVariable(
3862 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init,
3863 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true);
3864 return Reg.isValid();
3865}
3866
3867bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
3868 const SPIRVType *ResType,
3869 MachineInstr &I) const {
3870 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
3871 return selectExtInst(ResVReg, ResType, I, CL::log10);
3872 }
3873
3874 // There is no log10 instruction in the GLSL Extended Instruction set, so it
3875 // is implemented as:
3876 // log10(x) = log2(x) * (1 / log2(10))
3877 // = log2(x) * 0.30103
3878
3879 MachineIRBuilder MIRBuilder(I);
3880 MachineBasicBlock &BB = *I.getParent();
3881
3882 // Build log2(x).
3883 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
3884 bool Result =
3885 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
3886 .addDef(VarReg)
3887 .addUse(GR.getSPIRVTypeID(ResType))
3888 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3889 .addImm(GL::Log2)
3890 .add(I.getOperand(1))
3891 .constrainAllUses(TII, TRI, RBI);
3892
3893 // Build 0.30103.
3894 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
3895 ResType->getOpcode() == SPIRV::OpTypeFloat);
3896 // TODO: Add matrix implementation once supported by the HLSL frontend.
3897 const SPIRVType *SpirvScalarType =
3898 ResType->getOpcode() == SPIRV::OpTypeVector
3899 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg())
3900 : ResType;
3901 Register ScaleReg =
3902 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType);
3903
3904 // Multiply log2(x) by 0.30103 to get log10(x) result.
3905 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
3906 ? SPIRV::OpVectorTimesScalar
3907 : SPIRV::OpFMulS;
3908 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3909 .addDef(ResVReg)
3910 .addUse(GR.getSPIRVTypeID(ResType))
3911 .addUse(VarReg)
3912 .addUse(ScaleReg)
3913 .constrainAllUses(TII, TRI, RBI);
3914}
3915
3916// Generate the instructions to load 3-element vector builtin input
3917// IDs/Indices.
3918// Like: GlobalInvocationId, LocalInvocationId, etc....
3919bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
3920 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
3921 const SPIRVType *ResType, MachineInstr &I) const {
3922 MachineIRBuilder MIRBuilder(I);
3923 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
3924 const SPIRVType *Vec3Ty =
3925 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder);
3926 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
3927 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
3928
3929 // Create new register for the input ID builtin variable.
3930 Register NewRegister =
3931 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
3932 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64));
3933 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
3934
3935 // Build global variable with the necessary decorations for the input ID
3936 // builtin variable.
3937 Register Variable = GR.buildGlobalVariable(
3938 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
3939 SPIRV::StorageClass::Input, nullptr, true, true,
3940 SPIRV::LinkageType::Import, MIRBuilder, false);
3941
3942 // Create new register for loading value.
3943 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3944 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
3945 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64));
3946 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF());
3947
3948 // Load v3uint value from the global variable.
3949 bool Result =
3950 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
3951 .addDef(LoadedRegister)
3952 .addUse(GR.getSPIRVTypeID(Vec3Ty))
3953 .addUse(Variable);
3954
3955 // Get the input ID index. Expecting operand is a constant immediate value,
3956 // wrapped in a type assignment.
3957 assert(I.getOperand(2).isReg());
3958 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI);
3959
3960 // Extract the input ID from the loaded vector value.
3961 MachineBasicBlock &BB = *I.getParent();
3962 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
3963 .addDef(ResVReg)
3964 .addUse(GR.getSPIRVTypeID(ResType))
3965 .addUse(LoadedRegister)
3966 .addImm(ThreadId);
3967 return Result && MIB.constrainAllUses(TII, TRI, RBI);
3968}
3969
3970SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type,
3971 MachineInstr &I) const {
3972 MachineIRBuilder MIRBuilder(I);
3973 if (Type->getOpcode() != SPIRV::OpTypeVector)
3974 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder);
3975
3976 uint64_t VectorSize = Type->getOperand(2).getImm();
3977 if (VectorSize == 4)
3978 return Type;
3979
3980 Register ScalarTypeReg = Type->getOperand(1).getReg();
3981 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg);
3982 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder);
3983}
3984
3985bool SPIRVInstructionSelector::loadHandleBeforePosition(
3986 Register &HandleReg, const SPIRVType *ResType, GIntrinsic &HandleDef,
3987 MachineInstr &Pos) const {
3988
3989 assert(HandleDef.getIntrinsicID() ==
3990 Intrinsic::spv_resource_handlefrombinding);
3991 uint32_t Set = foldImm(HandleDef.getOperand(2), MRI);
3992 uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI);
3993 uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI);
3994 Register IndexReg = HandleDef.getOperand(5).getReg();
3995 bool IsNonUniform = ArraySize > 1 && foldImm(HandleDef.getOperand(6), MRI);
3996
3997 MachineIRBuilder MIRBuilder(HandleDef);
3998 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize,
3999 IndexReg, IsNonUniform, MIRBuilder);
4000
4001 if (IsNonUniform)
4002 buildOpDecorate(HandleReg, HandleDef, TII, SPIRV::Decoration::NonUniformEXT,
4003 {});
4004
4005 // TODO: For now we assume the resource is an image, which needs to be
4006 // loaded to get the handle. That will not be true for storage buffers.
4007 return BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(),
4008 TII.get(SPIRV::OpLoad))
4009 .addDef(HandleReg)
4010 .addUse(GR.getSPIRVTypeID(ResType))
4011 .addUse(VarReg)
4012 .constrainAllUses(TII, TRI, RBI);
4013}
4014
4015namespace llvm {
4018 const SPIRVSubtarget &Subtarget,
4019 const RegisterBankInfo &RBI) {
4020 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
4021}
4022} // namespace llvm
unsigned const MachineRegisterInfo * MRI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
basic Basic Alias true
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(...)
Definition: Debug.h:106
uint64_t Size
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
uint64_t IntrinsicInst * II
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::vector< std::pair< SPIRV::InstructionSet::InstructionSet, uint32_t > > ExtInstList
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static APFloat getOneFP(const Type *LLVMFloatTy)
static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC)
static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg)
static bool mayApplyGenericSelection(unsigned Opcode)
static APFloat getZeroFP(const Type *LLVMFloatTy)
static void addMemoryOperands(MachineMemOperand *MemOp, MachineInstrBuilder &MIB)
static unsigned getFCmpOpcode(unsigned PredNum)
bool isTypeFoldingSupported(unsigned Opcode)
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static unsigned getBoolCmpOpcode(unsigned PredNum)
static unsigned getICmpOpcode(unsigned PredNum)
static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, SmallPtrSet< SPIRVType *, 4 > &Visited)
static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)
static unsigned getPtrCmpOpcode(unsigned Pred)
bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, const SPIRVType *ResType)
raw_pwrite_stream & OS
This file contains some functions that are useful when dealing with strings.
BinaryOperator * Mul
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition: APFloat.h:1090
APInt bitcastToAPInt() const
Definition: APFloat.h:1351
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition: APFloat.h:1081
Class for arbitrary precision integers.
Definition: APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:234
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1520
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:673
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:676
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:702
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:703
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:679
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:688
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:677
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:678
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:697
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:696
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:700
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:687
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:681
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:684
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:698
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:685
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:680
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:682
@ ICMP_EQ
equal
Definition: InstrTypes.h:694
@ ICMP_NE
not equal
Definition: InstrTypes.h:695
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:701
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:689
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:699
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:686
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:683
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:271
const APFloat & getValueAPF() const
Definition: Constants.h:314
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:157
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:148
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:373
This class represents an Operation in the Expression.
A debug info location.
Definition: DebugLoc.h:33
const Function & getFunction() const
Definition: Function.h:171
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:369
Represents a call to an intrinsic.
Intrinsic::ID getIntrinsicID() const
bool hasPrivateLinkage() const
Definition: GlobalValue.h:528
bool isDeclarationForLinker() const
Definition: GlobalValue.h:619
unsigned getAddressSpace() const
Definition: GlobalValue.h:206
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:657
bool hasInternalLinkage() const
Definition: GlobalValue.h:527
bool hasLinkOnceODRLinkage() const
Definition: GlobalValue.h:520
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:311
constexpr bool isScalar() const
Definition: LowLevelType.h:146
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr bool isVector() const
Definition: LowLevelType.h:148
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:57
constexpr bool isPointer() const
Definition: LowLevelType.h:149
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelType.h:100
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Helper class to build MachineInstr.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:71
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:577
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:349
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:501
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:587
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
Register getReg() const
getReg - Returns the register number.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
Analysis providing profile information.
Holds all the information related to register banks.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
Definition: Register.h:84
constexpr bool isValid() const
Definition: Register.h:115
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:384
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:458
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:519
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:937
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:150
Class to represent struct types.
Definition: DerivedTypes.h:218
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:406
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isArrayTy() const
True if this is an instance of ArrayType.
Definition: Type.h:261
Type * getArrayElementType() const
Definition: Type.h:411
uint64_t getArrayNumElements() const
@ HalfTyID
16-bit floating point type
Definition: Type.h:56
@ FloatTyID
32-bit floating point type
Definition: Type.h:58
@ DoubleTyID
64-bit floating point type
Definition: Type.h:59
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:258
TypeID getTypeID() const
Return the type id for the type.
Definition: Type.h:136
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:355
bool hasName() const
Definition: Value.h:261
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Represents a version number in the form major[.minor[.subminor[.build]]].
Definition: VersionTuple.h:29
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:353
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsConst[]
Key for Kernel::Arg::Metadata::mIsConst.
Reg
All possible values of the reg field in the ModR/M byte.
StorageClass
Definition: XCOFF.h:170
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void buildOpName(Register Target, const StringRef &Name, MachineIRBuilder &MIRBuilder)
Definition: SPIRVUtils.cpp:103
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
Definition: SPIRVUtils.cpp:83
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1683
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:155
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
Definition: TargetOpcodes.h:30
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
Definition: SPIRVUtils.cpp:326
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
Definition: SPIRVUtils.cpp:245
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
Definition: SPIRVUtils.h:163
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)
Definition: SPIRVUtils.cpp:130
MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I)
Definition: SPIRVUtils.cpp:177
Type * toTypedPointer(Type *Ty)
Definition: SPIRVUtils.h:353
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)
Definition: SPIRVUtils.cpp:211
AtomicOrdering
Atomic ordering for LLVM's memory model.
SPIRV::Scope::Scope getMemScope(LLVMContext &Ctx, SyncScope::ID Id)
Definition: SPIRVUtils.cpp:281
InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &Subtarget, const RegisterBankInfo &RBI)
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:217
bool hasInitializer(const GlobalVariable *GV)
Definition: SPIRVUtils.h:241
MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)
Definition: SPIRVUtils.cpp:704
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
Definition: SPIRVUtils.cpp:263
std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)
bool isTriviallyDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Check whether an instruction MI is dead: it only defines dead virtual registers, and doesn't have oth...
Definition: Utils.cpp:222
#define N