LLVM 18.0.0git
SPIRVInstructionSelector.cpp
Go to the documentation of this file.
1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
21#include "SPIRVRegisterInfo.h"
22#include "SPIRVTargetMachine.h"
23#include "SPIRVUtils.h"
24#include "llvm/ADT/APFloat.h"
30#include "llvm/IR/IntrinsicsSPIRV.h"
31#include "llvm/Support/Debug.h"
32
33#define DEBUG_TYPE "spirv-isel"
34
35using namespace llvm;
36namespace CL = SPIRV::OpenCLExtInst;
37namespace GL = SPIRV::GLSLExtInst;
38
40 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
41
42namespace {
43
44#define GET_GLOBALISEL_PREDICATE_BITSET
45#include "SPIRVGenGlobalISel.inc"
46#undef GET_GLOBALISEL_PREDICATE_BITSET
47
48class SPIRVInstructionSelector : public InstructionSelector {
49 const SPIRVSubtarget &STI;
50 const SPIRVInstrInfo &TII;
52 const RegisterBankInfo &RBI;
55
56public:
57 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
58 const SPIRVSubtarget &ST,
59 const RegisterBankInfo &RBI);
60 void setupMF(MachineFunction &MF, GISelKnownBits *KB,
61 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
62 BlockFrequencyInfo *BFI) override;
63 // Common selection code. Instruction-specific selection occurs in spvSelect.
64 bool select(MachineInstr &I) override;
65 static const char *getName() { return DEBUG_TYPE; }
66
67#define GET_GLOBALISEL_PREDICATES_DECL
68#include "SPIRVGenGlobalISel.inc"
69#undef GET_GLOBALISEL_PREDICATES_DECL
70
71#define GET_GLOBALISEL_TEMPORARIES_DECL
72#include "SPIRVGenGlobalISel.inc"
73#undef GET_GLOBALISEL_TEMPORARIES_DECL
74
75private:
76 // tblgen-erated 'select' implementation, used as the initial selector for
77 // the patterns that don't require complex C++.
78 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
79
80 // All instruction-specific selection that didn't happen in "select()".
81 // Is basically a large Switch/Case delegating to all other select method.
82 bool spvSelect(Register ResVReg, const SPIRVType *ResType,
83 MachineInstr &I) const;
84
85 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
86 const MachineInstr *Init = nullptr) const;
87
88 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType,
89 MachineInstr &I, Register SrcReg,
90 unsigned Opcode) const;
91 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
92 unsigned Opcode) const;
93
94 bool selectLoad(Register ResVReg, const SPIRVType *ResType,
95 MachineInstr &I) const;
96 bool selectStore(MachineInstr &I) const;
97
98 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
99
100 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType,
101 MachineInstr &I, unsigned NewOpcode) const;
102
103 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType,
104 MachineInstr &I) const;
105
106 bool selectFence(MachineInstr &I) const;
107
108 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
109 MachineInstr &I) const;
110
111 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
112 MachineInstr &I) const;
113
114 bool selectConstVector(Register ResVReg, const SPIRVType *ResType,
115 MachineInstr &I) const;
116
117 bool selectCmp(Register ResVReg, const SPIRVType *ResType,
118 unsigned comparisonOpcode, MachineInstr &I) const;
119
120 bool selectICmp(Register ResVReg, const SPIRVType *ResType,
121 MachineInstr &I) const;
122 bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
123 MachineInstr &I) const;
124
125 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
126 int OpIdx) const;
127 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
128 int OpIdx) const;
129
130 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm,
131 MachineInstr &I) const;
132
133 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
134 bool IsSigned) const;
135 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
136 bool IsSigned, unsigned Opcode) const;
137 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
138 bool IsSigned) const;
139
140 bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
141 MachineInstr &I) const;
142
143 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
144 const SPIRVType *intTy, const SPIRVType *boolTy) const;
145
146 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
147 MachineInstr &I) const;
148 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType,
149 MachineInstr &I) const;
150 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType,
151 MachineInstr &I) const;
152 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType,
153 MachineInstr &I) const;
154 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType,
155 MachineInstr &I) const;
156 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType,
157 MachineInstr &I) const;
158 bool selectGEP(Register ResVReg, const SPIRVType *ResType,
159 MachineInstr &I) const;
160
161 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType,
162 MachineInstr &I) const;
163
164 bool selectBranch(MachineInstr &I) const;
165 bool selectBranchCond(MachineInstr &I) const;
166
167 bool selectPhi(Register ResVReg, const SPIRVType *ResType,
168 MachineInstr &I) const;
169
170 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
171 MachineInstr &I, CL::OpenCLExtInst CLInst) const;
172 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
173 MachineInstr &I, CL::OpenCLExtInst CLInst,
174 GL::GLSLExtInst GLInst) const;
175 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
176 MachineInstr &I, const ExtInstList &ExtInsts) const;
177
178 bool selectLog10(Register ResVReg, const SPIRVType *ResType,
179 MachineInstr &I) const;
180
181 Register buildI32Constant(uint32_t Val, MachineInstr &I,
182 const SPIRVType *ResType = nullptr) const;
183
184 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
185 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
186 MachineInstr &I) const;
187};
188
189} // end anonymous namespace
190
191#define GET_GLOBALISEL_IMPL
192#include "SPIRVGenGlobalISel.inc"
193#undef GET_GLOBALISEL_IMPL
194
195SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
196 const SPIRVSubtarget &ST,
197 const RegisterBankInfo &RBI)
198 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
199 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
201#include "SPIRVGenGlobalISel.inc"
204#include "SPIRVGenGlobalISel.inc"
206{
207}
208
209void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
210 CodeGenCoverage *CoverageInfo,
212 BlockFrequencyInfo *BFI) {
213 MRI = &MF.getRegInfo();
214 GR.setCurrentFunc(MF);
215 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
216}
217
218static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI);
219
220// Defined in SPIRVLegalizerInfo.cpp.
221extern bool isTypeFoldingSupported(unsigned Opcode);
222
223bool SPIRVInstructionSelector::select(MachineInstr &I) {
224 assert(I.getParent() && "Instruction should be in a basic block!");
225 assert(I.getParent()->getParent() && "Instruction should be in a function!");
226
227 Register Opcode = I.getOpcode();
228 // If it's not a GMIR instruction, we've selected it already.
230 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
231 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg());
232 if (isTypeFoldingSupported(Def->getOpcode())) {
233 auto Res = selectImpl(I, *CoverageInfo);
234 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
235 if (Res)
236 return Res;
237 }
238 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg());
239 I.removeFromParent();
240 return true;
241 } else if (I.getNumDefs() == 1) {
242 // Make all vregs 32 bits (for SPIR-V IDs).
243 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32));
244 }
246 }
247
248 if (I.getNumOperands() != I.getNumExplicitOperands()) {
249 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
250 return false;
251 }
252
253 // Common code for getting return reg+type, and removing selected instr
254 // from parent occurs here. Instr-specific selection happens in spvSelect().
255 bool HasDefs = I.getNumDefs() > 0;
256 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
257 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
258 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
259 if (spvSelect(ResVReg, ResType, I)) {
260 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs).
261 MRI->setType(ResVReg, LLT::scalar(32));
262 I.removeFromParent();
263 return true;
264 }
265 return false;
266}
267
268bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
269 const SPIRVType *ResType,
270 MachineInstr &I) const {
271 assert(!isTypeFoldingSupported(I.getOpcode()) ||
272 I.getOpcode() == TargetOpcode::G_CONSTANT);
273 const unsigned Opcode = I.getOpcode();
274 switch (Opcode) {
275 case TargetOpcode::G_CONSTANT:
276 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(),
277 I);
278 case TargetOpcode::G_GLOBAL_VALUE:
279 return selectGlobalValue(ResVReg, I);
280 case TargetOpcode::G_IMPLICIT_DEF:
281 return selectOpUndef(ResVReg, ResType, I);
282
283 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
284 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
285 return selectIntrinsic(ResVReg, ResType, I);
286 case TargetOpcode::G_BITREVERSE:
287 return selectBitreverse(ResVReg, ResType, I);
288
289 case TargetOpcode::G_BUILD_VECTOR:
290 return selectConstVector(ResVReg, ResType, I);
291
292 case TargetOpcode::G_SHUFFLE_VECTOR: {
293 MachineBasicBlock &BB = *I.getParent();
294 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
295 .addDef(ResVReg)
296 .addUse(GR.getSPIRVTypeID(ResType))
297 .addUse(I.getOperand(1).getReg())
298 .addUse(I.getOperand(2).getReg());
299 for (auto V : I.getOperand(3).getShuffleMask())
300 MIB.addImm(V);
301 return MIB.constrainAllUses(TII, TRI, RBI);
302 }
303 case TargetOpcode::G_MEMMOVE:
304 case TargetOpcode::G_MEMCPY:
305 case TargetOpcode::G_MEMSET:
306 return selectMemOperation(ResVReg, I);
307
308 case TargetOpcode::G_ICMP:
309 return selectICmp(ResVReg, ResType, I);
310 case TargetOpcode::G_FCMP:
311 return selectFCmp(ResVReg, ResType, I);
312
313 case TargetOpcode::G_FRAME_INDEX:
314 return selectFrameIndex(ResVReg, ResType, I);
315
316 case TargetOpcode::G_LOAD:
317 return selectLoad(ResVReg, ResType, I);
318 case TargetOpcode::G_STORE:
319 return selectStore(I);
320
321 case TargetOpcode::G_BR:
322 return selectBranch(I);
323 case TargetOpcode::G_BRCOND:
324 return selectBranchCond(I);
325
326 case TargetOpcode::G_PHI:
327 return selectPhi(ResVReg, ResType, I);
328
329 case TargetOpcode::G_FPTOSI:
330 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
331 case TargetOpcode::G_FPTOUI:
332 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
333
334 case TargetOpcode::G_SITOFP:
335 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
336 case TargetOpcode::G_UITOFP:
337 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
338
339 case TargetOpcode::G_CTPOP:
340 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount);
341 case TargetOpcode::G_SMIN:
342 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
343 case TargetOpcode::G_UMIN:
344 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
345
346 case TargetOpcode::G_SMAX:
347 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
348 case TargetOpcode::G_UMAX:
349 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
350
351 case TargetOpcode::G_FMA:
352 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
353
354 case TargetOpcode::G_FPOW:
355 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
356 case TargetOpcode::G_FPOWI:
357 return selectExtInst(ResVReg, ResType, I, CL::pown);
358
359 case TargetOpcode::G_FEXP:
360 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
361 case TargetOpcode::G_FEXP2:
362 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
363
364 case TargetOpcode::G_FLOG:
365 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
366 case TargetOpcode::G_FLOG2:
367 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
368 case TargetOpcode::G_FLOG10:
369 return selectLog10(ResVReg, ResType, I);
370
371 case TargetOpcode::G_FABS:
372 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
373 case TargetOpcode::G_ABS:
374 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
375
376 case TargetOpcode::G_FMINNUM:
377 case TargetOpcode::G_FMINIMUM:
378 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::FMin);
379 case TargetOpcode::G_FMAXNUM:
380 case TargetOpcode::G_FMAXIMUM:
381 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::FMax);
382
383 case TargetOpcode::G_FCOPYSIGN:
384 return selectExtInst(ResVReg, ResType, I, CL::copysign);
385
386 case TargetOpcode::G_FCEIL:
387 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
388 case TargetOpcode::G_FFLOOR:
389 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
390
391 case TargetOpcode::G_FCOS:
392 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
393 case TargetOpcode::G_FSIN:
394 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
395
396 case TargetOpcode::G_FSQRT:
397 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
398
399 case TargetOpcode::G_CTTZ:
400 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
401 return selectExtInst(ResVReg, ResType, I, CL::ctz);
402 case TargetOpcode::G_CTLZ:
403 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
404 return selectExtInst(ResVReg, ResType, I, CL::clz);
405
406 case TargetOpcode::G_INTRINSIC_ROUND:
407 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
408 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
409 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
410 case TargetOpcode::G_INTRINSIC_TRUNC:
411 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
412 case TargetOpcode::G_FRINT:
413 case TargetOpcode::G_FNEARBYINT:
414 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
415
416 case TargetOpcode::G_SMULH:
417 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
418 case TargetOpcode::G_UMULH:
419 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
420
421 case TargetOpcode::G_SEXT:
422 return selectExt(ResVReg, ResType, I, true);
423 case TargetOpcode::G_ANYEXT:
424 case TargetOpcode::G_ZEXT:
425 return selectExt(ResVReg, ResType, I, false);
426 case TargetOpcode::G_TRUNC:
427 return selectTrunc(ResVReg, ResType, I);
428 case TargetOpcode::G_FPTRUNC:
429 case TargetOpcode::G_FPEXT:
430 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
431
432 case TargetOpcode::G_PTRTOINT:
433 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
434 case TargetOpcode::G_INTTOPTR:
435 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
436 case TargetOpcode::G_BITCAST:
437 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
438 case TargetOpcode::G_ADDRSPACE_CAST:
439 return selectAddrSpaceCast(ResVReg, ResType, I);
440 case TargetOpcode::G_PTR_ADD: {
441 // Currently, we get G_PTR_ADD only as a result of translating
442 // global variables, initialized with constant expressions like GV + Const
443 // (see test opencl/basic/progvar_prog_scope_init.ll).
444 // TODO: extend the handler once we have other cases.
445 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
446 Register GV = I.getOperand(1).getReg();
447 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
448 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
449 (*II).getOpcode() == TargetOpcode::COPY ||
450 (*II).getOpcode() == SPIRV::OpVariable) &&
451 isImm(I.getOperand(2), MRI));
452 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
453 MachineBasicBlock &BB = *I.getParent();
454 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
455 .addDef(ResVReg)
456 .addUse(GR.getSPIRVTypeID(ResType))
457 .addImm(static_cast<uint32_t>(
458 SPIRV::Opcode::InBoundsPtrAccessChain))
459 .addUse(GV)
460 .addUse(Idx)
461 .addUse(I.getOperand(2).getReg());
462 return MIB.constrainAllUses(TII, TRI, RBI);
463 }
464
465 case TargetOpcode::G_ATOMICRMW_OR:
466 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
467 case TargetOpcode::G_ATOMICRMW_ADD:
468 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
469 case TargetOpcode::G_ATOMICRMW_AND:
470 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
471 case TargetOpcode::G_ATOMICRMW_MAX:
472 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
473 case TargetOpcode::G_ATOMICRMW_MIN:
474 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
475 case TargetOpcode::G_ATOMICRMW_SUB:
476 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
477 case TargetOpcode::G_ATOMICRMW_XOR:
478 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
479 case TargetOpcode::G_ATOMICRMW_UMAX:
480 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
481 case TargetOpcode::G_ATOMICRMW_UMIN:
482 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
483 case TargetOpcode::G_ATOMICRMW_XCHG:
484 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
485 case TargetOpcode::G_ATOMIC_CMPXCHG:
486 return selectAtomicCmpXchg(ResVReg, ResType, I);
487
488 case TargetOpcode::G_FENCE:
489 return selectFence(I);
490
491 default:
492 return false;
493 }
494}
495
496bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
497 const SPIRVType *ResType,
499 CL::OpenCLExtInst CLInst) const {
500 return selectExtInst(ResVReg, ResType, I,
501 {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
502}
503
504bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
505 const SPIRVType *ResType,
507 CL::OpenCLExtInst CLInst,
508 GL::GLSLExtInst GLInst) const {
509 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
510 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
511 return selectExtInst(ResVReg, ResType, I, ExtInsts);
512}
513
514bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
515 const SPIRVType *ResType,
517 const ExtInstList &Insts) const {
518
519 for (const auto &Ex : Insts) {
520 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
521 uint32_t Opcode = Ex.second;
522 if (STI.canUseExtInstSet(Set)) {
523 MachineBasicBlock &BB = *I.getParent();
524 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
525 .addDef(ResVReg)
526 .addUse(GR.getSPIRVTypeID(ResType))
527 .addImm(static_cast<uint32_t>(Set))
528 .addImm(Opcode);
529 const unsigned NumOps = I.getNumOperands();
530 for (unsigned i = 1; i < NumOps; ++i)
531 MIB.add(I.getOperand(i));
532 return MIB.constrainAllUses(TII, TRI, RBI);
533 }
534 }
535 return false;
536}
537
538bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg,
539 const SPIRVType *ResType,
541 Register SrcReg,
542 unsigned Opcode) const {
543 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
544 .addDef(ResVReg)
545 .addUse(GR.getSPIRVTypeID(ResType))
546 .addUse(SrcReg)
547 .constrainAllUses(TII, TRI, RBI);
548}
549
550bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
551 const SPIRVType *ResType,
553 unsigned Opcode) const {
554 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(),
555 Opcode);
556}
557
558static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) {
559 switch (Ord) {
561 return SPIRV::Scope::Invocation;
563 return SPIRV::Scope::Device;
564 default:
565 llvm_unreachable("Unsupported synchronization Scope ID.");
566 }
567}
568
570 MachineInstrBuilder &MIB) {
571 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
572 if (MemOp->isVolatile())
573 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
574 if (MemOp->isNonTemporal())
575 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
576 if (MemOp->getAlign().value())
577 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
578
579 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
580 MIB.addImm(SpvMemOp);
581 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
582 MIB.addImm(MemOp->getAlign().value());
583 }
584}
585
587 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
588 if (Flags & MachineMemOperand::Flags::MOVolatile)
589 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
590 if (Flags & MachineMemOperand::Flags::MONonTemporal)
591 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
592
593 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
594 MIB.addImm(SpvMemOp);
595}
596
597bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
598 const SPIRVType *ResType,
599 MachineInstr &I) const {
600 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
601 Register Ptr = I.getOperand(1 + OpOffset).getReg();
602 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
603 .addDef(ResVReg)
604 .addUse(GR.getSPIRVTypeID(ResType))
605 .addUse(Ptr);
606 if (!I.getNumMemOperands()) {
607 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
608 I.getOpcode() ==
609 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
610 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
611 } else {
612 addMemoryOperands(*I.memoperands_begin(), MIB);
613 }
614 return MIB.constrainAllUses(TII, TRI, RBI);
615}
616
617bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
618 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
619 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
620 Register Ptr = I.getOperand(1 + OpOffset).getReg();
621 MachineBasicBlock &BB = *I.getParent();
622 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
623 .addUse(Ptr)
624 .addUse(StoreVal);
625 if (!I.getNumMemOperands()) {
626 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
627 I.getOpcode() ==
628 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
629 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
630 } else {
631 addMemoryOperands(*I.memoperands_begin(), MIB);
632 }
633 return MIB.constrainAllUses(TII, TRI, RBI);
634}
635
636bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
637 MachineInstr &I) const {
638 MachineBasicBlock &BB = *I.getParent();
639 Register SrcReg = I.getOperand(1).getReg();
640 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
641 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
642 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
643 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
644 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
645 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII);
646 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII);
647 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
648 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
649 // TODO: check if we have such GV, add init, use buildGlobalVariable.
650 Type *LLVMArrTy = ArrayType::get(
651 IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num);
652 GlobalVariable *GV =
653 new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage);
654 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
655 GR.add(GV, GR.CurMF, VarReg);
656
657 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
658 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
659 .addDef(VarReg)
660 .addUse(GR.getSPIRVTypeID(VarTy))
661 .addImm(SPIRV::StorageClass::UniformConstant)
662 .addUse(Const)
663 .constrainAllUses(TII, TRI, RBI);
664 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
665 ValTy, I, TII, SPIRV::StorageClass::UniformConstant);
666 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
667 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast);
668 }
669 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
670 .addUse(I.getOperand(0).getReg())
671 .addUse(SrcReg)
672 .addUse(I.getOperand(2).getReg());
673 if (I.getNumMemOperands())
674 addMemoryOperands(*I.memoperands_begin(), MIB);
675 bool Result = MIB.constrainAllUses(TII, TRI, RBI);
676 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg())
677 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg)
678 .addUse(MIB->getOperand(0).getReg());
679 return Result;
680}
681
682bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
683 const SPIRVType *ResType,
685 unsigned NewOpcode) const {
686 assert(I.hasOneMemOperand());
687 const MachineMemOperand *MemOp = *I.memoperands_begin();
688 uint32_t Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID()));
689 Register ScopeReg = buildI32Constant(Scope, I);
690
691 Register Ptr = I.getOperand(1).getReg();
692 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
693 // auto ScSem =
694 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
695 AtomicOrdering AO = MemOp->getSuccessOrdering();
696 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
697 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I);
698
699 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
700 .addDef(ResVReg)
701 .addUse(GR.getSPIRVTypeID(ResType))
702 .addUse(Ptr)
703 .addUse(ScopeReg)
704 .addUse(MemSemReg)
705 .addUse(I.getOperand(2).getReg())
706 .constrainAllUses(TII, TRI, RBI);
707}
708
709bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
710 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
711 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
712 Register MemSemReg = buildI32Constant(MemSem, I);
713 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
714 uint32_t Scope = static_cast<uint32_t>(getScope(Ord));
715 Register ScopeReg = buildI32Constant(Scope, I);
716 MachineBasicBlock &BB = *I.getParent();
717 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
718 .addUse(ScopeReg)
719 .addUse(MemSemReg)
720 .constrainAllUses(TII, TRI, RBI);
721}
722
723bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
724 const SPIRVType *ResType,
725 MachineInstr &I) const {
726 Register ScopeReg;
727 Register MemSemEqReg;
728 Register MemSemNeqReg;
729 Register Ptr = I.getOperand(2).getReg();
730 if (!isa<GIntrinsic>(I)) {
731 assert(I.hasOneMemOperand());
732 const MachineMemOperand *MemOp = *I.memoperands_begin();
733 unsigned Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID()));
734 ScopeReg = buildI32Constant(Scope, I);
735
736 unsigned ScSem = static_cast<uint32_t>(
737 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)));
738 AtomicOrdering AO = MemOp->getSuccessOrdering();
739 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem;
740 MemSemEqReg = buildI32Constant(MemSemEq, I);
741 AtomicOrdering FO = MemOp->getFailureOrdering();
742 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem;
743 MemSemNeqReg =
744 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I);
745 } else {
746 ScopeReg = I.getOperand(5).getReg();
747 MemSemEqReg = I.getOperand(6).getReg();
748 MemSemNeqReg = I.getOperand(7).getReg();
749 }
750
751 Register Cmp = I.getOperand(3).getReg();
752 Register Val = I.getOperand(4).getReg();
753 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val);
754 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass);
755 const DebugLoc &DL = I.getDebugLoc();
756 bool Result =
757 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
758 .addDef(ACmpRes)
759 .addUse(GR.getSPIRVTypeID(SpvValTy))
760 .addUse(Ptr)
761 .addUse(ScopeReg)
762 .addUse(MemSemEqReg)
763 .addUse(MemSemNeqReg)
764 .addUse(Val)
765 .addUse(Cmp)
766 .constrainAllUses(TII, TRI, RBI);
767 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
768 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
769 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
770 .addDef(CmpSuccReg)
771 .addUse(GR.getSPIRVTypeID(BoolTy))
772 .addUse(ACmpRes)
773 .addUse(Cmp)
774 .constrainAllUses(TII, TRI, RBI);
775 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
776 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
777 .addDef(TmpReg)
778 .addUse(GR.getSPIRVTypeID(ResType))
779 .addUse(ACmpRes)
780 .addUse(GR.getOrCreateUndef(I, ResType, TII))
781 .addImm(0)
782 .constrainAllUses(TII, TRI, RBI);
783 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
784 .addDef(ResVReg)
785 .addUse(GR.getSPIRVTypeID(ResType))
786 .addUse(CmpSuccReg)
787 .addUse(TmpReg)
788 .addImm(1)
789 .constrainAllUses(TII, TRI, RBI);
790 return Result;
791}
792
793static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) {
794 switch (SC) {
795 case SPIRV::StorageClass::Workgroup:
796 case SPIRV::StorageClass::CrossWorkgroup:
797 case SPIRV::StorageClass::Function:
798 return true;
799 default:
800 return false;
801 }
802}
803
804// In SPIR-V address space casting can only happen to and from the Generic
805// storage class. We can also only case Workgroup, CrossWorkgroup, or Function
806// pointers to and from Generic pointers. As such, we can convert e.g. from
807// Workgroup to Function by going via a Generic pointer as an intermediary. All
808// other combinations can only be done by a bitcast, and are probably not safe.
809bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
810 const SPIRVType *ResType,
811 MachineInstr &I) const {
812 // If the AddrSpaceCast user is single and in OpConstantComposite or
813 // OpVariable, we should select OpSpecConstantOp.
814 auto UIs = MRI->use_instructions(ResVReg);
815 if (!UIs.empty() && ++UIs.begin() == UIs.end() &&
816 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite ||
817 UIs.begin()->getOpcode() == SPIRV::OpVariable ||
818 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) {
819 Register NewReg = I.getOperand(1).getReg();
820 MachineBasicBlock &BB = *I.getParent();
821 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
822 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII,
823 SPIRV::StorageClass::Generic);
824 bool Result =
825 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
826 .addDef(ResVReg)
827 .addUse(GR.getSPIRVTypeID(ResType))
828 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric))
829 .addUse(NewReg)
830 .constrainAllUses(TII, TRI, RBI);
831 return Result;
832 }
833 Register SrcPtr = I.getOperand(1).getReg();
834 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
835 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr);
836 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg);
837
838 // Casting from an eligable pointer to Generic.
839 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
840 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
841 // Casting from Generic to an eligable pointer.
842 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
843 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
844 // Casting between 2 eligable pointers using Generic as an intermediary.
845 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
846 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass);
847 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType(
848 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic);
849 MachineBasicBlock &BB = *I.getParent();
850 const DebugLoc &DL = I.getDebugLoc();
851 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
852 .addDef(Tmp)
853 .addUse(GR.getSPIRVTypeID(GenericPtrTy))
854 .addUse(SrcPtr)
855 .constrainAllUses(TII, TRI, RBI);
856 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
857 .addDef(ResVReg)
858 .addUse(GR.getSPIRVTypeID(ResType))
859 .addUse(Tmp)
860 .constrainAllUses(TII, TRI, RBI);
861 }
862 // TODO Should this case just be disallowed completely?
863 // We're casting 2 other arbitrary address spaces, so have to bitcast.
864 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
865}
866
867static unsigned getFCmpOpcode(unsigned PredNum) {
868 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
869 switch (Pred) {
871 return SPIRV::OpFOrdEqual;
873 return SPIRV::OpFOrdGreaterThanEqual;
875 return SPIRV::OpFOrdGreaterThan;
877 return SPIRV::OpFOrdLessThanEqual;
879 return SPIRV::OpFOrdLessThan;
881 return SPIRV::OpFOrdNotEqual;
883 return SPIRV::OpOrdered;
885 return SPIRV::OpFUnordEqual;
887 return SPIRV::OpFUnordGreaterThanEqual;
889 return SPIRV::OpFUnordGreaterThan;
891 return SPIRV::OpFUnordLessThanEqual;
893 return SPIRV::OpFUnordLessThan;
895 return SPIRV::OpFUnordNotEqual;
897 return SPIRV::OpUnordered;
898 default:
899 llvm_unreachable("Unknown predicate type for FCmp");
900 }
901}
902
903static unsigned getICmpOpcode(unsigned PredNum) {
904 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
905 switch (Pred) {
906 case CmpInst::ICMP_EQ:
907 return SPIRV::OpIEqual;
908 case CmpInst::ICMP_NE:
909 return SPIRV::OpINotEqual;
911 return SPIRV::OpSGreaterThanEqual;
913 return SPIRV::OpSGreaterThan;
915 return SPIRV::OpSLessThanEqual;
917 return SPIRV::OpSLessThan;
919 return SPIRV::OpUGreaterThanEqual;
921 return SPIRV::OpUGreaterThan;
923 return SPIRV::OpULessThanEqual;
925 return SPIRV::OpULessThan;
926 default:
927 llvm_unreachable("Unknown predicate type for ICmp");
928 }
929}
930
931static unsigned getPtrCmpOpcode(unsigned Pred) {
932 switch (static_cast<CmpInst::Predicate>(Pred)) {
933 case CmpInst::ICMP_EQ:
934 return SPIRV::OpPtrEqual;
935 case CmpInst::ICMP_NE:
936 return SPIRV::OpPtrNotEqual;
937 default:
938 llvm_unreachable("Unknown predicate type for pointer comparison");
939 }
940}
941
942// Return the logical operation, or abort if none exists.
943static unsigned getBoolCmpOpcode(unsigned PredNum) {
944 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
945 switch (Pred) {
946 case CmpInst::ICMP_EQ:
947 return SPIRV::OpLogicalEqual;
948 case CmpInst::ICMP_NE:
949 return SPIRV::OpLogicalNotEqual;
950 default:
951 llvm_unreachable("Unknown predicate type for Bool comparison");
952 }
953}
954
955bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
956 const SPIRVType *ResType,
957 MachineInstr &I) const {
958 MachineBasicBlock &BB = *I.getParent();
959 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
960 .addDef(ResVReg)
961 .addUse(GR.getSPIRVTypeID(ResType))
962 .addUse(I.getOperand(1).getReg())
963 .constrainAllUses(TII, TRI, RBI);
964}
965
966bool SPIRVInstructionSelector::selectConstVector(Register ResVReg,
967 const SPIRVType *ResType,
968 MachineInstr &I) const {
969 // TODO: only const case is supported for now.
970 assert(std::all_of(
971 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) {
972 if (MO.isDef())
973 return true;
974 if (!MO.isReg())
975 return false;
976 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg());
977 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE &&
978 ConstTy->getOperand(1).isReg());
979 Register ConstReg = ConstTy->getOperand(1).getReg();
980 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg);
981 assert(Const);
982 return (Const->getOpcode() == TargetOpcode::G_CONSTANT ||
983 Const->getOpcode() == TargetOpcode::G_FCONSTANT);
984 }));
985
986 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
987 TII.get(SPIRV::OpConstantComposite))
988 .addDef(ResVReg)
989 .addUse(GR.getSPIRVTypeID(ResType));
990 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
991 MIB.addUse(I.getOperand(i).getReg());
992 return MIB.constrainAllUses(TII, TRI, RBI);
993}
994
995bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
996 const SPIRVType *ResType,
997 unsigned CmpOpc,
998 MachineInstr &I) const {
999 Register Cmp0 = I.getOperand(2).getReg();
1000 Register Cmp1 = I.getOperand(3).getReg();
1001 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
1002 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
1003 "CMP operands should have the same type");
1004 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
1005 .addDef(ResVReg)
1006 .addUse(GR.getSPIRVTypeID(ResType))
1007 .addUse(Cmp0)
1008 .addUse(Cmp1)
1009 .constrainAllUses(TII, TRI, RBI);
1010}
1011
1012bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
1013 const SPIRVType *ResType,
1014 MachineInstr &I) const {
1015 auto Pred = I.getOperand(1).getPredicate();
1016 unsigned CmpOpc;
1017
1018 Register CmpOperand = I.getOperand(2).getReg();
1019 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer))
1020 CmpOpc = getPtrCmpOpcode(Pred);
1021 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
1022 CmpOpc = getBoolCmpOpcode(Pred);
1023 else
1024 CmpOpc = getICmpOpcode(Pred);
1025 return selectCmp(ResVReg, ResType, CmpOpc, I);
1026}
1027
1028void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB,
1029 const MachineInstr &I,
1030 int OpIdx) const {
1031 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
1032 "Expected G_FCONSTANT");
1033 const ConstantFP *FPImm = I.getOperand(1).getFPImm();
1034 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB);
1035}
1036
1037void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB,
1038 const MachineInstr &I,
1039 int OpIdx) const {
1040 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1041 "Expected G_CONSTANT");
1042 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB);
1043}
1044
1046SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
1047 const SPIRVType *ResType) const {
1048 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
1049 const SPIRVType *SpvI32Ty =
1050 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
1051 // Find a constant in DT or build a new one.
1052 auto ConstInt = ConstantInt::get(LLVMTy, Val);
1053 Register NewReg = GR.find(ConstInt, GR.CurMF);
1054 if (!NewReg.isValid()) {
1055 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
1056 GR.add(ConstInt, GR.CurMF, NewReg);
1058 MachineBasicBlock &BB = *I.getParent();
1059 if (Val == 0) {
1060 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1061 .addDef(NewReg)
1062 .addUse(GR.getSPIRVTypeID(SpvI32Ty));
1063 } else {
1064 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1065 .addDef(NewReg)
1066 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
1067 .addImm(APInt(32, Val).getZExtValue());
1068 }
1070 }
1071 return NewReg;
1072}
1073
1074bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
1075 const SPIRVType *ResType,
1076 MachineInstr &I) const {
1077 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
1078 return selectCmp(ResVReg, ResType, CmpOp, I);
1079}
1080
1081Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
1082 MachineInstr &I) const {
1083 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1084 return GR.getOrCreateConsIntVector(0, I, ResType, TII);
1085 return GR.getOrCreateConstInt(0, I, ResType, TII);
1086}
1087
1088Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
1089 const SPIRVType *ResType,
1090 MachineInstr &I) const {
1091 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1092 APInt One =
1093 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
1094 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1095 return GR.getOrCreateConsIntVector(One.getZExtValue(), I, ResType, TII);
1096 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII);
1097}
1098
1099bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
1100 const SPIRVType *ResType,
1101 MachineInstr &I,
1102 bool IsSigned) const {
1103 // To extend a bool, we need to use OpSelect between constants.
1104 Register ZeroReg = buildZerosVal(ResType, I);
1105 Register OneReg = buildOnesVal(IsSigned, ResType, I);
1106 bool IsScalarBool =
1107 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
1108 unsigned Opcode =
1109 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond;
1110 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1111 .addDef(ResVReg)
1112 .addUse(GR.getSPIRVTypeID(ResType))
1113 .addUse(I.getOperand(1).getReg())
1114 .addUse(OneReg)
1115 .addUse(ZeroReg)
1116 .constrainAllUses(TII, TRI, RBI);
1117}
1118
1119bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
1120 const SPIRVType *ResType,
1121 MachineInstr &I, bool IsSigned,
1122 unsigned Opcode) const {
1123 Register SrcReg = I.getOperand(1).getReg();
1124 // We can convert bool value directly to float type without OpConvert*ToF,
1125 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
1126 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
1127 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1128 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
1129 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
1130 const unsigned NumElts = ResType->getOperand(2).getImm();
1131 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
1132 }
1133 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1134 selectSelect(SrcReg, TmpType, I, false);
1135 }
1136 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode);
1137}
1138
1139bool SPIRVInstructionSelector::selectExt(Register ResVReg,
1140 const SPIRVType *ResType,
1141 MachineInstr &I, bool IsSigned) const {
1142 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool))
1143 return selectSelect(ResVReg, ResType, I, IsSigned);
1144 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1145 return selectUnOp(ResVReg, ResType, I, Opcode);
1146}
1147
1148bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
1149 Register ResVReg,
1150 MachineInstr &I,
1151 const SPIRVType *IntTy,
1152 const SPIRVType *BoolTy) const {
1153 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
1154 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1155 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
1156 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
1157 Register Zero = buildZerosVal(IntTy, I);
1158 Register One = buildOnesVal(false, IntTy, I);
1159 MachineBasicBlock &BB = *I.getParent();
1160 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1161 .addDef(BitIntReg)
1162 .addUse(GR.getSPIRVTypeID(IntTy))
1163 .addUse(IntReg)
1164 .addUse(One)
1165 .constrainAllUses(TII, TRI, RBI);
1166 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
1167 .addDef(ResVReg)
1168 .addUse(GR.getSPIRVTypeID(BoolTy))
1169 .addUse(BitIntReg)
1170 .addUse(Zero)
1171 .constrainAllUses(TII, TRI, RBI);
1172}
1173
1174bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
1175 const SPIRVType *ResType,
1176 MachineInstr &I) const {
1177 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) {
1178 Register IntReg = I.getOperand(1).getReg();
1179 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg);
1180 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
1181 }
1182 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
1183 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1184 return selectUnOp(ResVReg, ResType, I, Opcode);
1185}
1186
1187bool SPIRVInstructionSelector::selectConst(Register ResVReg,
1188 const SPIRVType *ResType,
1189 const APInt &Imm,
1190 MachineInstr &I) const {
1191 unsigned TyOpcode = ResType->getOpcode();
1192 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero());
1193 MachineBasicBlock &BB = *I.getParent();
1194 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) &&
1195 Imm.isZero())
1196 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1197 .addDef(ResVReg)
1198 .addUse(GR.getSPIRVTypeID(ResType))
1199 .constrainAllUses(TII, TRI, RBI);
1200 if (TyOpcode == SPIRV::OpTypeInt) {
1201 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!");
1202 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII);
1203 if (Reg == ResVReg)
1204 return true;
1205 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1206 .addDef(ResVReg)
1207 .addUse(Reg)
1208 .constrainAllUses(TII, TRI, RBI);
1209 }
1210 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1211 .addDef(ResVReg)
1212 .addUse(GR.getSPIRVTypeID(ResType));
1213 // <=32-bit integers should be caught by the sdag pattern.
1214 assert(Imm.getBitWidth() > 32);
1215 addNumImm(Imm, MIB);
1216 return MIB.constrainAllUses(TII, TRI, RBI);
1217}
1218
1219bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
1220 const SPIRVType *ResType,
1221 MachineInstr &I) const {
1222 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1223 .addDef(ResVReg)
1224 .addUse(GR.getSPIRVTypeID(ResType))
1225 .constrainAllUses(TII, TRI, RBI);
1226}
1227
1229 assert(MO.isReg());
1230 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1231 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE)
1232 return false;
1233 assert(TypeInst->getOperand(1).isReg());
1234 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1235 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT;
1236}
1237
1238static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
1239 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1240 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1241 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT);
1242 return ImmInst->getOperand(1).getCImm()->getZExtValue();
1243}
1244
1245bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
1246 const SPIRVType *ResType,
1247 MachineInstr &I) const {
1248 MachineBasicBlock &BB = *I.getParent();
1249 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
1250 .addDef(ResVReg)
1251 .addUse(GR.getSPIRVTypeID(ResType))
1252 // object to insert
1253 .addUse(I.getOperand(3).getReg())
1254 // composite to insert into
1255 .addUse(I.getOperand(2).getReg());
1256 for (unsigned i = 4; i < I.getNumOperands(); i++)
1257 MIB.addImm(foldImm(I.getOperand(i), MRI));
1258 return MIB.constrainAllUses(TII, TRI, RBI);
1259}
1260
1261bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
1262 const SPIRVType *ResType,
1263 MachineInstr &I) const {
1264 MachineBasicBlock &BB = *I.getParent();
1265 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1266 .addDef(ResVReg)
1267 .addUse(GR.getSPIRVTypeID(ResType))
1268 .addUse(I.getOperand(2).getReg());
1269 for (unsigned i = 3; i < I.getNumOperands(); i++)
1270 MIB.addImm(foldImm(I.getOperand(i), MRI));
1271 return MIB.constrainAllUses(TII, TRI, RBI);
1272}
1273
1274bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
1275 const SPIRVType *ResType,
1276 MachineInstr &I) const {
1277 if (isImm(I.getOperand(4), MRI))
1278 return selectInsertVal(ResVReg, ResType, I);
1279 MachineBasicBlock &BB = *I.getParent();
1280 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
1281 .addDef(ResVReg)
1282 .addUse(GR.getSPIRVTypeID(ResType))
1283 .addUse(I.getOperand(2).getReg())
1284 .addUse(I.getOperand(3).getReg())
1285 .addUse(I.getOperand(4).getReg())
1286 .constrainAllUses(TII, TRI, RBI);
1287}
1288
1289bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
1290 const SPIRVType *ResType,
1291 MachineInstr &I) const {
1292 if (isImm(I.getOperand(3), MRI))
1293 return selectExtractVal(ResVReg, ResType, I);
1294 MachineBasicBlock &BB = *I.getParent();
1295 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
1296 .addDef(ResVReg)
1297 .addUse(GR.getSPIRVTypeID(ResType))
1298 .addUse(I.getOperand(2).getReg())
1299 .addUse(I.getOperand(3).getReg())
1300 .constrainAllUses(TII, TRI, RBI);
1301}
1302
1303bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
1304 const SPIRVType *ResType,
1305 MachineInstr &I) const {
1306 const bool IsGEPInBounds = I.getOperand(2).getImm();
1307
1308 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
1309 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
1310 // we have to use Op[InBounds]AccessChain.
1311 const unsigned Opcode = STI.isVulkanEnv()
1312 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
1313 : SPIRV::OpAccessChain)
1314 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
1315 : SPIRV::OpPtrAccessChain);
1316
1317 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1318 .addDef(ResVReg)
1319 .addUse(GR.getSPIRVTypeID(ResType))
1320 // Object to get a pointer to.
1321 .addUse(I.getOperand(3).getReg());
1322 // Adding indices.
1323 const unsigned StartingIndex =
1324 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
1325 ? 5
1326 : 4;
1327 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
1328 Res.addUse(I.getOperand(i).getReg());
1329 return Res.constrainAllUses(TII, TRI, RBI);
1330}
1331
1332bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
1333 const SPIRVType *ResType,
1334 MachineInstr &I) const {
1335 MachineBasicBlock &BB = *I.getParent();
1336 switch (cast<GIntrinsic>(I).getIntrinsicID()) {
1337 case Intrinsic::spv_load:
1338 return selectLoad(ResVReg, ResType, I);
1339 case Intrinsic::spv_store:
1340 return selectStore(I);
1341 case Intrinsic::spv_extractv:
1342 return selectExtractVal(ResVReg, ResType, I);
1343 case Intrinsic::spv_insertv:
1344 return selectInsertVal(ResVReg, ResType, I);
1345 case Intrinsic::spv_extractelt:
1346 return selectExtractElt(ResVReg, ResType, I);
1347 case Intrinsic::spv_insertelt:
1348 return selectInsertElt(ResVReg, ResType, I);
1349 case Intrinsic::spv_gep:
1350 return selectGEP(ResVReg, ResType, I);
1351 case Intrinsic::spv_unref_global:
1352 case Intrinsic::spv_init_global: {
1353 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
1354 MachineInstr *Init = I.getNumExplicitOperands() > 2
1355 ? MRI->getVRegDef(I.getOperand(2).getReg())
1356 : nullptr;
1357 assert(MI);
1358 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init);
1359 }
1360 case Intrinsic::spv_undef: {
1361 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1362 .addDef(ResVReg)
1363 .addUse(GR.getSPIRVTypeID(ResType));
1364 return MIB.constrainAllUses(TII, TRI, RBI);
1365 }
1366 case Intrinsic::spv_const_composite: {
1367 // If no values are attached, the composite is null constant.
1368 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
1369 unsigned Opcode =
1370 IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite;
1371 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1372 .addDef(ResVReg)
1373 .addUse(GR.getSPIRVTypeID(ResType));
1374 // skip type MD node we already used when generated assign.type for this
1375 if (!IsNull) {
1376 for (unsigned i = I.getNumExplicitDefs() + 1;
1377 i < I.getNumExplicitOperands(); ++i) {
1378 MIB.addUse(I.getOperand(i).getReg());
1379 }
1380 }
1381 return MIB.constrainAllUses(TII, TRI, RBI);
1382 }
1383 case Intrinsic::spv_assign_name: {
1384 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
1385 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
1386 for (unsigned i = I.getNumExplicitDefs() + 2;
1387 i < I.getNumExplicitOperands(); ++i) {
1388 MIB.addImm(I.getOperand(i).getImm());
1389 }
1390 return MIB.constrainAllUses(TII, TRI, RBI);
1391 }
1392 case Intrinsic::spv_switch: {
1393 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
1394 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
1395 if (I.getOperand(i).isReg())
1396 MIB.addReg(I.getOperand(i).getReg());
1397 else if (I.getOperand(i).isCImm())
1398 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
1399 else if (I.getOperand(i).isMBB())
1400 MIB.addMBB(I.getOperand(i).getMBB());
1401 else
1402 llvm_unreachable("Unexpected OpSwitch operand");
1403 }
1404 return MIB.constrainAllUses(TII, TRI, RBI);
1405 }
1406 case Intrinsic::spv_cmpxchg:
1407 return selectAtomicCmpXchg(ResVReg, ResType, I);
1408 case Intrinsic::spv_unreachable:
1409 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable));
1410 break;
1411 case Intrinsic::spv_alloca:
1412 return selectFrameIndex(ResVReg, ResType, I);
1413 case Intrinsic::spv_assume:
1414 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1415 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
1416 .addUse(I.getOperand(1).getReg());
1417 break;
1418 case Intrinsic::spv_expect:
1419 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1420 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
1421 .addDef(ResVReg)
1422 .addUse(GR.getSPIRVTypeID(ResType))
1423 .addUse(I.getOperand(2).getReg())
1424 .addUse(I.getOperand(3).getReg());
1425 break;
1426 default:
1427 llvm_unreachable("Intrinsic selection not implemented");
1428 }
1429 return true;
1430}
1431
1432bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
1433 const SPIRVType *ResType,
1434 MachineInstr &I) const {
1435 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
1436 .addDef(ResVReg)
1437 .addUse(GR.getSPIRVTypeID(ResType))
1438 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
1439 .constrainAllUses(TII, TRI, RBI);
1440}
1441
1442bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
1443 // InstructionSelector walks backwards through the instructions. We can use
1444 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
1445 // first, so can generate an OpBranchConditional here. If there is no
1446 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
1447 const MachineInstr *PrevI = I.getPrevNode();
1448 MachineBasicBlock &MBB = *I.getParent();
1449 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
1450 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
1451 .addUse(PrevI->getOperand(0).getReg())
1452 .addMBB(PrevI->getOperand(1).getMBB())
1453 .addMBB(I.getOperand(0).getMBB())
1454 .constrainAllUses(TII, TRI, RBI);
1455 }
1456 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
1457 .addMBB(I.getOperand(0).getMBB())
1458 .constrainAllUses(TII, TRI, RBI);
1459}
1460
1461bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
1462 // InstructionSelector walks backwards through the instructions. For an
1463 // explicit conditional branch with no fallthrough, we use both a G_BR and a
1464 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
1465 // generate the OpBranchConditional in selectBranch above.
1466 //
1467 // If an OpBranchConditional has been generated, we simply return, as the work
1468 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
1469 // implicit fallthrough to the next basic block, so we need to create an
1470 // OpBranchConditional with an explicit "false" argument pointing to the next
1471 // basic block that LLVM would fall through to.
1472 const MachineInstr *NextI = I.getNextNode();
1473 // Check if this has already been successfully selected.
1474 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
1475 return true;
1476 // Must be relying on implicit block fallthrough, so generate an
1477 // OpBranchConditional with the "next" basic block as the "false" target.
1478 MachineBasicBlock &MBB = *I.getParent();
1479 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
1480 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
1481 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
1482 .addUse(I.getOperand(0).getReg())
1483 .addMBB(I.getOperand(1).getMBB())
1484 .addMBB(NextMBB)
1485 .constrainAllUses(TII, TRI, RBI);
1486}
1487
1488bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
1489 const SPIRVType *ResType,
1490 MachineInstr &I) const {
1491 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi))
1492 .addDef(ResVReg)
1493 .addUse(GR.getSPIRVTypeID(ResType));
1494 const unsigned NumOps = I.getNumOperands();
1495 for (unsigned i = 1; i < NumOps; i += 2) {
1496 MIB.addUse(I.getOperand(i + 0).getReg());
1497 MIB.addMBB(I.getOperand(i + 1).getMBB());
1498 }
1499 return MIB.constrainAllUses(TII, TRI, RBI);
1500}
1501
1502bool SPIRVInstructionSelector::selectGlobalValue(
1503 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
1504 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
1505 MachineIRBuilder MIRBuilder(I);
1506 const GlobalValue *GV = I.getOperand(1).getGlobal();
1507 Type *GVType = GV->getValueType();
1508 SPIRVType *PointerBaseType;
1509 if (GVType->isArrayTy()) {
1510 SPIRVType *ArrayElementType =
1511 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder,
1512 SPIRV::AccessQualifier::ReadWrite, false);
1513 PointerBaseType = GR.getOrCreateSPIRVArrayType(
1514 ArrayElementType, GVType->getArrayNumElements(), I, TII);
1515 } else {
1516 PointerBaseType = GR.getOrCreateSPIRVType(
1517 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
1518 }
1519 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
1520 PointerBaseType, I, TII,
1522 std::string GlobalIdent = GV->getGlobalIdentifier();
1523 // We have functions as operands in tests with blocks of instruction e.g. in
1524 // transcoding/global_block.ll. These operands are not used and should be
1525 // substituted by zero constants. Their type is expected to be always
1526 // OpTypePointer Function %uchar.
1527 if (isa<Function>(GV)) {
1528 const Constant *ConstVal = GV;
1529 MachineBasicBlock &BB = *I.getParent();
1530 Register NewReg = GR.find(ConstVal, GR.CurMF);
1531 if (!NewReg.isValid()) {
1532 Register NewReg = ResVReg;
1533 GR.add(ConstVal, GR.CurMF, NewReg);
1534 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1535 .addDef(NewReg)
1536 .addUse(GR.getSPIRVTypeID(ResType))
1537 .constrainAllUses(TII, TRI, RBI);
1538 }
1539 assert(NewReg != ResVReg);
1540 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1541 .addDef(ResVReg)
1542 .addUse(NewReg)
1543 .constrainAllUses(TII, TRI, RBI);
1544 }
1545 auto GlobalVar = cast<GlobalVariable>(GV);
1546 assert(GlobalVar->getName() != "llvm.global.annotations");
1547
1548 bool HasInit = GlobalVar->hasInitializer() &&
1549 !isa<UndefValue>(GlobalVar->getInitializer());
1550 // Skip empty declaration for GVs with initilaizers till we get the decl with
1551 // passed initializer.
1552 if (HasInit && !Init)
1553 return true;
1554
1555 unsigned AddrSpace = GV->getAddressSpace();
1556 SPIRV::StorageClass::StorageClass Storage =
1557 addressSpaceToStorageClass(AddrSpace);
1558 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage &&
1559 Storage != SPIRV::StorageClass::Function;
1560 SPIRV::LinkageType::LinkageType LnkType =
1562 ? SPIRV::LinkageType::Import
1563 : SPIRV::LinkageType::Export;
1564
1565 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV,
1566 Storage, Init, GlobalVar->isConstant(),
1567 HasLnkTy, LnkType, MIRBuilder, true);
1568 return Reg.isValid();
1569}
1570
1571bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
1572 const SPIRVType *ResType,
1573 MachineInstr &I) const {
1574 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
1575 return selectExtInst(ResVReg, ResType, I, CL::log10);
1576 }
1577
1578 // There is no log10 instruction in the GLSL Extended Instruction set, so it
1579 // is implemented as:
1580 // log10(x) = log2(x) * (1 / log2(10))
1581 // = log2(x) * 0.30103
1582
1583 MachineIRBuilder MIRBuilder(I);
1584 MachineBasicBlock &BB = *I.getParent();
1585
1586 // Build log2(x).
1587 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1588 bool Result =
1589 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1590 .addDef(VarReg)
1591 .addUse(GR.getSPIRVTypeID(ResType))
1592 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1593 .addImm(GL::Log2)
1594 .add(I.getOperand(1))
1595 .constrainAllUses(TII, TRI, RBI);
1596
1597 // Build 0.30103.
1598 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
1599 ResType->getOpcode() == SPIRV::OpTypeFloat);
1600 // TODO: Add matrix implementation once supported by the HLSL frontend.
1601 const SPIRVType *SpirvScalarType =
1602 ResType->getOpcode() == SPIRV::OpTypeVector
1603 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg())
1604 : ResType;
1605 Register ScaleReg =
1606 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType);
1607
1608 // Multiply log2(x) by 0.30103 to get log10(x) result.
1609 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
1610 ? SPIRV::OpVectorTimesScalar
1611 : SPIRV::OpFMulS;
1612 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1613 .addDef(ResVReg)
1614 .addUse(GR.getSPIRVTypeID(ResType))
1615 .addUse(VarReg)
1616 .addUse(ScaleReg)
1617 .constrainAllUses(TII, TRI, RBI);
1618
1619 return Result;
1620}
1621
1622namespace llvm {
1625 const SPIRVSubtarget &Subtarget,
1626 const RegisterBankInfo &RBI) {
1627 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
1628}
1629} // namespace llvm
unsigned const MachineRegisterInfo * MRI
#define Success
static unsigned getIntrinsicID(const SDNode *N)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file declares a class to represent arbitrary precision floating point values and provide a varie...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::vector< std::pair< SPIRV::InstructionSet::InstructionSet, uint32_t > > ExtInstList
static SPIRV::Scope::Scope getScope(SyncScope::ID Ord)
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static void addMemoryOperands(MachineMemOperand *MemOp, MachineInstrBuilder &MIB)
static unsigned getFCmpOpcode(unsigned PredNum)
bool isTypeFoldingSupported(unsigned Opcode)
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static unsigned getBoolCmpOpcode(unsigned PredNum)
static unsigned getICmpOpcode(unsigned PredNum)
static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)
static unsigned getPtrCmpOpcode(unsigned Pred)
static constexpr uint32_t Opcode
Definition: aarch32.h:200
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1485
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:780
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:783
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:809
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:810
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:786
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:795
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:784
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:785
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:804
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:803
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:807
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:794
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:788
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:791
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:805
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:792
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:787
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:789
@ ICMP_EQ
equal
Definition: InstrTypes.h:801
@ ICMP_NE
not equal
Definition: InstrTypes.h:802
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:808
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:796
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:806
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:793
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:790
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:261
const APFloat & getValueAPF() const
Definition: Constants.h:297
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:146
This is an important base class in LLVM.
Definition: Constant.h:41
A debug info location.
Definition: DebugLoc.h:33
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:273
LinkageTypes getLinkage() const
Definition: GlobalValue.h:541
unsigned getAddressSpace() const
Definition: GlobalValue.h:201
static std::string getGlobalIdentifier(StringRef Name, GlobalValue::LinkageTypes Linkage, StringRef FileName)
Return the modified name for a global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:144
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:507
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
Type * getValueType() const
Definition: GlobalValue.h:292
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:285
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Helper class to build MachineInstr.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:543
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
Register getReg() const
getReg - Returns the register number.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Analysis providing profile information.
Holds all the information related to register banks.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isValid() const
Definition: Register.h:116
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isArrayTy() const
True if this is an instance of ArrayType.
Definition: Type.h:252
Type * getArrayElementType() const
Definition: Type.h:404
uint64_t getArrayNumElements() const
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:316
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition: LLVMContext.h:54
@ System
Synchronized with respect to all concurrently executing threads.
Definition: LLVMContext.h:57
Reg
All possible values of the reg field in the ModR/M byte.
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
Definition: SPIRVUtils.cpp:79
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:152
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
Definition: TargetOpcodes.h:30
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
Definition: SPIRVUtils.cpp:225
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
Definition: SPIRVUtils.cpp:174
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)
Definition: SPIRVUtils.cpp:113
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace)
Definition: SPIRVUtils.cpp:154
bool isSpvIntrinsic(MachineInstr &MI, Intrinsic::ID IntrinsicID)
Definition: SPIRVUtils.cpp:231
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &Subtarget, const RegisterBankInfo &RBI)
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
Definition: SPIRVUtils.cpp:192