LLVM 24.0.0git
SPIRVInstructionSelector.cpp
Go to the documentation of this file.
1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "MCTargetDesc/SPIRVMCTargetDesc.h"
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
20#include "SPIRVRegisterInfo.h"
21#include "SPIRVTargetMachine.h"
22#include "SPIRVTypeInst.h"
23#include "SPIRVUtils.h"
24#include "llvm/ADT/APFloat.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/STLFunctionalExtras.h"
27#include "llvm/ADT/SmallSet.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
30#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
31#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/Register.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/IR/DiagnosticInfo.h"
37#include "llvm/IR/IntrinsicsSPIRV.h"
38#include "llvm/Support/Debug.h"
39#include "llvm/Support/ErrorHandling.h"
40#include <functional>
41#include <optional>
42
43#define DEBUG_TYPE "spirv-isel"
44
45using namespace llvm;
46namespace CL = SPIRV::OpenCLExtInst;
47namespace GL = SPIRV::GLSLExtInst;
48
49using ExtInstList =
50 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
51
52namespace {
53
54struct ImageOperands {
55 std::optional<Register> Bias;
56 std::optional<Register> Offset;
57 std::optional<Register> MinLod;
58 std::optional<Register> GradX;
59 std::optional<Register> GradY;
60 std::optional<Register> Lod;
61 std::optional<Register> Compare;
62};
63
64struct SplitParts {
65 SPIRVTypeInst Type = nullptr;
66 Register High;
67 Register Low;
68 bool IsScalar = false;
69};
70
71llvm::SPIRV::SelectionControl::SelectionControl
72getSelectionOperandForImm(int Imm) {
73 if (Imm == 2)
74 return SPIRV::SelectionControl::Flatten;
75 if (Imm == 1)
76 return SPIRV::SelectionControl::DontFlatten;
77 if (Imm == 0)
78 return SPIRV::SelectionControl::None;
79 llvm_unreachable("Invalid immediate");
80}
81
82#define GET_GLOBALISEL_PREDICATE_BITSET
83#include "SPIRVGenGlobalISel.inc"
84#undef GET_GLOBALISEL_PREDICATE_BITSET
85
86class SPIRVInstructionSelector : public InstructionSelector {
87 const SPIRVSubtarget &STI;
88 const SPIRVInstrInfo &TII;
89 const SPIRVRegisterInfo &TRI;
90 const RegisterBankInfo &RBI;
91 SPIRVGlobalRegistry &GR;
92 MachineRegisterInfo *MRI;
93 MachineFunction *HasVRegsReset = nullptr;
94
95 /// We need to keep track of the number we give to anonymous global values to
96 /// generate the same name every time when this is needed.
97 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
98 SmallPtrSet<MachineInstr *, 8> DeadMIs;
99
100public:
101 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
102 const SPIRVSubtarget &ST,
103 const RegisterBankInfo &RBI);
104 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
105 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
106 BlockFrequencyInfo *BFI) override;
107 // Common selection code. Instruction-specific selection occurs in spvSelect.
108 bool select(MachineInstr &I) override;
109 static const char *getName() { return DEBUG_TYPE; }
110
111#define GET_GLOBALISEL_PREDICATES_DECL
112#include "SPIRVGenGlobalISel.inc"
113#undef GET_GLOBALISEL_PREDICATES_DECL
114
115#define GET_GLOBALISEL_TEMPORARIES_DECL
116#include "SPIRVGenGlobalISel.inc"
117#undef GET_GLOBALISEL_TEMPORARIES_DECL
118
119private:
120 void resetVRegsType(MachineFunction &MF);
121 void removeDeadInstruction(MachineInstr &MI) const;
122 void removeOpNamesForDeadMI(MachineInstr &MI) const;
123
124 // tblgen-erated 'select' implementation, used as the initial selector for
125 // the patterns that don't require complex C++.
126 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
127
128 // All instruction-specific selection that didn't happen in "select()".
129 // Is basically a large Switch/Case delegating to all other select method.
130 bool spvSelect(Register ResVReg, SPIRVTypeInst ResType,
131 MachineInstr &I) const;
132
133 bool selectFirstBitHigh(Register ResVReg, SPIRVTypeInst ResType,
134 MachineInstr &I, bool IsSigned) const;
135
136 bool selectFirstBitLow(Register ResVReg, SPIRVTypeInst ResType,
137 MachineInstr &I) const;
138
139 bool selectFirstBitSet16(Register ResVReg, SPIRVTypeInst ResType,
140 MachineInstr &I, unsigned ExtendOpcode,
141 unsigned BitSetOpcode) const;
142
143 bool selectFirstBitSet32(Register ResVReg, SPIRVTypeInst ResType,
144 MachineInstr &I, Register SrcReg,
145 unsigned BitSetOpcode) const;
146
147 bool selectFirstBitSet64(Register ResVReg, SPIRVTypeInst ResType,
148 MachineInstr &I, Register SrcReg,
149 unsigned BitSetOpcode, bool SwapPrimarySide) const;
150
151 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
152 const MachineInstr *Init = nullptr) const;
153
154 bool selectOpWithSrcs(Register ResVReg, SPIRVTypeInst ResType,
155 MachineInstr &I, ArrayRef<Register> SrcRegs,
156 unsigned Opcode) const;
157
158 bool selectUnOp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
159 unsigned Opcode) const;
160
161 bool selectBitcast(Register ResVReg, SPIRVTypeInst ResType,
162 MachineInstr &I) const;
163
164 bool selectLoad(Register ResVReg, SPIRVTypeInst ResType,
165 MachineInstr &I) const;
166 bool selectAtomicLoad(Register ResVReg, SPIRVTypeInst ResType,
167 MachineInstr &I) const;
168 bool selectStore(MachineInstr &I) const;
169 bool selectAtomicStore(MachineInstr &I) const;
170
171 bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType,
172 MachineInstr &I) const;
173 bool selectStackRestore(MachineInstr &I) const;
174
175 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
176 Register getOrCreateMemSetGlobal(MachineInstr &I) const;
177 bool selectCopyMemory(MachineInstr &I, Register SrcReg) const;
178 bool selectCopyMemorySized(MachineInstr &I, Register SrcReg) const;
179
180 bool selectAtomicRMW(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
181 unsigned NewOpcode, unsigned NegateOpcode = 0) const;
182
183 // Creates an integer-typed register with bitwidth equal to pointer size.
184 Register createPtrSizedIntReg(MachineIRBuilder &MIRBuilder) const;
185 // Emit an OpConvertPtrToU that converts the pointer value in \p PtrVal into
186 // an integer of equal bitwidth, returning the register holding the result.
187 Register convertPtrToInt(Register PtrVal, MachineIRBuilder &MIRBuilder) const;
188 // Emit an OpBitcast that reinterprets the pointer \p Ptr as a pointer to an
189 // integer of pointer size in storage class \p SC, returning the result.
190 Register castPtrToPtrToInt(Register Ptr, SPIRV::StorageClass::StorageClass SC,
191 MachineIRBuilder &MIRBuilder) const;
192 // Handle atomic loads, stores and exchanges of pointer types by casting
193 // to/from integer types as needed.
194 bool selectAtomicPtrValue(
195 Register ResVReg, SPIRVTypeInst ResType, MachineIRBuilder &MIRBuilder,
196 function_ref<Register(SPIRVTypeInst IntType)> EmitAtomic) const;
197
198 bool selectAtomicCmpXchg(Register ResVReg, SPIRVTypeInst ResType,
199 MachineInstr &I) const;
200
201 bool selectFence(MachineInstr &I) const;
202
203 bool selectAddrSpaceCast(Register ResVReg, SPIRVTypeInst ResType,
204 MachineInstr &I) const;
205
206 bool selectPtrMask(Register ResVReg, SPIRVTypeInst ResType,
207 MachineInstr &I) const;
208
209 bool selectAnyOrAll(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
210 unsigned OpType) const;
211
212 bool selectAll(Register ResVReg, SPIRVTypeInst ResType,
213 MachineInstr &I) const;
214
215 bool selectAny(Register ResVReg, SPIRVTypeInst ResType,
216 MachineInstr &I) const;
217
218 bool selectBitreverse(Register ResVReg, SPIRVTypeInst ResType,
219 MachineInstr &I) const;
220
221 bool selectBitreverseViaI32(Register ResVReg, SPIRVTypeInst ResType,
222 MachineInstr &I, Register Op) const;
223
224 bool selectBitreverse64(Register ResVReg, SPIRVTypeInst ResType,
225 MachineInstr &I, Register SrcReg) const;
226
227 bool selectBitreverseNative(Register ResVReg, SPIRVTypeInst ResType,
228 MachineInstr &I, Register Op) const;
229
230 bool selectBuildVector(Register ResVReg, SPIRVTypeInst ResType,
231 MachineInstr &I) const;
232 bool selectSplatVector(Register ResVReg, SPIRVTypeInst ResType,
233 MachineInstr &I) const;
234 bool selectConcatVectors(Register ResVReg, SPIRVTypeInst ResType,
235 MachineInstr &I) const;
236
237 bool selectCmp(Register ResVReg, SPIRVTypeInst ResType,
238 unsigned comparisonOpcode, MachineInstr &I) const;
239 bool selectDiscard(Register ResVReg, SPIRVTypeInst ResType,
240 MachineInstr &I) const;
241
242 bool selectICmp(Register ResVReg, SPIRVTypeInst ResType,
243 MachineInstr &I) const;
244 bool selectFCmp(Register ResVReg, SPIRVTypeInst ResType,
245 MachineInstr &I) const;
246
247 bool selectSign(Register ResVReg, SPIRVTypeInst ResType,
248 MachineInstr &I) const;
249
250 bool selectFloatDot(Register ResVReg, SPIRVTypeInst ResType,
251 MachineInstr &I) const;
252
253 bool selectOverflowArith(Register ResVReg, SPIRVTypeInst ResType,
254 MachineInstr &I, unsigned Opcode) const;
255 bool selectDebugTrap(Register ResVReg, SPIRVTypeInst ResType,
256 MachineInstr &I) const;
257 bool selectPrefetch(MachineInstr &I) const;
258
259 bool selectIntegerDot(Register ResVReg, SPIRVTypeInst ResType,
260 MachineInstr &I, bool Signed) const;
261
262 bool selectIntegerDotExpansion(Register ResVReg, SPIRVTypeInst ResType,
263 MachineInstr &I) const;
264
265 bool selectOpIsInf(Register ResVReg, SPIRVTypeInst ResType,
266 MachineInstr &I) const;
267
268 bool selectOpIsNan(Register ResVReg, SPIRVTypeInst ResType,
269 MachineInstr &I) const;
270
271 bool selectOpIsFinite(Register ResVReg, SPIRVTypeInst ResType,
272 MachineInstr &I) const;
273
274 bool selectOpIsNormal(Register ResVReg, SPIRVTypeInst ResType,
275 MachineInstr &I) const;
276
277 bool selectPopCount(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
278 unsigned Opcode) const;
279
280 bool selectPopCount16(Register ResVReg, SPIRVTypeInst ResType,
281 MachineInstr &I, unsigned ExtOpcode,
282 unsigned Opcode) const;
283
284 bool selectPopCount32(Register ResVReg, SPIRVTypeInst ResType,
285 MachineInstr &I, Register SrcReg,
286 unsigned Opcode) const;
287
288 bool selectPopCount64(Register ResVReg, SPIRVTypeInst ResType,
289 MachineInstr &I, Register SrcReg,
290 unsigned Opcode) const;
291
292 template <bool Signed>
293 bool selectDot4AddPacked(Register ResVReg, SPIRVTypeInst ResType,
294 MachineInstr &I) const;
295 template <bool Signed>
296 bool selectDot4AddPackedExpansion(Register ResVReg, SPIRVTypeInst ResType,
297 MachineInstr &I) const;
298
299 bool selectWavePrefixBitCount(Register ResVReg, SPIRVTypeInst ResType,
300 MachineInstr &I) const;
301
302 template <typename PickOpcodeFn>
303 bool selectWaveReduce(Register ResVReg, SPIRVTypeInst ResType,
304 MachineInstr &I, bool IsUnsigned,
305 PickOpcodeFn &&PickOpcode) const;
306
307 bool selectWaveReduceOp(Register ResVReg, SPIRVTypeInst ResType,
308 MachineInstr &I, unsigned Opcode) const;
309
310 bool selectWaveReduceMax(Register ResVReg, SPIRVTypeInst ResType,
311 MachineInstr &I, bool IsUnsigned) const;
312
313 bool selectWaveReduceMin(Register ResVReg, SPIRVTypeInst ResType,
314 MachineInstr &I, bool IsUnsigned) const;
315
316 bool selectWaveReduceSum(Register ResVReg, SPIRVTypeInst ResType,
317 MachineInstr &I) const;
318
319 bool selectWaveReduceProduct(Register ResVReg, const SPIRVTypeInst ResType,
320 MachineInstr &I) const;
321
322 template <typename PickOpcodeFn>
323 bool selectWaveExclusiveScan(Register ResVReg, SPIRVTypeInst ResType,
324 MachineInstr &I, bool IsUnsigned,
325 PickOpcodeFn &&PickOpcode) const;
326
327 bool selectWaveExclusiveScanSum(Register ResVReg, SPIRVTypeInst ResType,
328 MachineInstr &I) const;
329
330 bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType,
331 MachineInstr &I) const;
332
333 bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
334 unsigned Direction) const;
335
336 bool selectConst(Register ResVReg, SPIRVTypeInst ResType,
337 MachineInstr &I) const;
338
339 bool selectSelect(Register ResVReg, SPIRVTypeInst ResType,
340 MachineInstr &I) const;
341 bool selectBoolToInt(Register ResVReg, SPIRVTypeInst ResType,
342 Register BooleanVReg, MachineInstr &InsertAt,
343 bool IsSigned) const;
344 bool selectIToF(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
345 bool IsSigned, unsigned Opcode) const;
346 bool selectExt(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
347 bool IsSigned) const;
348
349 bool selectTrunc(Register ResVReg, SPIRVTypeInst ResType,
350 MachineInstr &I) const;
351
352 bool selectSUCmp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
353 bool IsSigned) const;
354
355 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
356 SPIRVTypeInst intTy, SPIRVTypeInst boolTy) const;
357
358 bool selectOpUndef(Register ResVReg, SPIRVTypeInst ResType,
359 MachineInstr &I) const;
360 bool selectFreeze(Register ResVReg, SPIRVTypeInst ResType,
361 MachineInstr &I) const;
362 bool selectIntrinsic(Register ResVReg, SPIRVTypeInst ResType,
363 MachineInstr &I) const;
364 bool selectExtractVal(Register ResVReg, SPIRVTypeInst ResType,
365 MachineInstr &I) const;
366 bool selectInsertVal(Register ResVReg, SPIRVTypeInst ResType,
367 MachineInstr &I) const;
368 bool selectExtractElt(Register ResVReg, SPIRVTypeInst ResType,
369 MachineInstr &I) const;
370 bool selectInsertElt(Register ResVReg, SPIRVTypeInst ResType,
371 MachineInstr &I) const;
372 bool selectGEP(Register ResVReg, SPIRVTypeInst ResType,
373 MachineInstr &I) const;
374
375 bool selectMaskedGather(Register ResVReg, SPIRVTypeInst ResType,
376 MachineInstr &I) const;
377 bool selectMaskedScatter(MachineInstr &I) const;
378
379 bool diagnoseUnsupported(const MachineInstr &I, const Twine &Msg) const;
380
381 bool selectAbort(MachineInstr &I) const;
382 bool selectTrap(MachineInstr &I) const;
383 bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType,
384 MachineInstr &I) const;
385 bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType,
386 MachineInstr &I) const;
387
388 bool selectBranch(MachineInstr &I) const;
389 bool selectBranchCond(MachineInstr &I) const;
390
391 bool selectPhi(Register ResVReg, MachineInstr &I) const;
392
393 bool selectExtInst(Register ResVReg, SPIRVTypeInst RestType, MachineInstr &I,
394 GL::GLSLExtInst GLInst, bool setMIFlags = true,
395 bool useMISrc = true,
396 ArrayRef<Register> SrcRegs = {}) const;
397 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
398 CL::OpenCLExtInst CLInst, bool setMIFlags = true,
399 bool useMISrc = true,
400 ArrayRef<Register> SrcRegs = {}) const;
401 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
402 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst,
403 bool setMIFlags = true, bool useMISrc = true,
404 ArrayRef<Register> SrcRegs = {}) const;
405 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
406 const ExtInstList &ExtInsts, bool setMIFlags = true,
407 bool useMISrc = true,
408 ArrayRef<Register> SrcRegs = {}) const;
409
410 bool selectLog10(Register ResVReg, SPIRVTypeInst ResType,
411 MachineInstr &I) const;
412
413 bool selectFpowi(Register ResVReg, SPIRVTypeInst ResType,
414 MachineInstr &I) const;
415
416 bool selectSaturate(Register ResVReg, SPIRVTypeInst ResType,
417 MachineInstr &I) const;
418
419 bool selectWaveOpInst(Register ResVReg, SPIRVTypeInst ResType,
420 MachineInstr &I, unsigned Opcode) const;
421
422 bool selectBarrierInst(MachineInstr &I, unsigned Scope, unsigned MemSem,
423 bool WithGroupSync) const;
424
425 bool selectWaveActiveCountBits(Register ResVReg, SPIRVTypeInst ResType,
426 MachineInstr &I) const;
427
428 bool selectWaveActiveAllEqual(Register ResVReg, SPIRVTypeInst ResType,
429 MachineInstr &I) const;
430
431 bool selectUnmergeValues(MachineInstr &I) const;
432
433 bool selectHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
434 MachineInstr &I) const;
435
436 bool selectCounterHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
437 MachineInstr &I) const;
438
439 bool selectReadImageIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
440 MachineInstr &I) const;
441 bool selectGetDimensionsIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
442 MachineInstr &I) const;
443 bool selectGetDimensionsLevelsIntrinsic(Register &ResVReg,
444 SPIRVTypeInst ResType,
445 MachineInstr &I) const;
446 bool selectGetDimensionsMSIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
447 MachineInstr &I) const;
448 bool
449 selectImageQuerySize(Register ImageReg, Register &ResVReg, MachineInstr &I,
450 std::optional<Register> LodReg = std::nullopt) const;
451 bool selectSampleBasicIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
452 MachineInstr &I) const;
453 bool selectCalculateLodIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
454 MachineInstr &I) const;
455 bool selectSampleBiasIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
456 MachineInstr &I) const;
457 bool selectSampleGradIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
458 MachineInstr &I) const;
459 bool selectSampleLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
460 MachineInstr &I) const;
461 bool selectLoadLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
462 MachineInstr &I) const;
463 bool selectSampleCmpIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
464 MachineInstr &I) const;
465 bool selectSampleCmpLevelZeroIntrinsic(Register &ResVReg,
466 SPIRVTypeInst ResType,
467 MachineInstr &I) const;
468 bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
469 MachineInstr &I) const;
470 bool selectImageWriteIntrinsic(MachineInstr &I) const;
471 bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
472 MachineInstr &I) const;
473 bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
474 MachineInstr &I) const;
475 bool selectResourceNonUniformIndex(Register &ResVReg, SPIRVTypeInst ResType,
476 MachineInstr &I) const;
477 bool selectModf(Register ResVReg, SPIRVTypeInst ResType,
478 MachineInstr &I) const;
479 bool selectUpdateCounter(Register &ResVReg, SPIRVTypeInst ResType,
480 MachineInstr &I) const;
481 bool selectFrexp(Register ResVReg, SPIRVTypeInst ResType,
482 MachineInstr &I) const;
483
484 bool selectCopySign(Register ResVReg, SPIRVTypeInst ResType,
485 MachineInstr &I) const;
486
487 bool selectLdexp(Register ResVReg, SPIRVTypeInst ResType,
488 MachineInstr &I) const;
489 bool selectSincos(Register ResVReg, SPIRVTypeInst ResType,
490 MachineInstr &I) const;
491 bool selectExp10(Register ResVReg, SPIRVTypeInst ResType,
492 MachineInstr &I) const;
493 bool selectDerivativeInst(Register ResVReg, SPIRVTypeInst ResType,
494 MachineInstr &I, const unsigned DPdOpCode) const;
495 // Utilities
496 Register buildI32Constant(uint32_t Val, MachineInstr &I,
497 SPIRVTypeInst ResType = nullptr) const;
498 Register buildI32ConstantInEntryBlock(uint32_t Val, MachineInstr &I,
499 SPIRVTypeInst ResType = nullptr) const;
500
501 Register buildZerosVal(SPIRVTypeInst ResType, MachineInstr &I) const;
502 bool isScalarOrVectorIntConstantZero(Register Reg) const;
503 Register buildZerosValF(SPIRVTypeInst ResType, MachineInstr &I) const;
504 Register buildOnesVal(bool AllOnes, SPIRVTypeInst ResType,
505 MachineInstr &I) const;
506 Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &I) const;
507
508 bool wrapIntoSpecConstantOp(MachineInstr &I,
509 SmallVector<Register> &CompositeArgs) const;
510
511 Register getUcharPtrTypeReg(MachineInstr &I,
512 SPIRV::StorageClass::StorageClass SC) const;
513 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
514 Register Src, Register DestType,
515 uint32_t Opcode) const;
516 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
517 SPIRVTypeInst SrcPtrTy) const;
518 Register buildPointerToResource(SPIRVTypeInst ResType,
519 SPIRV::StorageClass::StorageClass SC,
520 uint32_t Set, uint32_t Binding,
521 uint32_t ArraySize, Register IndexReg,
522 StringRef Name,
523 MachineIRBuilder MIRBuilder) const;
524 SPIRVTypeInst widenTypeToVec4(SPIRVTypeInst Type, MachineInstr &I) const;
525 bool extractSubvector(Register &ResVReg, SPIRVTypeInst ResType,
526 Register &ReadReg, MachineInstr &InsertionPoint) const;
527 bool generateImageReadOrFetch(Register &ResVReg, SPIRVTypeInst ResType,
528 Register ImageReg, Register IdxReg,
529 DebugLoc Loc, MachineInstr &Pos,
530 const ImageOperands *ImOps = nullptr) const;
531 bool generateSampleImage(Register ResVReg, SPIRVTypeInst ResType,
532 Register ImageReg, Register SamplerReg,
533 Register CoordinateReg, const ImageOperands &ImOps,
534 DebugLoc Loc, MachineInstr &I) const;
535 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
536 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
537 Register ResVReg, SPIRVTypeInst ResType,
538 MachineInstr &I) const;
539 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
540 Register ResVReg, SPIRVTypeInst ResType,
541 MachineInstr &I) const;
542 bool loadHandleBeforePosition(Register &HandleReg, SPIRVTypeInst ResType,
543 GIntrinsic &HandleDef, MachineInstr &Pos) const;
544 void decorateUsesAsNonUniform(Register &NonUniformReg) const;
545 bool errorIfInstrOutsideShader(MachineInstr &I) const;
546
547 std::optional<SplitParts> splitEvenOddLanes(Register PopCountReg,
548 unsigned ComponentCount,
549 MachineInstr &I,
550 SPIRVTypeInst I32Type) const;
551
552 bool
553 handle64BitOverflow(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
554 Register SrcReg, unsigned int Opcode,
555 std::function<bool(Register, SPIRVTypeInst,
556 MachineInstr &, Register, unsigned)>
557 CallbackFunction) const;
558};
559
560bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
561 const TargetExtType *TET = cast<TargetExtType>(HandleType);
562 if (TET->getTargetExtName() == "spirv.Image") {
563 return false;
564 }
565 assert(TET->getTargetExtName() == "spirv.SignedImage");
566 return TET->getTypeParameter(0)->isIntegerTy();
567}
568} // end anonymous namespace
569
570#define GET_GLOBALISEL_IMPL
571#include "SPIRVGenGlobalISel.inc"
572#undef GET_GLOBALISEL_IMPL
573
574SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
575 const SPIRVSubtarget &ST,
576 const RegisterBankInfo &RBI)
577 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
578 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
579 MRI(nullptr),
580#define GET_GLOBALISEL_PREDICATES_INIT
581#include "SPIRVGenGlobalISel.inc"
582#undef GET_GLOBALISEL_PREDICATES_INIT
583#define GET_GLOBALISEL_TEMPORARIES_INIT
584#include "SPIRVGenGlobalISel.inc"
585#undef GET_GLOBALISEL_TEMPORARIES_INIT
586{
587}
588
589void SPIRVInstructionSelector::setupMF(MachineFunction &MF,
590 GISelValueTracking *VT,
591 CodeGenCoverage *CoverageInfo,
592 ProfileSummaryInfo *PSI,
593 BlockFrequencyInfo *BFI) {
594 MRI = &MF.getRegInfo();
595 GR.setCurrentFunc(MF);
596 InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
597}
598
599// Ensure that register classes correspond to pattern matching rules.
600void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
601 if (HasVRegsReset == &MF)
602 return;
603 HasVRegsReset = &MF;
604
605 MachineRegisterInfo &MRI = MF.getRegInfo();
606 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
607 Register Reg = Register::index2VirtReg(I);
608 LLT RegType = MRI.getType(Reg);
609 if (RegType.isScalar())
610 MRI.setType(Reg, LLT::scalar(64));
611 else if (RegType.isPointer())
612 MRI.setType(Reg, LLT::pointer(0, 64));
613 else if (RegType.isVector())
614 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64)));
615 }
616 for (const auto &MBB : MF) {
617 for (const auto &MI : MBB) {
618 if (isPreISelGenericOpcode(MI.getOpcode()))
619 GR.erase(&MI);
620 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
621 continue;
622
623 Register DstReg = MI.getOperand(0).getReg();
624 LLT DstType = MRI.getType(DstReg);
625 Register SrcReg = MI.getOperand(1).getReg();
626 LLT SrcType = MRI.getType(SrcReg);
627 if (DstType != SrcType)
628 MRI.setType(DstReg, MRI.getType(SrcReg));
629
630 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg);
631 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg);
632 if (DstRC != SrcRC && SrcRC)
633 MRI.setRegClass(DstReg, SrcRC);
634 }
635 }
636}
637
638// Return true if the MachineInstr represents a constant register
639static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef) {
640
641 SmallVector<MachineInstr *> Stack = {OpDef};
642 SmallPtrSet<MachineInstr *, 4> Visited;
643
644 while (!Stack.empty()) {
645 MachineInstr *MI = Stack.pop_back_val();
646 MI = passCopy(MI, MRI);
647 if (!Visited.insert(MI).second)
648 continue;
649 switch (MI->getOpcode()) {
650 case TargetOpcode::G_INTRINSIC:
651 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
652 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
653 GIntrinsic *GIntr = cast<GIntrinsic>(MI);
654 unsigned IntrID = GIntr->getIntrinsicID();
655 if (IntrID != Intrinsic::spv_const_composite &&
656 IntrID != Intrinsic::spv_undef && IntrID != Intrinsic::spv_poison)
657 return false;
658 continue;
659 }
660 case TargetOpcode::G_BUILD_VECTOR:
661 case TargetOpcode::G_SPLAT_VECTOR:
662 for (unsigned i = OpDef->getNumExplicitDefs();
663 i < OpDef->getNumOperands(); i++) {
664 if (!OpDef->getOperand(i).isReg())
665 continue;
666 MachineInstr *OpNestedDef =
667 MRI->getVRegDef(OpDef->getOperand(i).getReg());
668 Stack.push_back(OpNestedDef);
669 }
670 continue;
671 case TargetOpcode::G_CONSTANT:
672 case TargetOpcode::G_FCONSTANT:
673 case TargetOpcode::G_IMPLICIT_DEF:
674 case SPIRV::OpConstantTrue:
675 case SPIRV::OpConstantFalse:
676 case SPIRV::OpConstantI:
677 case SPIRV::OpConstantF:
678 case SPIRV::OpConstantComposite:
679 case SPIRV::OpConstantCompositeContinuedINTEL:
680 case SPIRV::OpConstantSampler:
681 case SPIRV::OpConstantNull:
682 case SPIRV::OpUndef:
683 case SPIRV::OpPoisonKHR:
684 case SPIRV::OpConstantFunctionPointerINTEL:
685 continue;
686 default:
687 return false;
688 }
689 }
690 return true;
691}
692
693// Return true if the virtual register represents a constant
694static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
695 if (MachineInstr *OpDef = MRI->getVRegDef(OpReg))
696 return isConstReg(MRI, OpDef);
697 return false;
698}
699
700// TODO(168736): We should make this either a flag in tabelgen
701// or reduce our dependence on the global registry, so we can remove this
702// function. It can easily be missed when new intrinsics are added.
703
704// Most SPIR-V intrinsics are considered to have side-effects in their tablegen
705// definition because they are referenced in the global registry. This is a list
706// of intrinsics that have no side effects other than their references in the
707// global registry.
708static bool intrinsicHasSideEffects(Intrinsic::ID ID) {
709 switch (ID) {
710 // This is not an exhaustive list and may need to be updated.
711 case Intrinsic::spv_all:
712 case Intrinsic::spv_alloca:
713 case Intrinsic::spv_any:
714 case Intrinsic::spv_bitcast:
715 case Intrinsic::spv_const_composite:
716 case Intrinsic::spv_degrees:
717 case Intrinsic::spv_distance:
718 case Intrinsic::spv_extractelt:
719 case Intrinsic::spv_extractv:
720 case Intrinsic::spv_faceforward:
721 case Intrinsic::spv_fdot:
722 case Intrinsic::spv_firstbitlow:
723 case Intrinsic::spv_firstbitshigh:
724 case Intrinsic::spv_firstbituhigh:
725 case Intrinsic::spv_frac:
726 case Intrinsic::spv_gep:
727 case Intrinsic::spv_global_offset:
728 case Intrinsic::spv_global_size:
729 case Intrinsic::spv_group_id:
730 case Intrinsic::spv_insertelt:
731 case Intrinsic::spv_insertv:
732 case Intrinsic::spv_isinf:
733 case Intrinsic::spv_isnan:
734 case Intrinsic::spv_isfinite:
735 case Intrinsic::spv_isnormal:
736 case Intrinsic::spv_lerp:
737 case Intrinsic::spv_length:
738 case Intrinsic::spv_normalize:
739 case Intrinsic::spv_num_subgroups:
740 case Intrinsic::spv_num_workgroups:
741 case Intrinsic::spv_ptrcast:
742 case Intrinsic::spv_radians:
743 case Intrinsic::spv_reflect:
744 case Intrinsic::spv_refract:
745 case Intrinsic::spv_resource_getbasepointer:
746 case Intrinsic::spv_resource_getpointer:
747 case Intrinsic::spv_resource_handlefrombinding:
748 case Intrinsic::spv_resource_handlefromimplicitbinding:
749 case Intrinsic::spv_resource_nonuniformindex:
750 case Intrinsic::spv_resource_sample:
751 case Intrinsic::spv_rsqrt:
752 case Intrinsic::spv_saturate:
753 case Intrinsic::spv_sdot:
754 case Intrinsic::spv_sign:
755 case Intrinsic::spv_smoothstep:
756 case Intrinsic::spv_subgroup_id:
757 case Intrinsic::spv_subgroup_local_invocation_id:
758 case Intrinsic::spv_subgroup_max_size:
759 case Intrinsic::spv_subgroup_size:
760 case Intrinsic::spv_thread_id:
761 case Intrinsic::spv_thread_id_in_group:
762 case Intrinsic::spv_udot:
763 case Intrinsic::spv_undef:
764 case Intrinsic::spv_value_md:
765 case Intrinsic::spv_workgroup_size:
766 return false;
767 default:
768 return true;
769 }
770}
771
772// TODO(168736): We should make this either a flag in tabelgen
773// or reduce our dependence on the global registry, so we can remove this
774// function. It can easily be missed when new intrinsics are added.
775static bool isOpcodeWithNoSideEffects(unsigned Opcode) {
776 switch (Opcode) {
777 case SPIRV::OpTypeVoid:
778 case SPIRV::OpTypeBool:
779 case SPIRV::OpTypeInt:
780 case SPIRV::OpTypeFloat:
781 case SPIRV::OpTypeVector:
782 case SPIRV::OpTypeVectorIdEXT:
783 case SPIRV::OpTypeMatrix:
784 case SPIRV::OpTypeImage:
785 case SPIRV::OpTypeSampler:
786 case SPIRV::OpTypeSampledImage:
787 case SPIRV::OpTypeArray:
788 case SPIRV::OpTypeRuntimeArray:
789 case SPIRV::OpTypeStruct:
790 case SPIRV::OpTypeOpaque:
791 case SPIRV::OpTypePointer:
792 case SPIRV::OpTypeFunction:
793 case SPIRV::OpTypeEvent:
794 case SPIRV::OpTypeDeviceEvent:
795 case SPIRV::OpTypeReserveId:
796 case SPIRV::OpTypeQueue:
797 case SPIRV::OpTypePipe:
798 case SPIRV::OpTypeForwardPointer:
799 case SPIRV::OpTypePipeStorage:
800 case SPIRV::OpTypeNamedBarrier:
801 case SPIRV::OpTypeAccelerationStructureNV:
802 case SPIRV::OpTypeCooperativeMatrixNV:
803 case SPIRV::OpTypeCooperativeMatrixKHR:
804 return true;
805 default:
806 return false;
807 }
808}
809
810bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) {
811 // If there are no definitions, then assume there is some other
812 // side-effect that makes this instruction live.
813 if (MI.getNumDefs() == 0)
814 return false;
815
816 for (const auto &MO : MI.all_defs()) {
817 Register Reg = MO.getReg();
818 if (Reg.isPhysical()) {
819 LLVM_DEBUG(dbgs() << "Not dead: def of physical register " << Reg);
820 return false;
821 }
822 for (const auto &UseMI : MRI.use_nodbg_instructions(Reg)) {
823 if (UseMI.getOpcode() != SPIRV::OpName) {
824 LLVM_DEBUG(dbgs() << "Not dead: def " << MO << " has use in " << UseMI);
825 return false;
826 }
827 }
828 }
829
830 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
831 MI.isLifetimeMarker()) {
832 LLVM_DEBUG(
833 dbgs()
834 << "Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n");
835 return false;
836 }
837 if (MI.isPHI()) {
838 LLVM_DEBUG(dbgs() << "Dead: Phi instruction with no uses.\n");
839 return true;
840 }
841
842 // It is possible that the only side effect is that the instruction is
843 // referenced in the global registry. If that is the only side effect, the
844 // intrinsic is dead.
845 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
846 MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
847 const auto &Intr = cast<GIntrinsic>(MI);
848 if (!intrinsicHasSideEffects(Intr.getIntrinsicID())) {
849 LLVM_DEBUG(dbgs() << "Dead: Intrinsic with no real side effects.\n");
850 return true;
851 }
852 }
853
854 if (MI.mayStore() || MI.isCall() ||
855 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
856 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) {
857 LLVM_DEBUG(dbgs() << "Not dead: instruction has side effects.\n");
858 return false;
859 }
860
861 if (isPreISelGenericOpcode(MI.getOpcode())) {
862 // TODO: Is there a generic way to check if the opcode has side effects?
863 LLVM_DEBUG(dbgs() << "Dead: Generic opcode with no uses.\n");
864 return true;
865 }
866
867 if (isOpcodeWithNoSideEffects(MI.getOpcode())) {
868 LLVM_DEBUG(dbgs() << "Dead: known opcode with no side effects\n");
869 return true;
870 }
871
872 return false;
873}
874
875void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &MI) const {
876 // Delete the OpName that uses the result if there is one.
877 for (const auto &MO : MI.all_defs()) {
878 Register Reg = MO.getReg();
879 if (Reg.isPhysical())
880 continue;
881 SmallVector<MachineInstr *, 4> UselessOpNames;
882 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
883 assert(UseMI.getOpcode() == SPIRV::OpName &&
884 "There is still a use of the dead function.");
885 UselessOpNames.push_back(&UseMI);
886 }
887 for (MachineInstr *OpNameMI : UselessOpNames) {
888 GR.invalidateMachineInstr(OpNameMI);
889 OpNameMI->eraseFromParent();
890 }
891 }
892}
893
894void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &MI) const {
895 salvageDebugInfo(*MRI, MI);
896 GR.invalidateMachineInstr(&MI);
897 removeOpNamesForDeadMI(MI);
898 MI.eraseFromParent();
899}
900
901bool SPIRVInstructionSelector::select(MachineInstr &I) {
902 resetVRegsType(*I.getParent()->getParent());
903
904 assert(I.getParent() && "Instruction should be in a basic block!");
905 assert(I.getParent()->getParent() && "Instruction should be in a function!");
906
907 LLVM_DEBUG(dbgs() << "Checking if instruction is dead: " << I;);
908 if (isDead(I, *MRI)) {
909 LLVM_DEBUG(dbgs() << "Instruction is dead.\n");
910 removeDeadInstruction(I);
911 return true;
912 }
913
914 Register Opcode = I.getOpcode();
915 // If it's not a GMIR instruction, we've selected it already.
916 if (!isPreISelGenericOpcode(Opcode)) {
917 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
918 Register DstReg = I.getOperand(0).getReg();
919 Register SrcReg = I.getOperand(1).getReg();
920 auto *Def = MRI->getVRegDef(SrcReg);
921 if (isTypeFoldingSupported(Def->getOpcode()) &&
922 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
923 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
924 if (Def->getOpcode() == TargetOpcode::G_SELECT) {
925 Register SelectDstReg = Def->getOperand(0).getReg();
926 bool SuccessToSelectSelect [[maybe_unused]] = selectSelect(
927 SelectDstReg, GR.getSPIRVTypeForVReg(SelectDstReg), *Def);
928 assert(SuccessToSelectSelect);
929 GR.invalidateMachineInstr(Def);
930 Def->eraseFromParent();
931 MRI->replaceRegWith(DstReg, SelectDstReg);
932 GR.invalidateMachineInstr(&I);
933 I.eraseFromParent();
934 return true;
935 }
936
937 bool Res = selectImpl(I, *CoverageInfo);
938 LLVM_DEBUG({
939 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
940 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
941 I.print(dbgs());
942 }
943 });
944 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
945 if (Res) {
946 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI))
947 DeadMIs.insert(Def);
948 return Res;
949 }
950 }
951 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg));
952 MRI->replaceRegWith(SrcReg, DstReg);
953 GR.invalidateMachineInstr(&I);
954 I.eraseFromParent();
955 return true;
956 } else if (I.getNumDefs() == 1) {
957 // Make all vregs 64 bits (for SPIR-V IDs).
958 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64));
959 }
960 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
961 return true;
962 }
963
964 if (DeadMIs.contains(&I)) {
965 // if the instruction has been already made dead by folding it away
966 // erase it
967 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
968 removeDeadInstruction(I);
969 DeadMIs.erase(&I);
970 return true;
971 }
972
973 if (I.getNumOperands() != I.getNumExplicitOperands()) {
974 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
975 return false;
976 }
977
978 // Common code for getting return reg+type, and removing selected instr
979 // from parent occurs here. Instr-specific selection happens in spvSelect().
980 bool HasDefs = I.getNumDefs() > 0;
981 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
982 SPIRVTypeInst ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
983 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
984 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
985 if (spvSelect(ResVReg, ResType, I)) {
986 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
987 for (unsigned i = 0; i < I.getNumDefs(); ++i)
988 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64));
989 GR.invalidateMachineInstr(&I);
990 I.eraseFromParent();
991 return true;
992 }
993 return false;
994}
995
996static bool mayApplyGenericSelection(unsigned Opcode) {
997 switch (Opcode) {
998 case TargetOpcode::G_CONSTANT:
999 case TargetOpcode::G_FCONSTANT:
1000 return false;
1001 }
1002 return isTypeFoldingSupported(Opcode);
1003}
1004
1005bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
1006 MachineInstr &I) const {
1007 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg);
1008 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg);
1009 if (DstRC != SrcRC && SrcRC)
1010 MRI->setRegClass(DestReg, SrcRC);
1011 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1012 .addDef(DestReg)
1013 .addUse(SrcReg)
1014 .constrainAllUses(TII, TRI, RBI);
1015 return true;
1016}
1017
1018bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
1019 SPIRVTypeInst ResType,
1020 MachineInstr &I) const {
1021 const unsigned Opcode = I.getOpcode();
1022 if (mayApplyGenericSelection(Opcode))
1023 return selectImpl(I, *CoverageInfo);
1024 switch (Opcode) {
1025 case TargetOpcode::G_CONSTANT:
1026 case TargetOpcode::G_FCONSTANT:
1027 return selectConst(ResVReg, ResType, I);
1028 case TargetOpcode::G_GLOBAL_VALUE:
1029 return selectGlobalValue(ResVReg, I);
1030 case TargetOpcode::G_IMPLICIT_DEF:
1031 return selectOpUndef(ResVReg, ResType, I);
1032 case TargetOpcode::G_FREEZE:
1033 return selectFreeze(ResVReg, ResType, I);
1034
1035 case TargetOpcode::G_INTRINSIC:
1036 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1037 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1038 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
1039 return selectIntrinsic(ResVReg, ResType, I);
1040 case TargetOpcode::G_BITREVERSE:
1041 return selectBitreverse(ResVReg, ResType, I);
1042
1043 case TargetOpcode::G_BUILD_VECTOR:
1044 return selectBuildVector(ResVReg, ResType, I);
1045 case TargetOpcode::G_SPLAT_VECTOR:
1046 return selectSplatVector(ResVReg, ResType, I);
1047 case TargetOpcode::G_CONCAT_VECTORS:
1048 return selectConcatVectors(ResVReg, ResType, I);
1049
1050 case TargetOpcode::G_SHUFFLE_VECTOR: {
1051 MachineBasicBlock &BB = *I.getParent();
1052 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
1053 .addDef(ResVReg)
1054 .addUse(GR.getSPIRVTypeID(ResType))
1055 .addUse(I.getOperand(1).getReg())
1056 .addUse(I.getOperand(2).getReg());
1057 for (auto V : I.getOperand(3).getShuffleMask())
1058 MIB.addImm(V);
1059 MIB.constrainAllUses(TII, TRI, RBI);
1060 return true;
1061 }
1062 case TargetOpcode::G_MEMMOVE:
1063 case TargetOpcode::G_MEMCPY:
1064 case TargetOpcode::G_MEMCPY_INLINE:
1065 case TargetOpcode::G_MEMSET:
1066 case TargetOpcode::G_MEMSET_INLINE:
1067 return selectMemOperation(ResVReg, I);
1068
1069 case TargetOpcode::G_ICMP:
1070 return selectICmp(ResVReg, ResType, I);
1071 case TargetOpcode::G_FCMP:
1072 return selectFCmp(ResVReg, ResType, I);
1073
1074 case TargetOpcode::G_FRAME_INDEX:
1075 return selectFrameIndex(ResVReg, ResType, I);
1076
1077 case TargetOpcode::G_LOAD:
1078 return selectLoad(ResVReg, ResType, I);
1079 case TargetOpcode::G_STORE:
1080 return selectStore(I);
1081
1082 case TargetOpcode::G_BR:
1083 return selectBranch(I);
1084 case TargetOpcode::G_BRCOND:
1085 return selectBranchCond(I);
1086
1087 case TargetOpcode::G_PHI:
1088 return selectPhi(ResVReg, I);
1089
1090 case TargetOpcode::G_FPTOSI:
1091 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
1092 case TargetOpcode::G_FPTOUI:
1093 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
1094
1095 case TargetOpcode::G_FPTOSI_SAT:
1096 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
1097 case TargetOpcode::G_FPTOUI_SAT:
1098 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
1099
1100 case TargetOpcode::G_SITOFP:
1101 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
1102 case TargetOpcode::G_UITOFP:
1103 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
1104
1105 case TargetOpcode::G_CTPOP:
1106 return selectPopCount(ResVReg, ResType, I, SPIRV::OpBitCount);
1107 case TargetOpcode::G_SMIN:
1108 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
1109 case TargetOpcode::G_UMIN:
1110 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
1111
1112 case TargetOpcode::G_SMAX:
1113 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
1114 case TargetOpcode::G_UMAX:
1115 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
1116
1117 case TargetOpcode::G_SCMP:
1118 return selectSUCmp(ResVReg, ResType, I, true);
1119 case TargetOpcode::G_UCMP:
1120 return selectSUCmp(ResVReg, ResType, I, false);
1121 case TargetOpcode::G_LROUND:
1122 case TargetOpcode::G_LLROUND: {
1123 Register regForLround =
1124 MRI->createVirtualRegister(MRI->getRegClass(ResVReg), "lround");
1125 MRI->setRegClass(regForLround, &SPIRV::iIDRegClass);
1126 GR.assignSPIRVTypeToVReg(GR.getSPIRVTypeForVReg(I.getOperand(1).getReg()),
1127 regForLround, *(I.getParent()->getParent()));
1128 selectExtInst(regForLround, GR.getSPIRVTypeForVReg(regForLround), I,
1129 CL::round, GL::Round, /* setMIFlags */ false);
1130 MachineBasicBlock &BB = *I.getParent();
1131 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConvertFToS))
1132 .addDef(ResVReg)
1133 .addUse(GR.getSPIRVTypeID(ResType))
1134 .addUse(regForLround);
1135 MIB.constrainAllUses(TII, TRI, RBI);
1136 return true;
1137 }
1138 case TargetOpcode::G_STRICT_FMA:
1139 case TargetOpcode::G_FMA: {
1140 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_fma)) {
1141 MachineBasicBlock &BB = *I.getParent();
1142 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpFmaKHR))
1143 .addDef(ResVReg)
1144 .addUse(GR.getSPIRVTypeID(ResType))
1145 .addUse(I.getOperand(1).getReg())
1146 .addUse(I.getOperand(2).getReg())
1147 .addUse(I.getOperand(3).getReg())
1148 .setMIFlags(I.getFlags());
1149 MIB.constrainAllUses(TII, TRI, RBI);
1150 return true;
1151 }
1152 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
1153 }
1154
1155 case TargetOpcode::G_FLDEXP:
1156 case TargetOpcode::G_STRICT_FLDEXP:
1157 return selectLdexp(ResVReg, ResType, I);
1158
1159 case TargetOpcode::G_FPOW:
1160 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
1161 case TargetOpcode::G_FPOWI:
1162 return selectFpowi(ResVReg, ResType, I);
1163
1164 case TargetOpcode::G_FEXP:
1165 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
1166 case TargetOpcode::G_FEXP2:
1167 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
1168 case TargetOpcode::G_FEXP10:
1169 return selectExp10(ResVReg, ResType, I);
1170
1171 case TargetOpcode::G_FMODF:
1172 return selectModf(ResVReg, ResType, I);
1173 case TargetOpcode::G_FSINCOS:
1174 return selectSincos(ResVReg, ResType, I);
1175
1176 case TargetOpcode::G_FLOG:
1177 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
1178 case TargetOpcode::G_FLOG2:
1179 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
1180 case TargetOpcode::G_FLOG10:
1181 return selectLog10(ResVReg, ResType, I);
1182
1183 case TargetOpcode::G_FABS:
1184 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
1185 case TargetOpcode::G_ABS:
1186 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
1187
1188 case TargetOpcode::G_FMINNUM:
1189 case TargetOpcode::G_FMINIMUM:
1190 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin);
1191 case TargetOpcode::G_FMAXNUM:
1192 case TargetOpcode::G_FMAXIMUM:
1193 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax);
1194
1195 case TargetOpcode::G_FCOPYSIGN:
1196 return selectCopySign(ResVReg, ResType, I);
1197
1198 case TargetOpcode::G_FCEIL:
1199 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
1200 case TargetOpcode::G_FFLOOR:
1201 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
1202
1203 case TargetOpcode::G_FCOS:
1204 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
1205 case TargetOpcode::G_FSIN:
1206 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
1207 case TargetOpcode::G_FTAN:
1208 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan);
1209 case TargetOpcode::G_FACOS:
1210 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos);
1211 case TargetOpcode::G_FASIN:
1212 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin);
1213 case TargetOpcode::G_FATAN:
1214 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan);
1215 case TargetOpcode::G_FATAN2:
1216 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2);
1217 case TargetOpcode::G_FCOSH:
1218 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh);
1219 case TargetOpcode::G_FSINH:
1220 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh);
1221 case TargetOpcode::G_FTANH:
1222 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh);
1223
1224 case TargetOpcode::G_STRICT_FSQRT:
1225 case TargetOpcode::G_FSQRT:
1226 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
1227
1228 case TargetOpcode::G_CTTZ:
1229 case TargetOpcode::G_CTTZ_ZERO_POISON:
1230 return selectExtInst(ResVReg, ResType, I, CL::ctz);
1231 case TargetOpcode::G_CTLZ:
1232 case TargetOpcode::G_CTLZ_ZERO_POISON:
1233 return selectExtInst(ResVReg, ResType, I, CL::clz);
1234
1235 case TargetOpcode::G_INTRINSIC_ROUND:
1236 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
1237 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1238 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
1239 case TargetOpcode::G_INTRINSIC_TRUNC:
1240 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
1241 case TargetOpcode::G_FRINT:
1242 case TargetOpcode::G_FNEARBYINT:
1243 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
1244
1245 case TargetOpcode::G_SMULH:
1246 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
1247 case TargetOpcode::G_UMULH:
1248 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
1249
1250 case TargetOpcode::G_SADDSAT:
1251 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat);
1252 case TargetOpcode::G_UADDSAT:
1253 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat);
1254 case TargetOpcode::G_SSUBSAT:
1255 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat);
1256 case TargetOpcode::G_USUBSAT:
1257 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
1258
1259 case TargetOpcode::G_FFREXP:
1260 return selectFrexp(ResVReg, ResType, I);
1261
1262 case TargetOpcode::G_UADDO:
1263 return selectOverflowArith(ResVReg, ResType, I,
1264 isVectorType(ResType) ? SPIRV::OpIAddCarryV
1265 : SPIRV::OpIAddCarryS);
1266 case TargetOpcode::G_USUBO:
1267 return selectOverflowArith(ResVReg, ResType, I,
1268 isVectorType(ResType) ? SPIRV::OpISubBorrowV
1269 : SPIRV::OpISubBorrowS);
1270 case TargetOpcode::G_UMULO:
1271 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended);
1272 case TargetOpcode::G_SMULO:
1273 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended);
1274
1275 case TargetOpcode::G_SEXT:
1276 return selectExt(ResVReg, ResType, I, true);
1277 case TargetOpcode::G_ANYEXT:
1278 case TargetOpcode::G_ZEXT:
1279 return selectExt(ResVReg, ResType, I, false);
1280 case TargetOpcode::G_TRUNC:
1281 return selectTrunc(ResVReg, ResType, I);
1282 case TargetOpcode::G_FPTRUNC:
1283 case TargetOpcode::G_FPEXT:
1284 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
1285
1286 case TargetOpcode::G_PTRTOINT:
1287 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
1288 case TargetOpcode::G_INTTOPTR:
1289 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
1290 case TargetOpcode::G_BITCAST:
1291 return selectBitcast(ResVReg, ResType, I);
1292 case TargetOpcode::G_ADDRSPACE_CAST:
1293 return selectAddrSpaceCast(ResVReg, ResType, I);
1294 case TargetOpcode::G_PTRMASK:
1295 return selectPtrMask(ResVReg, ResType, I);
1296 case TargetOpcode::G_PTR_ADD: {
1297 // Currently, we get G_PTR_ADD only applied to global variables.
1298 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1299 Register GV = I.getOperand(1).getReg();
1300 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
1301 (void)II;
1302 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1303 (*II).getOpcode() == TargetOpcode::COPY ||
1304 (*II).getOpcode() == SPIRV::OpVariable ||
1305 (*II).getOpcode() == SPIRV::OpUntypedVariableKHR) &&
1306 getImm(I.getOperand(2), MRI));
1307 // It may be the initialization of a global variable.
1308 bool IsGVInit = false;
1309 for (MachineRegisterInfo::use_instr_iterator
1310 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()),
1311 UseEnd = MRI->use_instr_end();
1312 UseIt != UseEnd; UseIt = std::next(UseIt)) {
1313 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1314 (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
1315 (*UseIt).getOpcode() == SPIRV::OpVariable ||
1316 (*UseIt).getOpcode() == SPIRV::OpUntypedVariableKHR) {
1317 IsGVInit = true;
1318 break;
1319 }
1320 }
1321 MachineBasicBlock &BB = *I.getParent();
1322 // An untyped result needs OpUntypedInBoundsPtrAccessChainKHR, which spells
1323 // out the Base Type. The offset of G_PTR_ADD is a byte count, so the Base
1324 // Type is i8 and the offset is the Element index as is. The opcode is
1325 // picked by the result type alone because an untyped access chain accepts
1326 // a typed base while a typed access chain rejects an untyped result.
1327 SPIRVTypeInst GVType = GR.getSPIRVTypeForVReg(GV);
1328 const bool UseUntypedPointers =
1329 ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1330 if (UseUntypedPointers) {
1331 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1332 .addDef(ResVReg)
1333 .addUse(GR.getSPIRVTypeID(ResType))
1334 .addImm(static_cast<uint32_t>(
1335 SPIRV::Opcode::UntypedInBoundsPtrAccessChainKHR))
1336 .addUse(GR.getSPIRVTypeID(GR.getOrCreateSPIRVIntegerType(8, I, TII)))
1337 .addUse(GV)
1338 .addUse(I.getOperand(2).getReg())
1339 .constrainAllUses(TII, TRI, RBI);
1340 return true;
1341 }
1342 if (!IsGVInit) {
1343 SPIRVTypeInst GVPointeeType = GR.getPointeeType(GVType);
1344 SPIRVTypeInst ResPointeeType = GR.getPointeeType(ResType);
1345 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
1346 // Build a new virtual register that is associated with the required
1347 // data type.
1348 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV));
1349 MRI->setRegClass(NewVReg, MRI->getRegClass(GV));
1350 // Having a correctly typed base we are ready to build the actually
1351 // required GEP. It may not be a constant though, because all Operands
1352 // of OpSpecConstantOp is to originate from other const instructions,
1353 // and only the AccessChain named opcodes accept a global OpVariable
1354 // instruction. We can't use an AccessChain opcode because of the type
1355 // mismatch between result and base types.
1356 if (!GR.isBitcastCompatible(ResType, GVType))
1357 return diagnoseUnsupported(
1358 I, "incompatible result and operand types in a bitcast");
1359 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
1360 MachineInstrBuilder MIB =
1361 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
1362 .addDef(NewVReg)
1363 .addUse(ResTypeReg)
1364 .addUse(GV);
1365 MIB.constrainAllUses(TII, TRI, RBI);
1366 BuildMI(BB, I, I.getDebugLoc(),
1367 TII.get(STI.isLogicalSPIRV() ? SPIRV::OpInBoundsAccessChain
1368 : SPIRV::OpInBoundsPtrAccessChain))
1369 .addDef(ResVReg)
1370 .addUse(ResTypeReg)
1371 .addUse(NewVReg)
1372 .addUse(I.getOperand(2).getReg())
1373 .constrainAllUses(TII, TRI, RBI);
1374 } else {
1375 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1376 .addDef(ResVReg)
1377 .addUse(GR.getSPIRVTypeID(ResType))
1378 .addImm(
1379 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
1380 .addUse(GV)
1381 .addUse(I.getOperand(2).getReg())
1382 .constrainAllUses(TII, TRI, RBI);
1383 }
1384 return true;
1385 }
1386 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
1387 // initialize a global variable with a constant expression (e.g., the test
1388 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
1389 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
1390 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1391 .addDef(ResVReg)
1392 .addUse(GR.getSPIRVTypeID(ResType))
1393 .addImm(static_cast<uint32_t>(
1394 SPIRV::Opcode::InBoundsPtrAccessChain))
1395 .addUse(GV)
1396 .addUse(Idx)
1397 .addUse(I.getOperand(2).getReg());
1398 MIB.constrainAllUses(TII, TRI, RBI);
1399 return true;
1400 }
1401
1402 case TargetOpcode::G_ATOMICRMW_OR:
1403 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
1404 case TargetOpcode::G_ATOMICRMW_ADD:
1405 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
1406 case TargetOpcode::G_ATOMICRMW_AND:
1407 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
1408 case TargetOpcode::G_ATOMICRMW_MAX:
1409 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
1410 case TargetOpcode::G_ATOMICRMW_MIN:
1411 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
1412 case TargetOpcode::G_ATOMICRMW_SUB:
1413 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
1414 case TargetOpcode::G_ATOMICRMW_XOR:
1415 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
1416 case TargetOpcode::G_ATOMICRMW_UMAX:
1417 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
1418 case TargetOpcode::G_ATOMICRMW_UMIN:
1419 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
1420 case TargetOpcode::G_ATOMICRMW_XCHG:
1421 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
1422
1423 case TargetOpcode::G_ATOMICRMW_FADD:
1424 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT);
1425 case TargetOpcode::G_ATOMICRMW_FSUB:
1426 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
1427 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT,
1428 isVectorType(ResType) ? SPIRV::OpFNegateV
1429 : SPIRV::OpFNegate);
1430 case TargetOpcode::G_ATOMICRMW_FMIN:
1431 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT);
1432 case TargetOpcode::G_ATOMICRMW_FMAX:
1433 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT);
1434
1435 case TargetOpcode::G_FENCE:
1436 return selectFence(I);
1437
1438 case TargetOpcode::G_STACKSAVE:
1439 return selectStackSave(ResVReg, ResType, I);
1440 case TargetOpcode::G_STACKRESTORE:
1441 return selectStackRestore(I);
1442
1443 case TargetOpcode::G_UNMERGE_VALUES:
1444 return selectUnmergeValues(I);
1445
1446 case TargetOpcode::G_TRAP:
1447 case TargetOpcode::G_UBSANTRAP:
1448 return selectTrap(I);
1449
1450 // Discard gen opcodes for intrinsics which we do not expect to actually
1451 // represent code after lowering or intrinsics which are not implemented but
1452 // should not crash when found in a customer's LLVM IR input.
1453 case TargetOpcode::DBG_LABEL:
1454 return true;
1455 case TargetOpcode::G_DEBUGTRAP:
1456 return selectDebugTrap(ResVReg, ResType, I);
1457 case TargetOpcode::G_PREFETCH:
1458 return selectPrefetch(I);
1459
1460 default:
1461 return false;
1462 }
1463}
1464
1465bool SPIRVInstructionSelector::selectDebugTrap(Register ResVReg,
1466 SPIRVTypeInst ResType,
1467 MachineInstr &I) const {
1468 unsigned Opcode = SPIRV::OpNop;
1469 MachineBasicBlock &BB = *I.getParent();
1470 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1471 .constrainAllUses(TII, TRI, RBI);
1472 return true;
1473}
1474
1475bool SPIRVInstructionSelector::selectPrefetch(MachineInstr &I) const {
1476 // llvm.prefetch is missing info about how many bytes to prefetch that
1477 // OpUntypedPrefetchKHR instruction requires, so just drop the instruction
1478 // in case untyped pointers are enabled since OCL prefetch cannot take
1479 // untyped pointers.
1480 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_untyped_pointers))
1481 return true;
1482
1483 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
1484 MachineIRBuilder MIRBuilder(I);
1485 const unsigned PointerSize = GR.getPointerSize();
1486 const SPIRVTypeInst PointerSizeType =
1487 GR.getOrCreateSPIRVIntegerType(PointerSize, MIRBuilder);
1488 bool ZeroAsNull = !STI.isShader();
1489
1490 // Assume that a single element needs prefetch.
1491 Register ConstIntOne =
1492 GR.getOrCreateConstInt(1, I, PointerSizeType, TII, ZeroAsNull);
1493
1494 Register AddrVal = I.getOperand(0).getReg();
1495 Register ExtReg = MRI->createVirtualRegister(
1496 GR.getRegClass(GR.getOpTypeVoid(MIRBuilder)));
1497 return selectExtInst(ExtReg, GR.getOpTypeVoid(MIRBuilder), I, CL::prefetch,
1498 /*setMIFlags=*/false, /*useMSSrc=*/false,
1499 {AddrVal, ConstIntOne});
1500 }
1501 return true;
1502}
1503
1504bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1505 SPIRVTypeInst ResType,
1506 MachineInstr &I,
1507 GL::GLSLExtInst GLInst,
1508 bool setMIFlags, bool useMISrc,
1509 ArrayRef<Register> SrcRegs) const {
1510 if (!STI.canUseExtInstSet(
1511 SPIRV::InstructionSet::InstructionSet::GLSL_std_450))
1512 return diagnoseUnsupported(
1513 I,
1514 "this instruction is only supported with the GLSL extended instruction "
1515 "set.");
1516 return selectExtInst(ResVReg, ResType, I,
1517 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}},
1518 setMIFlags, useMISrc, SrcRegs);
1519}
1520
1521bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1522 SPIRVTypeInst ResType,
1523 MachineInstr &I,
1524 CL::OpenCLExtInst CLInst,
1525 bool setMIFlags, bool useMISrc,
1526 ArrayRef<Register> SrcRegs) const {
1527 return selectExtInst(ResVReg, ResType, I,
1528 {{SPIRV::InstructionSet::OpenCL_std, CLInst}},
1529 setMIFlags, useMISrc, SrcRegs);
1530}
1531
1532bool SPIRVInstructionSelector::selectExtInst(
1533 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
1534 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst, bool setMIFlags,
1535 bool useMISrc, ArrayRef<Register> SrcRegs) const {
1536 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1537 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1538 return selectExtInst(ResVReg, ResType, I, ExtInsts, setMIFlags, useMISrc,
1539 SrcRegs);
1540}
1541
1542bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1543 SPIRVTypeInst ResType,
1544 MachineInstr &I,
1545 const ExtInstList &Insts,
1546 bool setMIFlags, bool useMISrc,
1547 ArrayRef<Register> SrcRegs) const {
1548
1549 for (const auto &[InstructionSet, Opcode] : Insts) {
1550 if (!STI.canUseExtInstSet(InstructionSet))
1551 continue;
1552 MachineBasicBlock &BB = *I.getParent();
1553 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1554 .addDef(ResVReg)
1555 .addUse(GR.getSPIRVTypeID(ResType))
1556 .addImm(static_cast<uint32_t>(InstructionSet))
1557 .addImm(Opcode);
1558 if (setMIFlags)
1559 MIB.setMIFlags(I.getFlags());
1560 if (useMISrc) {
1561 const unsigned NumOps = I.getNumOperands();
1562 unsigned Index = 1;
1563 if (Index < NumOps &&
1564 I.getOperand(Index).getType() ==
1565 MachineOperand::MachineOperandType::MO_IntrinsicID)
1566 Index = 2;
1567 for (; Index < NumOps; ++Index)
1568 MIB.add(I.getOperand(Index));
1569 } else {
1570 for (Register SReg : SrcRegs) {
1571 MIB.addUse(SReg);
1572 }
1573 }
1574 MIB.constrainAllUses(TII, TRI, RBI);
1575 return true;
1576 }
1577 return false;
1578}
1579
1580bool SPIRVInstructionSelector::selectCopySign(Register ResVReg,
1581 SPIRVTypeInst ResType,
1582 MachineInstr &I) const {
1583 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std))
1584 return selectExtInst(ResVReg, ResType, I, CL::copysign);
1585
1586 // There is no copysign instruction in the GLSL Extended Instruction set, so
1587 // it is implemented with bit manipulation:
1588 // bitcast((bitcast(magnitude) & ~signBit) | (bitcast(sign) & signBit))
1589 Register MagnitudeReg = I.getOperand(1).getReg();
1590 Register SignReg = I.getOperand(2).getReg();
1591
1592 const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1593 const unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
1594 SPIRVTypeInst IntType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
1595 const APInt SignMaskVal = APInt::getSignMask(BitWidth);
1596
1597 Register SignMask, NotSignMask;
1598 unsigned AndOpcode, OrOpcode;
1599 if (ComponentCount > 1) {
1600 IntType = GR.getOrCreateSPIRVVectorType(IntType, ComponentCount, I, TII);
1601 SignMask = GR.getOrCreateConstVector(SignMaskVal, I, IntType, TII);
1602 NotSignMask = GR.getOrCreateConstVector(~SignMaskVal, I, IntType, TII);
1603 AndOpcode = SPIRV::OpBitwiseAndV;
1604 OrOpcode = SPIRV::OpBitwiseOrV;
1605 } else {
1606 SignMask = GR.getOrCreateConstInt(SignMaskVal, I, IntType, TII);
1607 NotSignMask = GR.getOrCreateConstInt(~SignMaskVal, I, IntType, TII);
1608 AndOpcode = SPIRV::OpBitwiseAndS;
1609 OrOpcode = SPIRV::OpBitwiseOrS;
1610 }
1611
1612 auto EmitBitOp = [&](Register &ResReg, ArrayRef<Register> SrcRegs,
1613 unsigned Opcode) {
1614 ResReg = MRI->createVirtualRegister(GR.getRegClass(IntType));
1615 return selectOpWithSrcs(ResReg, IntType, I, SrcRegs, Opcode);
1616 };
1617
1618 Register MagnitudeInt, SignInt, MagnitudeBits, SignBits, CombinedInt;
1619 if (!EmitBitOp(MagnitudeInt, {MagnitudeReg}, SPIRV::OpBitcast) ||
1620 !EmitBitOp(SignInt, {SignReg}, SPIRV::OpBitcast) ||
1621 !EmitBitOp(MagnitudeBits, {MagnitudeInt, NotSignMask}, AndOpcode) ||
1622 !EmitBitOp(SignBits, {SignInt, SignMask}, AndOpcode) ||
1623 !EmitBitOp(CombinedInt, {MagnitudeBits, SignBits}, OrOpcode))
1624 return false;
1625
1626 return selectOpWithSrcs(ResVReg, ResType, I, {CombinedInt}, SPIRV::OpBitcast);
1627}
1628
1629bool SPIRVInstructionSelector::selectFrexp(Register ResVReg,
1630 SPIRVTypeInst ResType,
1631 MachineInstr &I) const {
1632 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp},
1633 {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}};
1634 for (const auto &Ex : ExtInsts) {
1635 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1636 uint32_t Opcode = Ex.second;
1637 if (!STI.canUseExtInstSet(Set))
1638 continue;
1639
1640 MachineIRBuilder MIRBuilder(I);
1641 SPIRVTypeInst PointeeTy = GR.getSPIRVTypeForVReg(I.getOperand(1).getReg());
1642 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1643 PointeeTy, MIRBuilder, SPIRV::StorageClass::Function);
1644 Register PointerVReg =
1645 createVirtualRegister(PointerType, &GR, MRI, MRI->getMF());
1646
1647 auto It = getOpVariableMBBIt(*I.getMF());
1648 // An untyped pointer result type is only legal on OpUntypedVariableKHR,
1649 // but not on OpVariable.
1650 const bool IsUntyped =
1651 PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1652 auto VarMIB =
1653 BuildMI(*It->getParent(), It, It->getDebugLoc(),
1654 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
1655 : SPIRV::OpVariable))
1656 .addDef(PointerVReg)
1657 .addUse(GR.getSPIRVTypeID(PointerType))
1658 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
1659 if (IsUntyped)
1660 VarMIB.addUse(GR.getSPIRVTypeID(PointeeTy)); // Data Type
1661 VarMIB.constrainAllUses(TII, TRI, RBI);
1662
1663 SPIRVTypeInst MantissaTy = GR.getSPIRVTypeForVReg(I.getOperand(2).getReg());
1664 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1665 .addDef(ResVReg)
1666 .addUse(GR.getSPIRVTypeID(MantissaTy))
1667 .addImm(static_cast<uint32_t>(Ex.first))
1668 .addImm(Opcode)
1669 .add(I.getOperand(2))
1670 .addUse(PointerVReg)
1671 .constrainAllUses(TII, TRI, RBI);
1672
1673 Register ExpResReg = I.getOperand(1).getReg();
1674 if (!MRI->use_nodbg_empty(ExpResReg))
1675 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1676 .addDef(ExpResReg)
1677 .addUse(GR.getSPIRVTypeID(PointeeTy))
1678 .addUse(PointerVReg)
1679 .constrainAllUses(TII, TRI, RBI);
1680 return true;
1681 }
1682 return false;
1683}
1684
1685bool SPIRVInstructionSelector::selectLdexp(Register ResVReg,
1686 SPIRVTypeInst ResType,
1687 MachineInstr &I) const {
1688 Register XReg = I.getOperand(1).getReg();
1689 Register ExpReg = I.getOperand(2).getReg();
1690
1691 // Both OpenCL.std ldexp and GLSL.std.450 Ldexp require the exponent to have
1692 // the same number of components as the result, but G_FLDEXP's exponent is
1693 // scalar even for vector operands. Splat it when the result is a vector.
1694 SPIRVTypeInst ExpType = GR.getSPIRVTypeForVReg(ExpReg);
1695 if (ResType->getOpcode() == SPIRV::OpTypeVector &&
1696 ExpType->getOpcode() != SPIRV::OpTypeVector) {
1697 unsigned NumElts = ResType->getOperand(2).getImm();
1698 SPIRVTypeInst ExpVecType =
1699 GR.getOrCreateSPIRVVectorType(ExpType, NumElts, I, TII);
1700 Register SplatReg =
1701 createVirtualRegister(ExpVecType, &GR, MRI, MRI->getMF());
1702 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1703 TII.get(SPIRV::OpCompositeConstruct))
1704 .addDef(SplatReg)
1705 .addUse(GR.getSPIRVTypeID(ExpVecType));
1706 for (unsigned J = 0; J < NumElts; ++J)
1707 MIB.addUse(ExpReg);
1708 MIB.constrainAllUses(TII, TRI, RBI);
1709 ExpReg = SplatReg;
1710 }
1711
1712 return selectExtInst(ResVReg, ResType, I, CL::ldexp, GL::Ldexp,
1713 /*setMIFlags=*/true, /*useMISrc=*/false, {XReg, ExpReg});
1714}
1715
1716bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
1717 SPIRVTypeInst ResType,
1718 MachineInstr &I) const {
1719 Register CosResVReg = I.getOperand(1).getReg();
1720 unsigned SrcIdx = I.getNumExplicitDefs();
1721 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
1722
1723 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
1724 // OpenCL.std sincos(x, cosval*) -> returns sin(x), writes cos(x) to ptr.
1725 MachineIRBuilder MIRBuilder(I);
1726 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1727 ResType, MIRBuilder, SPIRV::StorageClass::Function);
1728 Register PointerVReg =
1729 createVirtualRegister(PointerType, &GR, MRI, MRI->getMF());
1730
1731 auto It = getOpVariableMBBIt(*I.getMF());
1732 // An untyped pointer result type is only legal on OpUntypedVariableKHR,
1733 // but not on OpVariable.
1734 const bool IsUntyped =
1735 PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1736 auto VarMIB =
1737 BuildMI(*It->getParent(), It, It->getDebugLoc(),
1738 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
1739 : SPIRV::OpVariable))
1740 .addDef(PointerVReg)
1741 .addUse(GR.getSPIRVTypeID(PointerType))
1742 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
1743 if (IsUntyped)
1744 VarMIB.addUse(GR.getSPIRVTypeID(ResType)); // Data Type
1745 VarMIB.constrainAllUses(TII, TRI, RBI);
1746 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1747 .addDef(ResVReg)
1748 .addUse(ResTypeReg)
1749 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1750 .addImm(CL::sincos)
1751 .add(I.getOperand(SrcIdx))
1752 .addUse(PointerVReg)
1753 .constrainAllUses(TII, TRI, RBI);
1754 if (!MRI->use_nodbg_empty(CosResVReg))
1755 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1756 .addDef(CosResVReg)
1757 .addUse(ResTypeReg)
1758 .addUse(PointerVReg)
1759 .constrainAllUses(TII, TRI, RBI);
1760 return true;
1761 } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
1762 // GLSL.std.450 has no combined sincos; emit separate Sin and Cos.
1763 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1764 .addDef(ResVReg)
1765 .addUse(ResTypeReg)
1766 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1767 .addImm(GL::Sin)
1768 .add(I.getOperand(SrcIdx))
1769 .constrainAllUses(TII, TRI, RBI);
1770 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1771 .addDef(CosResVReg)
1772 .addUse(ResTypeReg)
1773 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1774 .addImm(GL::Cos)
1775 .add(I.getOperand(SrcIdx))
1776 .constrainAllUses(TII, TRI, RBI);
1777 return true;
1778 }
1779 return false;
1780}
1781
1782bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
1783 SPIRVTypeInst ResType,
1784 MachineInstr &I,
1785 ArrayRef<Register> Srcs,
1786 unsigned Opcode) const {
1787 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1788 .addDef(ResVReg)
1789 .addUse(GR.getSPIRVTypeID(ResType));
1790 for (Register SReg : Srcs) {
1791 MIB.addUse(SReg);
1792 }
1793 MIB.constrainAllUses(TII, TRI, RBI);
1794 return true;
1795}
1796
1797std::optional<SplitParts> SPIRVInstructionSelector::splitEvenOddLanes(
1798 Register PopCountReg, unsigned ComponentCount, MachineInstr &I,
1799 SPIRVTypeInst I32Type) const {
1800 SplitParts Parts;
1801
1802 if (ComponentCount == 1) {
1803 // ---- Scalar path: extract element 1 (high word) and element 0 (low word)
1804 // ----
1805 Parts.IsScalar = true;
1806 Parts.Type = I32Type;
1807 Parts.High = MRI->createVirtualRegister(GR.getRegClass(I32Type));
1808 Parts.Low = MRI->createVirtualRegister(GR.getRegClass(I32Type));
1809
1810 bool ZeroAsNull = !STI.isShader();
1811 Register IdxZero = GR.getOrCreateConstInt(0, I, I32Type, TII, ZeroAsNull);
1812 Register IdxOne = GR.getOrCreateConstInt(1, I, I32Type, TII, ZeroAsNull);
1813
1814 if (!selectOpWithSrcs(Parts.High, I32Type, I, {PopCountReg, IdxOne},
1815 SPIRV::OpVectorExtractDynamic))
1816 return std::nullopt;
1817
1818 if (!selectOpWithSrcs(Parts.Low, I32Type, I, {PopCountReg, IdxZero},
1819 SPIRV::OpVectorExtractDynamic))
1820 return std::nullopt;
1821
1822 } else {
1823 // ---- Vector path: shuffle odd lanes → High, even lanes → Low ----
1824 MachineIRBuilder MIRBuilder(I);
1825 Parts.IsScalar = false;
1826 Parts.Type = GR.getOrCreateSPIRVVectorType(I32Type, ComponentCount,
1827 MIRBuilder, /*IsSigned=*/false);
1828 Parts.High = MRI->createVirtualRegister(GR.getRegClass(Parts.Type));
1829 Parts.Low = MRI->createVirtualRegister(GR.getRegClass(Parts.Type));
1830
1831 // High = odd-indexed elements (1, 3, 5, …) — the upper 32-bit halves.
1832 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1833 TII.get(SPIRV::OpVectorShuffle))
1834 .addDef(Parts.High)
1835 .addUse(GR.getSPIRVTypeID(Parts.Type))
1836 .addUse(PopCountReg)
1837 .addUse(PopCountReg);
1838 for (unsigned J = 1; J < ComponentCount * 2; J += 2)
1839 MIB.addImm(J);
1840 MIB.constrainAllUses(TII, TRI, RBI);
1841
1842 // Low = even-indexed elements (0, 2, 4, …) — the lower 32-bit halves.
1843 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1844 TII.get(SPIRV::OpVectorShuffle))
1845 .addDef(Parts.Low)
1846 .addUse(GR.getSPIRVTypeID(Parts.Type))
1847 .addUse(PopCountReg)
1848 .addUse(PopCountReg);
1849 for (unsigned J = 0; J < ComponentCount * 2; J += 2)
1850 MIB.addImm(J);
1851 MIB.constrainAllUses(TII, TRI, RBI);
1852 }
1853
1854 return Parts;
1855}
1856
1857bool SPIRVInstructionSelector::selectPopCount16(Register ResVReg,
1858 SPIRVTypeInst ResType,
1859 MachineInstr &I,
1860 unsigned ExtOpcode,
1861 unsigned Opcode) const {
1862 Register OpReg = I.getOperand(1).getReg();
1863 unsigned NumElems = GR.getScalarOrVectorComponentCount(OpReg);
1864
1865 MachineIRBuilder MIRBuilder(I);
1866 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
1867 SPIRVTypeInst I32VectorType =
1868 GR.getOrCreateSPIRVVectorType(I32Type, NumElems, MIRBuilder, false);
1869
1870 bool IsVector = NumElems > 1;
1871 SPIRVTypeInst ExtType = IsVector ? I32VectorType : I32Type;
1872 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ExtType));
1873 // Always use OpUConvert to always use a 0 extend
1874 if (!selectOpWithSrcs(ExtReg, ExtType, I, {OpReg}, SPIRV::OpUConvert))
1875 return false;
1876
1877 Register PopCountReg = MRI->createVirtualRegister(GR.getRegClass(ExtType));
1878 if (!selectPopCount32(PopCountReg, ExtType, I, ExtReg, Opcode))
1879 return false;
1880
1881 return selectOpWithSrcs(ResVReg, ResType, I, {PopCountReg}, ExtOpcode);
1882}
1883
1884bool SPIRVInstructionSelector::selectPopCount32(Register ResVReg,
1885 SPIRVTypeInst ResType,
1886 MachineInstr &I,
1887 Register SrcReg,
1888 unsigned Opcode) const {
1889 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
1890}
1891
1892bool SPIRVInstructionSelector::selectPopCount64(Register ResVReg,
1893 SPIRVTypeInst ResType,
1894 MachineInstr &I,
1895 Register SrcReg,
1896 unsigned Opcode) const {
1897 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
1898 if (ComponentCount > 2)
1899 return handle64BitOverflow(
1900 ResVReg, ResType, I, SrcReg, Opcode,
1901 [this](Register R, SPIRVTypeInst T, MachineInstr &I, Register S,
1902 unsigned O) { return this->selectPopCount64(R, T, I, S, O); });
1903
1904 MachineIRBuilder MIRBuilder(I);
1905
1906 // ---- Types ----
1907 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
1908 SPIRVTypeInst VecI32Type = GR.getOrCreateSPIRVVectorType(
1909 I32Type, 2 * ComponentCount, MIRBuilder, /*IsSigned=*/false);
1910
1911 // Converts 64 bit into and array of 32 bit, containing 2 elements.
1912 Register Vec32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
1913 if (!selectOpWithSrcs(Vec32, VecI32Type, I, {SrcReg}, SPIRV::OpBitcast))
1914 return false;
1915
1916 // Apply popcount on each 32 bit lane
1917 Register Pop32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
1918 if (!selectPopCount32(Pop32, VecI32Type, I, Vec32, Opcode))
1919 return false;
1920
1921 // Splits result into highbit lane and lowbit lane
1922 auto MaybeParts = splitEvenOddLanes(Pop32, ComponentCount, I, I32Type);
1923 if (!MaybeParts)
1924 return false;
1925 SplitParts &Parts = *MaybeParts;
1926
1927 // Sum high part and low part
1928 unsigned OpAdd = Parts.IsScalar ? SPIRV::OpIAddS : SPIRV::OpIAddV;
1929 Register Sum = MRI->createVirtualRegister(GR.getRegClass(Parts.Type));
1930 if (!selectOpWithSrcs(Sum, Parts.Type, I, {Parts.High, Parts.Low}, OpAdd))
1931 return false;
1932
1933 // Convert 32 bit sum into 64 bit scalar
1934 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
1935 unsigned ConvOp = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1936 return selectOpWithSrcs(ResVReg, ResType, I, {Sum}, ConvOp);
1937}
1938
1939bool SPIRVInstructionSelector::selectPopCount(Register ResVReg,
1940 SPIRVTypeInst ResType,
1941 MachineInstr &I,
1942 unsigned Opcode) const {
1943 // Vulkan restricts OpBitCount to 32-bit integers or vectors of 32-bit
1944 // integers unless VK_KHR_maintenance9 is enabled. Until VK_KHR_maintenance9
1945 // is core we will not generate OpBitCount with any other types when
1946 // targeting Vulkan.
1947 if (!STI.getTargetTriple().isVulkanOS())
1948 return selectUnOp(ResVReg, ResType, I, Opcode);
1949
1950 Register OpReg = I.getOperand(1).getReg();
1951 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
1952 unsigned ExtOpcode = GR.isScalarOrVectorSigned(ResType) ? SPIRV::OpSConvert
1953 : SPIRV::OpUConvert;
1954 switch (GR.getScalarOrVectorBitWidth(OpType)) {
1955 case 8:
1956 case 16:
1957 return selectPopCount16(ResVReg, ResType, I, ExtOpcode, Opcode);
1958 case 32:
1959 return selectPopCount32(ResVReg, ResType, I, OpReg, Opcode);
1960 case 64:
1961 return selectPopCount64(ResVReg, ResType, I, OpReg, Opcode);
1962 default:
1963 return diagnoseUnsupported(I, "unsupported operand bit width for popcount");
1964 }
1965}
1966
1967bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
1968 SPIRVTypeInst ResType,
1969 MachineInstr &I,
1970 unsigned Opcode) const {
1971 if (STI.isPhysicalSPIRV() && I.getOperand(1).isReg()) {
1972 Register SrcReg = I.getOperand(1).getReg();
1973 bool IsGV = false;
1974 for (MachineRegisterInfo::def_instr_iterator DefIt =
1975 MRI->def_instr_begin(SrcReg);
1976 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) {
1977 unsigned DefOpCode = DefIt->getOpcode();
1978 if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
1979 // We need special handling to look through the type assignment or the
1980 // COPY pseudo-op and see if this is a constant or a global.
1981 if (auto *VRD = getVRegDef(*MRI, DefIt->getOperand(1).getReg()))
1982 DefOpCode = VRD->getOpcode();
1983 }
1984 if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE ||
1985 DefOpCode == TargetOpcode::G_CONSTANT ||
1986 DefOpCode == SPIRV::OpVariable ||
1987 DefOpCode == SPIRV::OpUntypedVariableKHR ||
1988 DefOpCode == SPIRV::OpConstantI) {
1989 IsGV = true;
1990 break;
1991 }
1992 }
1993 if (IsGV) {
1994 uint32_t SpecOpcode = 0;
1995 switch (Opcode) {
1996 case SPIRV::OpConvertPtrToU:
1997 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
1998 break;
1999 case SPIRV::OpConvertUToPtr:
2000 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
2001 break;
2002 }
2003 if (SpecOpcode) {
2004 BuildMI(*I.getParent(), I, I.getDebugLoc(),
2005 TII.get(SPIRV::OpSpecConstantOp))
2006 .addDef(ResVReg)
2007 .addUse(GR.getSPIRVTypeID(ResType))
2008 .addImm(SpecOpcode)
2009 .addUse(SrcReg)
2010 .constrainAllUses(TII, TRI, RBI);
2011 return true;
2012 }
2013 }
2014 }
2015 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()},
2016 Opcode);
2017}
2018
2019bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
2020 SPIRVTypeInst ResType,
2021 MachineInstr &I) const {
2022 Register OpReg = I.getOperand(1).getReg();
2023 SPIRVTypeInst OpType =
2024 OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
2025 if (!GR.isBitcastCompatible(ResType, OpType))
2026 return diagnoseUnsupported(
2027 I, "incompatible result and operand types in a bitcast");
2028 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
2029}
2030
2031static void addMemoryOperands(MachineMemOperand *MemOp,
2032 MachineInstrBuilder &MIB,
2033 MachineIRBuilder &MIRBuilder,
2034 SPIRVGlobalRegistry &GR) {
2035 const SPIRVSubtarget *ST =
2036 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
2037 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
2038 if (MemOp->isVolatile())
2039 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
2040 if (MemOp->isNonTemporal())
2041 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
2042 // Aligned memory operand requires the Kernel capability.
2043 if (!ST->isShader() && MemOp->getAlign().value())
2044 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
2045
2046 [[maybe_unused]] MachineInstr *AliasList = nullptr;
2047 [[maybe_unused]] MachineInstr *NoAliasList = nullptr;
2048 if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
2049 if (auto *MD = MemOp->getAAInfo().Scope) {
2050 AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD);
2051 if (AliasList)
2052 SpvMemOp |=
2053 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
2054 }
2055 if (auto *MD = MemOp->getAAInfo().NoAlias) {
2056 NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD);
2057 if (NoAliasList)
2058 SpvMemOp |=
2059 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
2060 }
2061 }
2062
2063 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
2064 MIB.addImm(SpvMemOp);
2065 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
2066 MIB.addImm(MemOp->getAlign().value());
2067 if (AliasList)
2068 MIB.addUse(AliasList->getOperand(0).getReg());
2069 if (NoAliasList)
2070 MIB.addUse(NoAliasList->getOperand(0).getReg());
2071 }
2072}
2073
2074static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
2075 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
2076 if (Flags & MachineMemOperand::Flags::MOVolatile)
2077 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
2078 if (Flags & MachineMemOperand::Flags::MONonTemporal)
2079 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
2080
2081 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
2082 MIB.addImm(SpvMemOp);
2083}
2084
2085bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
2086 SPIRVTypeInst ResType,
2087 MachineInstr &I) const {
2088 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2089 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2090
2091 auto *PtrDef = getVRegDef(*MRI, Ptr);
2092 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
2093 if (IntPtrDef &&
2094 (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
2095 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
2096
2097 Register HandleReg = IntPtrDef->getOperand(2).getReg();
2098 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg);
2099 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
2100 Register NewHandleReg =
2101 MRI->createVirtualRegister(MRI->getRegClass(HandleReg));
2102 auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg));
2103 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) {
2104 return false;
2105 }
2106
2107 Register IdxReg = IntPtrDef->getOperand(3).getReg();
2108 return generateImageReadOrFetch(ResVReg, ResType, NewHandleReg, IdxReg,
2109 I.getDebugLoc(), I);
2110 }
2111 }
2112
2113 MachineIRBuilder MIRBuilder(I);
2114
2115 if (I.getNumMemOperands()) {
2116 const MachineMemOperand *MemOp = *I.memoperands_begin();
2117 if (MemOp->isAtomic())
2118 return selectAtomicLoad(ResVReg, ResType, I);
2119 }
2120
2121 auto MIB = MIRBuilder.buildInstr(SPIRV::OpLoad)
2122 .addDef(ResVReg)
2123 .addUse(GR.getSPIRVTypeID(ResType))
2124 .addUse(Ptr);
2125 if (!I.getNumMemOperands()) {
2126 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
2127 I.getOpcode() ==
2128 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
2129 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
2130 } else {
2131 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2132 }
2133 MIB.constrainAllUses(TII, TRI, RBI);
2134 return true;
2135}
2136
2137Register SPIRVInstructionSelector::createPtrSizedIntReg(
2138 MachineIRBuilder &MIRBuilder) const {
2139 SPIRVTypeInst IntType =
2140 GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder);
2141 Register Reg =
2142 MRI->createGenericVirtualRegister(LLT::scalar(GR.getPointerSize()));
2143 MRI->setRegClass(Reg, GR.getRegClass(IntType));
2144 GR.assignSPIRVTypeToVReg(IntType, Reg, MIRBuilder.getMF());
2145 return Reg;
2146}
2147
2148Register
2149SPIRVInstructionSelector::convertPtrToInt(Register PtrVal,
2150 MachineIRBuilder &MIRBuilder) const {
2151 SPIRVTypeInst IntType =
2152 GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder);
2153 Register IntReg = createPtrSizedIntReg(MIRBuilder);
2154 MIRBuilder.buildInstr(SPIRV::OpConvertPtrToU)
2155 .addDef(IntReg)
2156 .addUse(GR.getSPIRVTypeID(IntType)) // Result type
2157 .addUse(PtrVal) // Pointer operand
2158 .constrainAllUses(TII, TRI, RBI);
2159 return IntReg;
2160}
2161
2162Register SPIRVInstructionSelector::castPtrToPtrToInt(
2163 Register Ptr, SPIRV::StorageClass::StorageClass SC,
2164 MachineIRBuilder &MIRBuilder) const {
2165 SPIRVTypeInst IntType =
2166 GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder);
2167 SPIRVTypeInst PtrType =
2168 GR.getOrCreateSPIRVPointerType(IntType, MIRBuilder, SC);
2169 Register CastedPtr =
2170 MRI->createGenericVirtualRegister(LLT::scalar(GR.getPointerSize()));
2171 MRI->setRegClass(CastedPtr, GR.getRegClass(PtrType));
2172 GR.assignSPIRVTypeToVReg(PtrType, CastedPtr, MIRBuilder.getMF());
2173 MIRBuilder.buildInstr(SPIRV::OpBitcast)
2174 .addDef(CastedPtr)
2175 .addUse(GR.getSPIRVTypeID(PtrType))
2176 .addUse(Ptr)
2177 .constrainAllUses(TII, TRI, RBI);
2178 return CastedPtr;
2179}
2180
2181bool SPIRVInstructionSelector::selectAtomicPtrValue(
2182 Register ResVReg, SPIRVTypeInst ResType, MachineIRBuilder &MIRBuilder,
2183 function_ref<Register(SPIRVTypeInst IntType)> EmitAtomic) const {
2184 // Pointer-typed atomics are lowered by bitcasting the Ptr operand to a
2185 // pointer to an integer of the same size as the pointer, so that the actual
2186 // atomic instruction operates on integers as required by the spec. Value
2187 // operands and results are converted with OpConvertPtrToU/OpConvertUToPtr.
2188 unsigned PtrSize = GR.getPointerSize();
2189 SPIRVTypeInst IntType = GR.getOrCreateSPIRVIntegerType(PtrSize, MIRBuilder);
2190
2191 Register IntResult = EmitAtomic(IntType);
2192 if (IntResult.isValid())
2193 MIRBuilder.buildInstr(SPIRV::OpConvertUToPtr)
2194 .addDef(ResVReg)
2195 .addUse(GR.getSPIRVTypeID(ResType))
2196 .addUse(IntResult)
2197 .constrainAllUses(TII, TRI, RBI);
2198 return true;
2199}
2200
2201bool SPIRVInstructionSelector::selectAtomicLoad(Register ResVReg,
2202 SPIRVTypeInst ResType,
2203 MachineInstr &I) const {
2204 LLVMContext &Context = I.getMF()->getFunction().getContext();
2205
2206 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2207 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2208
2209 if (!ResType.isTypeIntOrFloat() && !ResType.isTypePtr())
2210 return diagnoseUnsupported(
2211 I, "Lowering to SPIR-V of atomic load is only "
2212 "allowed for integer, floating point or pointer types");
2213
2214 assert(I.getNumMemOperands());
2215 const MachineMemOperand &MemOp = **I.memoperands_begin();
2216 assert(MemOp.isAtomic());
2217
2218 uint32_t Scope = static_cast<uint32_t>(
2219 getMemScope(STI.getTargetTriple(), Context, MemOp.getSyncScopeID()));
2220 Register ScopeReg = buildI32Constant(Scope, I);
2221
2222 AtomicOrdering AO = MemOp.getSuccessOrdering();
2223 uint32_t StorageClass = static_cast<uint32_t>(getMemSemanticsForStorageClass(
2224 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI)));
2225 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
2226 if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS())
2227 MemSem |= static_cast<uint32_t>(SPIRV::MemorySemantics::Volatile);
2228 uint32_t Sem = getMemSemanticsWithStorageClass(STI.getTargetTriple(), MemSem,
2229 StorageClass);
2230 Register MemSemReg = buildI32Constant(Sem, I);
2231
2232 MachineIRBuilder MIRBuilder(I);
2233
2234 if (ResType.isTypePtr()) {
2235 if (!STI.isPhysicalSPIRV())
2236 return diagnoseUnsupported(
2237 I, "Lowering to SPIR-V of atomic load is only "
2238 "allowed for pointer types for physical addressing model");
2239 // If data to load is a pointer type we bitcast the Ptr parameter to pointer
2240 // to an integer type of the same size as the pointer size and then generate
2241 // OpAtomicLoad the return value of that OpAtomicLoad is an integer that is
2242 // converted back to a pointer type using OpConvertUToPtr.
2243 SPIRV::StorageClass::StorageClass SC =
2244 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI);
2245 return selectAtomicPtrValue(
2246 ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) {
2247 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2248 Register IntResult = createPtrSizedIntReg(MIRBuilder);
2249 MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
2250 .addDef(IntResult)
2251 .addUse(GR.getSPIRVTypeID(IntType))
2252 .addUse(CastedPtr)
2253 .addUse(ScopeReg)
2254 .addUse(MemSemReg)
2255 .constrainAllUses(TII, TRI, RBI);
2256 return IntResult;
2257 });
2258 }
2259 auto AtomicLoad = MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
2260 .addDef(ResVReg)
2261 .addUse(GR.getSPIRVTypeID(ResType))
2262 .addUse(Ptr)
2263 .addUse(ScopeReg)
2264 .addUse(MemSemReg);
2265 AtomicLoad.constrainAllUses(TII, TRI, RBI);
2266
2267 return true;
2268}
2269
2270bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
2271 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2272 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
2273 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2274
2275 auto *PtrDef = getVRegDef(*MRI, Ptr);
2276 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
2277 if (IntPtrDef &&
2278 (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
2279 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
2280
2281 Register HandleReg = IntPtrDef->getOperand(2).getReg();
2282 Register NewHandleReg =
2283 MRI->createVirtualRegister(MRI->getRegClass(HandleReg));
2284 auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg));
2285 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg);
2286 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) {
2287 return false;
2288 }
2289
2290 Register IdxReg = IntPtrDef->getOperand(3).getReg();
2291 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
2292 SPIRVTypeInst SampledType =
2293 GR.getSPIRVTypeForVReg(HandleType->getOperand(1).getReg());
2294 SPIRVTypeInst StoreValCompType =
2295 GR.getScalarOrVectorComponentType(GR.getSPIRVTypeForVReg(StoreVal));
2296 if (StoreValCompType && StoreValCompType != SampledType) {
2297 // A wide element (e.g. int64_t2) is emulated with a narrower packed
2298 // image. This bitcasts the value to match the format.
2299 SPIRVTypeInst PackedType = widenTypeToVec4(SampledType, I);
2300 Register PackedReg =
2301 MRI->createVirtualRegister(GR.getRegClass(PackedType));
2302 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
2303 .addDef(PackedReg)
2304 .addUse(GR.getSPIRVTypeID(PackedType))
2305 .addUse(StoreVal)
2306 .constrainAllUses(TII, TRI, RBI);
2307 StoreVal = PackedReg;
2308 }
2309
2310 auto BMI = BuildMI(*I.getParent(), I, I.getDebugLoc(),
2311 TII.get(SPIRV::OpImageWrite))
2312 .addUse(NewHandleReg)
2313 .addUse(IdxReg)
2314 .addUse(StoreVal);
2315
2316 const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(HandleType);
2317 if (sampledTypeIsSignedInteger(LLVMHandleType))
2318 BMI.addImm(0x1000); // SignExtend
2319
2320 BMI.constrainAllUses(TII, TRI, RBI);
2321 return true;
2322 }
2323 }
2324
2325 SPIRVTypeInst PointeeTy = GR.getPointeeType(GR.getSPIRVTypeForVReg(Ptr));
2326 SPIRVTypeInst StoreTy = GR.getSPIRVTypeForVReg(StoreVal);
2327 if (PointeeTy && PointeeTy->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
2328 StoreTy->getOpcode() != SPIRV::OpTypeVectorIdEXT &&
2329 GR.getScalarOrVectorComponentCount(PointeeTy) == 1) {
2330 MachineInstr *StoreValDef = getVRegDef(*MRI, StoreVal);
2331 Register Reg = StoreValDef->getOperand(0).getReg();
2332 if (auto FC = GFConstant::getConstant(Reg, *MRI))
2333 StoreVal =
2334 GR.getOrCreateConstVector(FC->getScalarValue(), I, PointeeTy, TII);
2335 else if (auto IC = GIConstant::getConstant(Reg, *MRI))
2336 StoreVal =
2337 GR.getOrCreateConstVector(IC->getScalarValue(), I, PointeeTy, TII);
2338 else
2339 llvm_unreachable("Unexpected <1 x T> Store type!");
2340 }
2341
2342 if (I.getNumMemOperands()) {
2343 const MachineMemOperand *MemOp = *I.memoperands_begin();
2344 if (MemOp->isAtomic())
2345 return selectAtomicStore(I);
2346 }
2347
2348 // Stores into a read-only storage class produce invalid SPIR-V. Reject such
2349 // input with a diagnostic rather than silently emitting an OpStore that
2350 // validation rejects.
2351 SPIRV::StorageClass::StorageClass PtrSC = GR.getPointerStorageClass(Ptr);
2352 if (PtrSC == SPIRV::StorageClass::UniformConstant ||
2353 PtrSC == SPIRV::StorageClass::Input ||
2354 PtrSC == SPIRV::StorageClass::PushConstant)
2355 return diagnoseUnsupported(
2356 I, "store into a read-only SPIR-V storage class is not allowed");
2357
2358 MachineIRBuilder MIRBuilder(I);
2359 auto MIB = MIRBuilder.buildInstr(SPIRV::OpStore).addUse(Ptr).addUse(StoreVal);
2360 if (!I.getNumMemOperands()) {
2361 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
2362 I.getOpcode() ==
2363 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
2364 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
2365 } else {
2366 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2367 }
2368 MIB.constrainAllUses(TII, TRI, RBI);
2369 return true;
2370}
2371
2372bool SPIRVInstructionSelector::selectAtomicStore(MachineInstr &I) const {
2373 LLVMContext &Context = I.getMF()->getFunction().getContext();
2374
2375 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2376 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
2377 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2378
2379 SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(Ptr);
2380 SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
2381 // For an untyped pointer the data type is the stored value's type.
2382 if (!PointeeType && PtrType &&
2383 PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
2384 PointeeType = GR.getSPIRVTypeForVReg(StoreVal);
2385 if (!PointeeType)
2386 return diagnoseUnsupported(I,
2387 "Lowering to SPIR-V of atomic store is only "
2388 "allowed for integer or floating point types");
2389
2390 assert(I.getNumMemOperands());
2391 const MachineMemOperand &MemOp = **I.memoperands_begin();
2392 assert(MemOp.isAtomic());
2393
2394 uint32_t Scope = static_cast<uint32_t>(
2395 getMemScope(STI.getTargetTriple(), Context, MemOp.getSyncScopeID()));
2396 Register ScopeReg = buildI32Constant(Scope, I);
2397
2398 AtomicOrdering AO = MemOp.getSuccessOrdering();
2399 uint32_t StorageClass = static_cast<uint32_t>(getMemSemanticsForStorageClass(
2400 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI)));
2401 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
2402 if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS())
2403 MemSem |= static_cast<uint32_t>(SPIRV::MemorySemantics::Volatile);
2404 uint32_t Sem = getMemSemanticsWithStorageClass(STI.getTargetTriple(), MemSem,
2405 StorageClass);
2406 Register MemSemReg = buildI32Constant(Sem, I);
2407 MachineIRBuilder MIRBuilder(I);
2408
2409 if (PointeeType.isTypePtr()) {
2410 if (!STI.isPhysicalSPIRV())
2411 return diagnoseUnsupported(
2412 I, "Lowering to SPIR-V of atomic store is only "
2413 "allowed for pointer types for physical addressing model");
2414 // If data to store is a pointer type we cast it to an integer type of the
2415 // same size as the pointer size using OpConvertPtrToU, bitcast Ptr
2416 // parameter to pointer to integer type and then generate OpAtomicStore
2417 // with casted values as required by spec.
2418 SPIRV::StorageClass::StorageClass SC =
2419 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI);
2420 return selectAtomicPtrValue(
2421 Register(), SPIRVTypeInst(), MIRBuilder, [&](SPIRVTypeInst IntType) {
2422 Register ValueAsInt = convertPtrToInt(StoreVal, MIRBuilder);
2423 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2424 MIRBuilder.buildInstr(SPIRV::OpAtomicStore)
2425 .addUse(CastedPtr)
2426 .addUse(ScopeReg)
2427 .addUse(MemSemReg)
2428 .addUse(ValueAsInt)
2429 .constrainAllUses(TII, TRI, RBI);
2430 // Stores produce no result, so no OpConvertUToPtr is needed.
2431 return Register();
2432 });
2433 }
2434
2435 if (!PointeeType.isTypeIntOrFloat())
2436 return diagnoseUnsupported(I,
2437 "Lowering to SPIR-V of atomic store is only "
2438 "allowed for integer or floating point types");
2439
2440 auto AtomicStore = MIRBuilder.buildInstr(SPIRV::OpAtomicStore)
2441 .addUse(Ptr)
2442 .addUse(ScopeReg)
2443 .addUse(MemSemReg)
2444 .addUse(StoreVal);
2445 AtomicStore.constrainAllUses(TII, TRI, RBI);
2446
2447 return true;
2448}
2449
2450bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg,
2451 SPIRVTypeInst ResType,
2452 MachineInstr &I) const {
2453 assert(I.getNumExplicitDefs() == 1 && "Expected single def for gather");
2454 // Operand indices:
2455 // 0: result (def)
2456 // 1: intrinsic ID
2457 // 2: vector of pointers
2458 // 3: alignment (i32 immediate)
2459 // 4: mask (vector of i1)
2460 // 5: passthru/fill value
2461 const Register PtrsReg = I.getOperand(2).getReg();
2462 const uint32_t Alignment = I.getOperand(3).getImm();
2463 const Register MaskReg = I.getOperand(4).getReg();
2464 const Register PassthruReg = I.getOperand(5).getReg();
2465 const Register AlignmentReg = buildI32Constant(Alignment, I);
2466
2467 MachineBasicBlock &BB = *I.getParent();
2468 auto MIB =
2469 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedGatherINTEL))
2470 .addDef(ResVReg)
2471 .addUse(GR.getSPIRVTypeID(ResType))
2472 .addUse(PtrsReg)
2473 .addUse(AlignmentReg)
2474 .addUse(MaskReg)
2475 .addUse(PassthruReg);
2476 MIB.constrainAllUses(TII, TRI, RBI);
2477 return true;
2478}
2479
2480bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &I) const {
2481 assert(I.getNumExplicitDefs() == 0 && "Expected no defs for scatter");
2482 // Operand indices (no explicit defs):
2483 // 0: intrinsic ID
2484 // 1: value vector
2485 // 2: vector of pointers
2486 // 3: alignment (i32 immediate)
2487 // 4: mask (vector of i1)
2488 const Register ValuesReg = I.getOperand(1).getReg();
2489 const Register PtrsReg = I.getOperand(2).getReg();
2490 const uint32_t Alignment = I.getOperand(3).getImm();
2491 const Register MaskReg = I.getOperand(4).getReg();
2492 const Register AlignmentReg = buildI32Constant(Alignment, I);
2493 MachineBasicBlock &BB = *I.getParent();
2494
2495 auto MIB =
2496 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedScatterINTEL))
2497 .addUse(PtrsReg)
2498 .addUse(AlignmentReg)
2499 .addUse(MaskReg)
2500 .addUse(ValuesReg);
2501 MIB.constrainAllUses(TII, TRI, RBI);
2502 return true;
2503}
2504
2505bool SPIRVInstructionSelector::diagnoseUnsupported(const MachineInstr &I,
2506 const Twine &Msg) const {
2507 const Function &F = I.getMF()->getFunction();
2508 F.getContext().diagnose(
2509 DiagnosticInfoUnsupported(F, Msg, I.getDebugLoc(), DS_Error));
2510 return false;
2511}
2512
2513bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
2514 SPIRVTypeInst ResType,
2515 MachineInstr &I) const {
2516 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2517 return diagnoseUnsupported(
2518 I, "llvm.stacksave intrinsic: this instruction requires the following "
2519 "SPIR-V extension: SPV_INTEL_variable_length_array");
2520 MachineBasicBlock &BB = *I.getParent();
2521 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL))
2522 .addDef(ResVReg)
2523 .addUse(GR.getSPIRVTypeID(ResType))
2524 .constrainAllUses(TII, TRI, RBI);
2525 return true;
2526}
2527
2528bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
2529 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2530 return diagnoseUnsupported(
2531 I,
2532 "llvm.stackrestore intrinsic: this instruction requires the following "
2533 "SPIR-V extension: SPV_INTEL_variable_length_array");
2534 if (!I.getOperand(0).isReg())
2535 return false;
2536 MachineBasicBlock &BB = *I.getParent();
2537 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL))
2538 .addUse(I.getOperand(0).getReg())
2539 .constrainAllUses(TII, TRI, RBI);
2540 return true;
2541}
2542
2543Register
2544SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &I) const {
2545 MachineIRBuilder MIRBuilder(I);
2546 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
2547
2548 // TODO: check if we have such GV, add init, use buildGlobalVariable.
2549 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
2550 Function &CurFunction = GR.CurMF->getFunction();
2551 Type *LLVMArrTy =
2552 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num);
2553 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
2554 true, GlobalValue::InternalLinkage,
2555 Constant::getNullValue(LLVMArrTy));
2556
2557 Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext());
2558 Type *ArrTy = ArrayType::get(ValTy, Num);
2559 SPIRVTypeInst VarTy = GR.getOrCreateSPIRVPointerType(
2560 ArrTy, MIRBuilder, SPIRV::StorageClass::UniformConstant);
2561
2562 SPIRVTypeInst SpvArrTy = GR.getOrCreateSPIRVType(
2563 ArrTy, MIRBuilder, SPIRV::AccessQualifier::None, false);
2564
2565 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
2566 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvArrTy, TII);
2567
2568 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2569 // With SPV_KHR_untyped_pointers enabled, getOrCreateSPIRVPointerType returns
2570 // an untyped pointer type. An untyped pointer result type is only legal on
2571 // OpUntypedVariableKHR, not on OpVariable.
2572 // Pick the matching opcode/operands so the synthesized constant global is
2573 // valid SPIR-V.
2574 const bool IsUntyped = VarTy->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
2575 auto MIBVar = BuildMI(*I.getParent(), I, I.getDebugLoc(),
2576 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
2577 : SPIRV::OpVariable))
2578 .addDef(VarReg)
2579 .addUse(GR.getSPIRVTypeID(VarTy))
2580 .addImm(SPIRV::StorageClass::UniformConstant);
2581 if (IsUntyped)
2582 MIBVar.addUse(GR.getSPIRVTypeID(SpvArrTy)); // Data Type
2583 MIBVar.addUse(Const); // Initializer
2584 MIBVar.constrainAllUses(TII, TRI, RBI);
2585
2586 GR.add(GV, MIBVar);
2587 GR.addGlobalObject(GV, GR.CurMF, VarReg);
2588
2589 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
2590 return VarReg;
2591}
2592
2593bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I,
2594 Register SrcReg) const {
2595 MachineBasicBlock &BB = *I.getParent();
2596 Register DstReg = I.getOperand(0).getReg();
2597 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(DstReg);
2598 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(SrcReg);
2599 if (GR.getPointeeType(DstTy) != GR.getPointeeType(SrcTy))
2600 return diagnoseUnsupported(
2601 I, "OpCopyMemory requires operands to have the same type");
2602 uint64_t CopySize = getIConstVal(I.getOperand(2).getReg(), MRI);
2603 SPIRVTypeInst PointeeTy = GR.getPointeeType(DstTy);
2604 const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(PointeeTy);
2605 if (!LLVMPointeeTy)
2606 return diagnoseUnsupported(
2607 I, "Unable to determine pointee type size for OpCopyMemory");
2608 const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
2609 if (CopySize != DL.getTypeStoreSize(const_cast<Type *>(LLVMPointeeTy)))
2610 return diagnoseUnsupported(
2611 I, "OpCopyMemory requires the size to match the pointee type size");
2612 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemory))
2613 .addUse(DstReg)
2614 .addUse(SrcReg);
2615 if (I.getNumMemOperands()) {
2616 MachineIRBuilder MIRBuilder(I);
2617 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2618 }
2619 MIB.constrainAllUses(TII, TRI, RBI);
2620 return true;
2621}
2622
2623bool SPIRVInstructionSelector::selectCopyMemorySized(MachineInstr &I,
2624 Register SrcReg) const {
2625 MachineBasicBlock &BB = *I.getParent();
2626 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
2627 .addUse(I.getOperand(0).getReg())
2628 .addUse(SrcReg)
2629 .addUse(I.getOperand(2).getReg());
2630 if (I.getNumMemOperands()) {
2631 MachineIRBuilder MIRBuilder(I);
2632 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2633 }
2634 MIB.constrainAllUses(TII, TRI, RBI);
2635 return true;
2636}
2637
2638bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
2639 MachineInstr &I) const {
2640 // Zero-sized memcpy/memmove/memset are no-ops.
2641 Register SizeReg = I.getOperand(2).getReg();
2642 if (MachineInstr *SizeDef = getDefInstrMaybeConstant(SizeReg, MRI);
2643 SizeDef && SizeDef->getOpcode() == TargetOpcode::G_CONSTANT &&
2644 getIConstVal(SizeReg, MRI) == 0)
2645 return true;
2646
2647 Register SrcReg = I.getOperand(1).getReg();
2648 if (I.getOpcode() == TargetOpcode::G_MEMSET ||
2649 I.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
2650 Register VarReg = getOrCreateMemSetGlobal(I);
2651 if (!VarReg.isValid())
2652 return false;
2653 Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext());
2654 SPIRVTypeInst SourceTy = GR.getOrCreateSPIRVPointerType(
2655 ValTy, I, SPIRV::StorageClass::UniformConstant);
2656 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2657 if (!selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast))
2658 return false;
2659 }
2660 if (STI.isLogicalSPIRV()) {
2661 if (!selectCopyMemory(I, SrcReg))
2662 return false;
2663 } else {
2664 if (!selectCopyMemorySized(I, SrcReg))
2665 return false;
2666 }
2667 if (ResVReg.isValid() && ResVReg != I.getOperand(0).getReg())
2668 if (!BuildCOPY(ResVReg, I.getOperand(0).getReg(), I))
2669 return false;
2670 return true;
2671}
2672
2673bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
2674 SPIRVTypeInst ResType,
2675 MachineInstr &I,
2676 unsigned NewOpcode,
2677 unsigned NegateOpcode) const {
2678 assert(I.hasOneMemOperand());
2679 const MachineMemOperand *MemOp = *I.memoperands_begin();
2680 uint32_t Scope = static_cast<uint32_t>(
2681 getMemScope(STI.getTargetTriple(), GR.CurMF->getFunction().getContext(),
2682 MemOp->getSyncScopeID()));
2683 Register ScopeReg = buildI32Constant(Scope, I);
2684
2685 Register Ptr = I.getOperand(1).getReg();
2686 uint32_t ScSem = static_cast<uint32_t>(
2687 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)));
2688 AtomicOrdering AO = MemOp->getSuccessOrdering();
2689 uint32_t OrderSem = static_cast<uint32_t>(getMemSemantics(AO));
2690 Register MemSemReg = buildI32Constant(
2691 getMemSemanticsWithStorageClass(STI.getTargetTriple(), OrderSem, ScSem),
2692 I);
2693
2694 Register ValueReg = I.getOperand(2).getReg();
2695 if (NegateOpcode != 0) {
2696 // Translation with negative value operand is requested
2697 Register TmpReg = createVirtualRegister(ResType, &GR, MRI, MRI->getMF());
2698 if (!selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode))
2699 return false;
2700 ValueReg = TmpReg;
2701 }
2702
2703 if (ResType.isTypePtr()) {
2704 if (NewOpcode != SPIRV::OpAtomicExchange)
2705 return diagnoseUnsupported(
2706 I, "Lowering to SPIR-V of this atomic operation is not "
2707 "allowed for pointer types");
2708 if (!STI.isPhysicalSPIRV())
2709 return diagnoseUnsupported(
2710 I, "Lowering to SPIR-V of atomic exchange is only "
2711 "allowed for pointer types for physical addressing model");
2712 // If the exchanged value is a pointer type we convert the value operand to
2713 // an integer type of the same size as the pointer size using
2714 // OpConvertPtrToU, bitcast the Ptr parameter to pointer to integer type and
2715 // then generate OpAtomicExchange on integers. The integer result is
2716 // converted back to a pointer type using OpConvertUToPtr, similar to atomic
2717 // load and store.
2718 MachineIRBuilder MIRBuilder(I);
2719 SPIRV::StorageClass::StorageClass SC = GR.getPointerStorageClass(Ptr);
2720 return selectAtomicPtrValue(
2721 ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) {
2722 Register ValueAsInt = convertPtrToInt(ValueReg, MIRBuilder);
2723 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2724 Register ExchangeResReg = createPtrSizedIntReg(MIRBuilder);
2725 MIRBuilder.buildInstr(SPIRV::OpAtomicExchange)
2726 .addDef(ExchangeResReg)
2727 .addUse(GR.getSPIRVTypeID(IntType))
2728 .addUse(CastedPtr)
2729 .addUse(ScopeReg)
2730 .addUse(MemSemReg)
2731 .addUse(ValueAsInt)
2732 .constrainAllUses(TII, TRI, RBI);
2733 return ExchangeResReg;
2734 });
2735 }
2736
2737 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
2738 .addDef(ResVReg)
2739 .addUse(GR.getSPIRVTypeID(ResType))
2740 .addUse(Ptr)
2741 .addUse(ScopeReg)
2742 .addUse(MemSemReg)
2743 .addUse(ValueReg)
2744 .constrainAllUses(TII, TRI, RBI);
2745 return true;
2746}
2747
2748bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
2749 unsigned ArgI = I.getNumOperands() - 1;
2750 Register SrcReg =
2751 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0);
2752 SPIRVTypeInst SrcType =
2753 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr;
2754 if (!SrcType || !isVectorType(SrcType))
2755 report_fatal_error(
2756 "cannot select G_UNMERGE_VALUES with a non-vector argument");
2757
2758 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(SrcType);
2759 MachineBasicBlock &BB = *I.getParent();
2760 unsigned CurrentIndex = 0;
2761 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2762 Register ResVReg = I.getOperand(i).getReg();
2763 SPIRVTypeInst ResType = GR.getSPIRVTypeForVReg(ResVReg);
2764 if (!ResType) {
2765 LLT ResLLT = MRI->getType(ResVReg);
2766 assert(ResLLT.isValid());
2767 if (ResLLT.isVector()) {
2768 ResType = GR.getOrCreateSPIRVVectorType(
2769 ScalarType, ResLLT.getNumElements(), I, TII);
2770 } else {
2771 ResType = ScalarType;
2772 }
2773 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
2774 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
2775 }
2776
2777 if (isVectorType(ResType)) {
2778 Register UndefReg = GR.getOrCreateUndef(I, SrcType, TII);
2779 auto MIB =
2780 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
2781 .addDef(ResVReg)
2782 .addUse(GR.getSPIRVTypeID(ResType))
2783 .addUse(SrcReg)
2784 .addUse(UndefReg);
2785 unsigned NumElements = GR.getScalarOrVectorComponentCount(ResType);
2786 for (unsigned j = 0; j < NumElements; ++j) {
2787 MIB.addImm(CurrentIndex + j);
2788 }
2789 CurrentIndex += NumElements;
2790 MIB.constrainAllUses(TII, TRI, RBI);
2791 } else {
2792 auto MIB =
2793 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2794 .addDef(ResVReg)
2795 .addUse(GR.getSPIRVTypeID(ResType))
2796 .addUse(SrcReg)
2797 .addImm(CurrentIndex);
2798 CurrentIndex++;
2799 MIB.constrainAllUses(TII, TRI, RBI);
2800 }
2801 }
2802 return true;
2803}
2804
2805bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
2806 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
2807 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
2808 Register MemSemReg = buildI32ConstantInEntryBlock(MemSem, I);
2809 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
2810 uint32_t Scope = static_cast<uint32_t>(getMemScope(
2811 STI.getTargetTriple(), GR.CurMF->getFunction().getContext(), Ord));
2812 Register ScopeReg = buildI32ConstantInEntryBlock(Scope, I);
2813 MachineBasicBlock &BB = *I.getParent();
2814 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
2815 .addUse(ScopeReg)
2816 .addUse(MemSemReg)
2817 .constrainAllUses(TII, TRI, RBI);
2818 return true;
2819}
2820
2821bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
2822 SPIRVTypeInst ResType,
2823 MachineInstr &I,
2824 unsigned Opcode) const {
2825 Type *ResTy = nullptr;
2826 StringRef ResName;
2827 if (!GR.findValueAttrs(&I, ResTy, ResName))
2828 return diagnoseUnsupported(
2829 I,
2830 "Not enough info to select the arithmetic with overflow instruction");
2831 if (!ResTy || !ResTy->isStructTy())
2832 return diagnoseUnsupported(I,
2833 "Expect struct type result for the arithmetic "
2834 "with overflow instruction");
2835 // "Result Type must be from OpTypeStruct. The struct must have two members,
2836 // and the two members must be the same type."
2837 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
2838 ResTy = StructType::get(ResElemTy, ResElemTy);
2839 // Build SPIR-V types and constant(s) if needed.
2840 MachineIRBuilder MIRBuilder(I);
2841 SPIRVTypeInst StructType = GR.getOrCreateSPIRVType(
2842 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
2843 assert(I.getNumDefs() > 1 && "Not enought operands");
2844 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
2845 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
2846 if (N > 1 || (isVectorType(ResType) &&
2847 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
2848 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
2849 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
2850 Register ZeroReg = buildZerosVal(ResType, I);
2851 // A new virtual register to store the result struct.
2852 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2853 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
2854 // Build the result name if needed.
2855 if (ResName.size() > 0)
2856 buildOpName(StructVReg, ResName, MIRBuilder);
2857 // Build the arithmetic with overflow instruction.
2858 MachineBasicBlock &BB = *I.getParent();
2859 auto MIB =
2860 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode))
2861 .addDef(StructVReg)
2862 .addUse(GR.getSPIRVTypeID(StructType));
2863 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
2864 MIB.addUse(I.getOperand(i).getReg());
2865 MIB.constrainAllUses(TII, TRI, RBI);
2866 // Build instructions to extract fields of the instruction's result.
2867 // A new virtual register to store the higher part of the result struct.
2868 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
2869 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass);
2870 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2871 auto MIB =
2872 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2873 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg())
2874 .addUse(GR.getSPIRVTypeID(ResType))
2875 .addUse(StructVReg)
2876 .addImm(i);
2877 MIB.constrainAllUses(TII, TRI, RBI);
2878 }
2879 // Build boolean value from the higher part.
2880 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
2881 .addDef(I.getOperand(1).getReg())
2882 .addUse(BoolTypeReg)
2883 .addUse(HigherVReg)
2884 .addUse(ZeroReg)
2885 .constrainAllUses(TII, TRI, RBI);
2886 return true;
2887}
2888
2889bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
2890 SPIRVTypeInst ResType,
2891 MachineInstr &I) const {
2892 assert(isa<GIntrinsic>(I) &&
2893 "selectAtomicCmpXchg only handles the spv_cmpxchg intrinsic");
2894 Register Ptr = I.getOperand(2).getReg();
2895 Register ScopeReg = I.getOperand(5).getReg();
2896 Register MemSemEqReg = I.getOperand(6).getReg();
2897 Register MemSemNeqReg = I.getOperand(7).getReg();
2898 Register Cmp = I.getOperand(3).getReg();
2899 Register Val = I.getOperand(4).getReg();
2900 SPIRVTypeInst SpvValTy = GR.getSPIRVTypeForVReg(Val);
2901 Register ACmpRes = createVirtualRegister(SpvValTy, &GR, MRI, *I.getMF());
2902 const DebugLoc &DL = I.getDebugLoc();
2903 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
2904 .addDef(ACmpRes)
2905 .addUse(GR.getSPIRVTypeID(SpvValTy))
2906 .addUse(Ptr)
2907 .addUse(ScopeReg)
2908 .addUse(MemSemEqReg)
2909 .addUse(MemSemNeqReg)
2910 .addUse(Val)
2911 .addUse(Cmp)
2912 .constrainAllUses(TII, TRI, RBI);
2913 SPIRVTypeInst BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
2914 Register CmpSuccReg = createVirtualRegister(BoolTy, &GR, MRI, *I.getMF());
2915 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
2916 .addDef(CmpSuccReg)
2917 .addUse(GR.getSPIRVTypeID(BoolTy))
2918 .addUse(ACmpRes)
2919 .addUse(Cmp)
2920 .constrainAllUses(TII, TRI, RBI);
2921 Register TmpReg = createVirtualRegister(ResType, &GR, MRI, *I.getMF());
2922 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
2923 .addDef(TmpReg)
2924 .addUse(GR.getSPIRVTypeID(ResType))
2925 .addUse(ACmpRes)
2926 .addUse(GR.getOrCreateUndef(I, ResType, TII))
2927 .addImm(0)
2928 .constrainAllUses(TII, TRI, RBI);
2929 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
2930 .addDef(ResVReg)
2931 .addUse(GR.getSPIRVTypeID(ResType))
2932 .addUse(CmpSuccReg)
2933 .addUse(TmpReg)
2934 .addImm(1)
2935 .constrainAllUses(TII, TRI, RBI);
2936 return true;
2937}
2938
2939static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
2940 switch (SC) {
2941 case SPIRV::StorageClass::DeviceOnlyINTEL:
2942 case SPIRV::StorageClass::HostOnlyINTEL:
2943 return true;
2944 default:
2945 return false;
2946 }
2947}
2948
2949// Returns true ResVReg is referred only from global vars and OpName's.
2950static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) {
2951 bool IsGRef = false;
2952 bool IsAllowedRefs =
2953 llvm::all_of(MRI->use_instructions(ResVReg), [&IsGRef](auto const &It) {
2954 unsigned Opcode = It.getOpcode();
2955 if (Opcode == SPIRV::OpConstantComposite ||
2956 Opcode == SPIRV::OpSpecConstantComposite ||
2957 Opcode == SPIRV::OpVariable ||
2958 Opcode == SPIRV::OpUntypedVariableKHR ||
2959 isSpvIntrinsic(It, Intrinsic::spv_init_global))
2960 return IsGRef = true;
2961 return Opcode == SPIRV::OpName;
2962 });
2963 return IsAllowedRefs && IsGRef;
2964}
2965
2966Register SPIRVInstructionSelector::getUcharPtrTypeReg(
2967 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
2968 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType(
2969 Type::getInt8Ty(I.getMF()->getFunction().getContext()), I, SC));
2970}
2971
2972MachineInstrBuilder
2973SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
2974 Register Src, Register DestType,
2975 uint32_t Opcode) const {
2976 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
2977 TII.get(SPIRV::OpSpecConstantOp))
2978 .addDef(Dest)
2979 .addUse(DestType)
2980 .addImm(Opcode)
2981 .addUse(Src);
2982}
2983
2984MachineInstrBuilder
2985SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
2986 SPIRVTypeInst SrcPtrTy) const {
2987 SPIRVTypeInst GenericPtrTy =
2988 GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I);
2989 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass);
2990 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace(
2991 SPIRV::StorageClass::Generic),
2992 GR.getPointerSize()));
2993 MachineFunction *MF = I.getParent()->getParent();
2994 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF);
2995 MachineInstrBuilder MIB = buildSpecConstantOp(
2996 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy),
2997 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
2998 GR.add(MIB.getInstr(), MIB);
2999 return MIB;
3000}
3001
3002// In SPIR-V address space casting can only happen to and from the Generic
3003// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
3004// pointers to and from Generic pointers. As such, we can convert e.g. from
3005// Workgroup to Function by going via a Generic pointer as an intermediary. All
3006// other combinations can only be done by a bitcast, and are probably not safe.
3007bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
3008 SPIRVTypeInst ResType,
3009 MachineInstr &I) const {
3010 MachineBasicBlock &BB = *I.getParent();
3011 const DebugLoc &DL = I.getDebugLoc();
3012
3013 Register SrcPtr = I.getOperand(1).getReg();
3014 SPIRVTypeInst SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
3015
3016 // don't generate a cast for a null that may be represented by OpTypeInt
3017 if (!SrcPtrTy || !SrcPtrTy.isPointer() || !ResType || !ResType.isPointer())
3018 return BuildCOPY(ResVReg, SrcPtr, I);
3019
3020 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy);
3021 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType);
3022
3023 if (isASCastInGVar(MRI, ResVReg)) {
3024 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
3025 // are expressed by OpSpecConstantOp with an Opcode.
3026 // TODO: maybe insert a check whether the Kernel capability was declared and
3027 // so PtrCastToGeneric/GenericCastToPtr are available.
3028 unsigned SpecOpcode = [&]() -> unsigned {
3029 if (SrcSC == SPIRV::StorageClass::CodeSectionINTEL)
3030 return static_cast<uint32_t>(SPIRV::Opcode::Bitcast);
3031 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
3032 return static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric);
3033 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
3034 return static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr);
3035 return 0u;
3036 }();
3037 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
3038 // correct value of ResType and use general i8* instead. Maybe this should
3039 // be addressed in the emit-intrinsic step to infer a correct
3040 // OpConstantComposite type.
3041 if (SpecOpcode) {
3042 buildSpecConstantOp(I, ResVReg, SrcPtr, getUcharPtrTypeReg(I, DstSC),
3043 SpecOpcode)
3044 .constrainAllUses(TII, TRI, RBI);
3045 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
3046 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
3047 MIB.constrainAllUses(TII, TRI, RBI);
3048 buildSpecConstantOp(
3049 I, ResVReg, MIB->getOperand(0).getReg(), getUcharPtrTypeReg(I, DstSC),
3050 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
3051 .constrainAllUses(TII, TRI, RBI);
3052 }
3053 return true;
3054 }
3055
3056 // don't generate a cast between identical storage classes
3057 if (SrcSC == DstSC)
3058 return BuildCOPY(ResVReg, SrcPtr, I);
3059
3060 if ((SrcSC == SPIRV::StorageClass::Function &&
3061 DstSC == SPIRV::StorageClass::Private) ||
3062 (DstSC == SPIRV::StorageClass::Function &&
3063 SrcSC == SPIRV::StorageClass::Private))
3064 return BuildCOPY(ResVReg, SrcPtr, I);
3065
3066 // Casting from an eligible pointer to Generic.
3067 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
3068 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
3069 // Casting from Generic to an eligible pointer.
3070 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
3071 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
3072 // Casting between 2 eligible pointers using Generic as an intermediary.
3073 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
3074 SPIRVTypeInst GenericPtrTy =
3075 GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I);
3076 Register Tmp = createVirtualRegister(GenericPtrTy, &GR, MRI, MRI->getMF());
3077 BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
3078 .addDef(Tmp)
3079 .addUse(GR.getSPIRVTypeID(GenericPtrTy))
3080 .addUse(SrcPtr)
3081 .constrainAllUses(TII, TRI, RBI);
3082 BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
3083 .addDef(ResVReg)
3084 .addUse(GR.getSPIRVTypeID(ResType))
3085 .addUse(Tmp)
3086 .constrainAllUses(TII, TRI, RBI);
3087 return true;
3088 }
3089
3090 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
3091 // be applied
3092 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
3093 return selectUnOp(ResVReg, ResType, I,
3094 SPIRV::OpPtrCastToCrossWorkgroupINTEL);
3095 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC))
3096 return selectUnOp(ResVReg, ResType, I,
3097 SPIRV::OpCrossWorkgroupCastToPtrINTEL);
3098 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic)
3099 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
3100 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC))
3101 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
3102
3103 // Bitcast for pointers requires that the address spaces must match
3104 return false;
3105}
3106
3107// G_PTRMASK - Apply a bitmask to a pointer value.
3108// Result = Ptr & Mask
3109// We need to convert the pointer to an integer, perform the AND operation,
3110// and convert back to a pointer.
3111bool SPIRVInstructionSelector::selectPtrMask(Register ResVReg,
3112 SPIRVTypeInst ResType,
3113 MachineInstr &I) const {
3114 if (STI.isLogicalSPIRV())
3115 return diagnoseUnsupported(
3116 I, "G_PTRMASK is not supported with logical SPIR-V");
3117 MachineBasicBlock &BB = *I.getParent();
3118 MachineFunction &MF = *BB.getParent();
3119 const DebugLoc &DL = I.getDebugLoc();
3120
3121 Register PtrReg = I.getOperand(1).getReg();
3122 Register MaskReg = I.getOperand(2).getReg();
3123
3124 SPIRVTypeInst MaskType = GR.getSPIRVTypeForVReg(MaskReg);
3125
3126 // Convert pointer to integer.
3127 Register PtrAsInt = MRI->createVirtualRegister(GR.getRegClass(MaskType));
3128 GR.assignSPIRVTypeToVReg(MaskType, PtrAsInt, MF);
3129
3130 BuildMI(BB, I, DL, TII.get(SPIRV::OpConvertPtrToU))
3131 .addDef(PtrAsInt)
3132 .addUse(GR.getSPIRVTypeID(MaskType))
3133 .addUse(PtrReg)
3134 .constrainAllUses(TII, TRI, RBI);
3135
3136 // Perform bitwise AND.
3137 Register MaskedInt = MRI->createVirtualRegister(GR.getRegClass(MaskType));
3138 GR.assignSPIRVTypeToVReg(MaskType, MaskedInt, MF);
3139
3140 unsigned AndOpcode = GR.getScalarOrVectorComponentCount(MaskType) > 1
3141 ? SPIRV::OpBitwiseAndV
3142 : SPIRV::OpBitwiseAndS;
3143
3144 BuildMI(BB, I, DL, TII.get(AndOpcode))
3145 .addDef(MaskedInt)
3146 .addUse(GR.getSPIRVTypeID(MaskType))
3147 .addUse(PtrAsInt)
3148 .addUse(MaskReg)
3149 .constrainAllUses(TII, TRI, RBI);
3150
3151 // Convert integer back to pointer.
3152 BuildMI(BB, I, DL, TII.get(SPIRV::OpConvertUToPtr))
3153 .addDef(ResVReg)
3154 .addUse(GR.getSPIRVTypeID(ResType))
3155 .addUse(MaskedInt)
3156 .constrainAllUses(TII, TRI, RBI);
3157
3158 return true;
3159}
3160
3161static unsigned getFCmpOpcode(unsigned PredNum) {
3162 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
3163 switch (Pred) {
3164 case CmpInst::FCMP_OEQ:
3165 return SPIRV::OpFOrdEqual;
3166 case CmpInst::FCMP_OGE:
3167 return SPIRV::OpFOrdGreaterThanEqual;
3168 case CmpInst::FCMP_OGT:
3169 return SPIRV::OpFOrdGreaterThan;
3170 case CmpInst::FCMP_OLE:
3171 return SPIRV::OpFOrdLessThanEqual;
3172 case CmpInst::FCMP_OLT:
3173 return SPIRV::OpFOrdLessThan;
3174 case CmpInst::FCMP_ONE:
3175 return SPIRV::OpFOrdNotEqual;
3176 case CmpInst::FCMP_ORD:
3177 return SPIRV::OpOrdered;
3178 case CmpInst::FCMP_UEQ:
3179 return SPIRV::OpFUnordEqual;
3180 case CmpInst::FCMP_UGE:
3181 return SPIRV::OpFUnordGreaterThanEqual;
3182 case CmpInst::FCMP_UGT:
3183 return SPIRV::OpFUnordGreaterThan;
3184 case CmpInst::FCMP_ULE:
3185 return SPIRV::OpFUnordLessThanEqual;
3186 case CmpInst::FCMP_ULT:
3187 return SPIRV::OpFUnordLessThan;
3188 case CmpInst::FCMP_UNE:
3189 return SPIRV::OpFUnordNotEqual;
3190 case CmpInst::FCMP_UNO:
3191 return SPIRV::OpUnordered;
3192 default:
3193 llvm_unreachable("Unknown predicate type for FCmp");
3194 }
3195}
3196
3197static unsigned getICmpOpcode(unsigned PredNum) {
3198 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
3199 switch (Pred) {
3200 case CmpInst::ICMP_EQ:
3201 return SPIRV::OpIEqual;
3202 case CmpInst::ICMP_NE:
3203 return SPIRV::OpINotEqual;
3204 case CmpInst::ICMP_SGE:
3205 return SPIRV::OpSGreaterThanEqual;
3206 case CmpInst::ICMP_SGT:
3207 return SPIRV::OpSGreaterThan;
3208 case CmpInst::ICMP_SLE:
3209 return SPIRV::OpSLessThanEqual;
3210 case CmpInst::ICMP_SLT:
3211 return SPIRV::OpSLessThan;
3212 case CmpInst::ICMP_UGE:
3213 return SPIRV::OpUGreaterThanEqual;
3214 case CmpInst::ICMP_UGT:
3215 return SPIRV::OpUGreaterThan;
3216 case CmpInst::ICMP_ULE:
3217 return SPIRV::OpULessThanEqual;
3218 case CmpInst::ICMP_ULT:
3219 return SPIRV::OpULessThan;
3220 default:
3221 llvm_unreachable("Unknown predicate type for ICmp");
3222 }
3223}
3224
3225static unsigned getPtrCmpOpcode(unsigned Pred) {
3226 switch (static_cast<CmpInst::Predicate>(Pred)) {
3227 case CmpInst::ICMP_EQ:
3228 return SPIRV::OpPtrEqual;
3229 case CmpInst::ICMP_NE:
3230 return SPIRV::OpPtrNotEqual;
3231 default:
3232 llvm_unreachable("Unknown predicate type for pointer comparison");
3233 }
3234}
3235
3236// Return the logical operation, or abort if none exists.
3237static unsigned getBoolCmpOpcode(unsigned PredNum) {
3238 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
3239 switch (Pred) {
3240 case CmpInst::ICMP_EQ:
3241 return SPIRV::OpLogicalEqual;
3242 case CmpInst::ICMP_NE:
3243 return SPIRV::OpLogicalNotEqual;
3244 default:
3245 llvm_unreachable("Unknown predicate type for Bool comparison");
3246 }
3247}
3248
3249static APFloat getZeroFP(const Type *LLVMFloatTy) {
3250 if (!LLVMFloatTy)
3251 return APFloat::getZero(APFloat::IEEEsingle());
3252 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
3253 case Type::HalfTyID:
3254 return APFloat::getZero(APFloat::IEEEhalf());
3255 case Type::BFloatTyID:
3256 return APFloat::getZero(APFloat::BFloat());
3257 default:
3258 case Type::FloatTyID:
3259 return APFloat::getZero(APFloat::IEEEsingle());
3260 case Type::DoubleTyID:
3261 return APFloat::getZero(APFloat::IEEEdouble());
3262 }
3263}
3264
3265static APFloat getOneFP(const Type *LLVMFloatTy) {
3266 if (!LLVMFloatTy)
3267 return APFloat::getOne(APFloat::IEEEsingle());
3268 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
3269 case Type::HalfTyID:
3270 return APFloat::getOne(APFloat::IEEEhalf());
3271 case Type::BFloatTyID:
3272 return APFloat::getOne(APFloat::BFloat());
3273 default:
3274 case Type::FloatTyID:
3275 return APFloat::getOne(APFloat::IEEEsingle());
3276 case Type::DoubleTyID:
3277 return APFloat::getOne(APFloat::IEEEdouble());
3278 }
3279}
3280
3281bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
3282 SPIRVTypeInst ResType,
3283 MachineInstr &I,
3284 unsigned OpAnyOrAll) const {
3285 assert(I.getNumOperands() == 3);
3286 assert(I.getOperand(2).isReg());
3287 MachineBasicBlock &BB = *I.getParent();
3288 Register InputRegister = I.getOperand(2).getReg();
3289 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3290
3291 assert(InputType && "VReg has no type assigned");
3292
3293 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool);
3294 if (IsBoolTy && !isVectorType(InputType)) {
3295 assert(ResVReg == I.getOperand(0).getReg());
3296 return BuildCOPY(ResVReg, InputRegister, I);
3297 }
3298
3299 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3300 unsigned SpirvNotEqualId =
3301 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
3302 SPIRVTypeInst SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
3303 SPIRVTypeInst SpvBoolTy = SpvBoolScalarTy;
3304 Register NotEqualReg = ResVReg;
3305
3306 if (isVectorType(InputType)) {
3307 NotEqualReg =
3308 IsBoolTy ? InputRegister
3309 : createVirtualRegister(SpvBoolTy, &GR, MRI, MRI->getMF());
3310 const unsigned NumElts = GR.getScalarOrVectorComponentCount(InputType);
3311 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII);
3312 }
3313
3314 if (!IsBoolTy) {
3315 Register ConstZeroReg =
3316 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I);
3317
3318 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId))
3319 .addDef(NotEqualReg)
3320 .addUse(GR.getSPIRVTypeID(SpvBoolTy))
3321 .addUse(InputRegister)
3322 .addUse(ConstZeroReg)
3323 .constrainAllUses(TII, TRI, RBI);
3324 }
3325
3326 if (isVectorType(InputType))
3327 BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
3328 .addDef(ResVReg)
3329 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy))
3330 .addUse(NotEqualReg)
3331 .constrainAllUses(TII, TRI, RBI);
3332 return true;
3333}
3334
3335bool SPIRVInstructionSelector::selectAll(Register ResVReg,
3336 SPIRVTypeInst ResType,
3337 MachineInstr &I) const {
3338 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
3339}
3340
3341bool SPIRVInstructionSelector::selectAny(Register ResVReg,
3342 SPIRVTypeInst ResType,
3343 MachineInstr &I) const {
3344 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
3345}
3346
3347// Select the OpDot instruction for the given float dot
3348bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
3349 SPIRVTypeInst ResType,
3350 MachineInstr &I) const {
3351 assert(I.getNumOperands() == 4);
3352 assert(I.getOperand(2).isReg());
3353 assert(I.getOperand(3).isReg());
3354
3355 [[maybe_unused]] SPIRVTypeInst VecType =
3356 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg());
3357
3358 assert(((VecType->getOpcode() == SPIRV::OpTypeVector &&
3359 GR.getScalarOrVectorComponentCount(VecType) > 1) ||
3360 VecType->getOpcode() == SPIRV::OpTypeVectorIdEXT) &&
3361 "dot product requires either a vector of at least 2 components or"
3362 " the SPV_EXT_long vector extension.");
3363
3364 [[maybe_unused]] SPIRVTypeInst EltType =
3365 GR.getScalarOrVectorComponentType(VecType);
3366
3367 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
3368
3369 MachineBasicBlock &BB = *I.getParent();
3370 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot))
3371 .addDef(ResVReg)
3372 .addUse(GR.getSPIRVTypeID(ResType))
3373 .addUse(I.getOperand(2).getReg())
3374 .addUse(I.getOperand(3).getReg())
3375 .constrainAllUses(TII, TRI, RBI);
3376 return true;
3377}
3378
3379bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
3380 SPIRVTypeInst ResType,
3381 MachineInstr &I,
3382 bool Signed) const {
3383 assert(I.getNumOperands() == 4);
3384 assert(I.getOperand(2).isReg());
3385 assert(I.getOperand(3).isReg());
3386 MachineBasicBlock &BB = *I.getParent();
3387
3388 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
3389 BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
3390 .addDef(ResVReg)
3391 .addUse(GR.getSPIRVTypeID(ResType))
3392 .addUse(I.getOperand(2).getReg())
3393 .addUse(I.getOperand(3).getReg())
3394 .constrainAllUses(TII, TRI, RBI);
3395 return true;
3396}
3397
3398// Since pre-1.6 SPIRV has no integer dot implementation,
3399// expand by piecewise multiplying and adding the results
3400bool SPIRVInstructionSelector::selectIntegerDotExpansion(
3401 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3402 assert(I.getNumOperands() == 4);
3403 assert(I.getOperand(2).isReg());
3404 assert(I.getOperand(3).isReg());
3405 MachineBasicBlock &BB = *I.getParent();
3406
3407 // Multiply the vectors, then sum the results
3408 Register Vec0 = I.getOperand(2).getReg();
3409 Register Vec1 = I.getOperand(3).getReg();
3410 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType));
3411 SPIRVTypeInst VecType = GR.getSPIRVTypeForVReg(Vec0);
3412
3413 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV))
3414 .addDef(TmpVec)
3415 .addUse(GR.getSPIRVTypeID(VecType))
3416 .addUse(Vec0)
3417 .addUse(Vec1)
3418 .constrainAllUses(TII, TRI, RBI);
3419
3420 assert(((VecType->getOpcode() == SPIRV::OpTypeVector &&
3421 GR.getScalarOrVectorComponentCount(VecType) > 1) ||
3422 VecType->getOpcode() == SPIRV::OpTypeVectorIdEXT) &&
3423 "dot product requires either a vector of at least 2 components "
3424 "or the SPV_EXT_long_vector extension.");
3425
3426 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType));
3427 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
3428 .addDef(Res)
3429 .addUse(GR.getSPIRVTypeID(ResType))
3430 .addUse(TmpVec)
3431 .addImm(0)
3432 .constrainAllUses(TII, TRI, RBI);
3433
3434 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) {
3435 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType));
3436
3437 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
3438 .addDef(Elt)
3439 .addUse(GR.getSPIRVTypeID(ResType))
3440 .addUse(TmpVec)
3441 .addImm(i)
3442 .constrainAllUses(TII, TRI, RBI);
3443
3444 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1
3445 ? MRI->createVirtualRegister(GR.getRegClass(ResType))
3446 : ResVReg;
3447
3448 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3449 .addDef(Sum)
3450 .addUse(GR.getSPIRVTypeID(ResType))
3451 .addUse(Res)
3452 .addUse(Elt)
3453 .constrainAllUses(TII, TRI, RBI);
3454 Res = Sum;
3455 }
3456
3457 return true;
3458}
3459
3460bool SPIRVInstructionSelector::selectOpIsInf(Register ResVReg,
3461 SPIRVTypeInst ResType,
3462 MachineInstr &I) const {
3463 MachineBasicBlock &BB = *I.getParent();
3464 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsInf))
3465 .addDef(ResVReg)
3466 .addUse(GR.getSPIRVTypeID(ResType))
3467 .addUse(I.getOperand(2).getReg())
3468 .constrainAllUses(TII, TRI, RBI);
3469 return true;
3470}
3471
3472bool SPIRVInstructionSelector::selectOpIsNan(Register ResVReg,
3473 SPIRVTypeInst ResType,
3474 MachineInstr &I) const {
3475 MachineBasicBlock &BB = *I.getParent();
3476 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNan))
3477 .addDef(ResVReg)
3478 .addUse(GR.getSPIRVTypeID(ResType))
3479 .addUse(I.getOperand(2).getReg())
3480 .constrainAllUses(TII, TRI, RBI);
3481 return true;
3482}
3483
3484bool SPIRVInstructionSelector::selectOpIsFinite(Register ResVReg,
3485 SPIRVTypeInst ResType,
3486 MachineInstr &I) const {
3487 MachineBasicBlock &BB = *I.getParent();
3488 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsFinite))
3489 .addDef(ResVReg)
3490 .addUse(GR.getSPIRVTypeID(ResType))
3491 .addUse(I.getOperand(2).getReg())
3492 .constrainAllUses(TII, TRI, RBI);
3493 return true;
3494}
3495
3496bool SPIRVInstructionSelector::selectOpIsNormal(Register ResVReg,
3497 SPIRVTypeInst ResType,
3498 MachineInstr &I) const {
3499 MachineBasicBlock &BB = *I.getParent();
3500 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNormal))
3501 .addDef(ResVReg)
3502 .addUse(GR.getSPIRVTypeID(ResType))
3503 .addUse(I.getOperand(2).getReg())
3504 .constrainAllUses(TII, TRI, RBI);
3505 return true;
3506}
3507
3508template <bool Signed>
3509bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
3510 SPIRVTypeInst ResType,
3511 MachineInstr &I) const {
3512 assert(I.getNumOperands() == 5);
3513 assert(I.getOperand(2).isReg());
3514 assert(I.getOperand(3).isReg());
3515 assert(I.getOperand(4).isReg());
3516 MachineBasicBlock &BB = *I.getParent();
3517
3518 Register Acc = I.getOperand(2).getReg();
3519 Register X = I.getOperand(3).getReg();
3520 Register Y = I.getOperand(4).getReg();
3521
3522 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
3523 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType));
3524 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
3525 .addDef(Dot)
3526 .addUse(GR.getSPIRVTypeID(ResType))
3527 .addUse(X)
3528 .addUse(Y);
3529 MIB.addImm(SPIRV::BuiltIn::PackedVectorFormat4x8Bit);
3530 MIB.constrainAllUses(TII, TRI, RBI);
3531
3532 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3533 .addDef(ResVReg)
3534 .addUse(GR.getSPIRVTypeID(ResType))
3535 .addUse(Dot)
3536 .addUse(Acc)
3537 .constrainAllUses(TII, TRI, RBI);
3538 return true;
3539}
3540
3541// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
3542// extract the elements of the packed inputs, multiply them and add the result
3543// to the accumulator.
3544template <bool Signed>
3545bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
3546 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3547 assert(I.getNumOperands() == 5);
3548 assert(I.getOperand(2).isReg());
3549 assert(I.getOperand(3).isReg());
3550 assert(I.getOperand(4).isReg());
3551 MachineBasicBlock &BB = *I.getParent();
3552
3553 Register Acc = I.getOperand(2).getReg();
3554 Register X = I.getOperand(3).getReg();
3555 Register Y = I.getOperand(4).getReg();
3556
3557 SPIRVTypeInst EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII);
3558 auto ExtractOp =
3559 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
3560
3561 bool ZeroAsNull = !STI.isShader();
3562 // Extract the i8 element, multiply and add it to the accumulator
3563 for (unsigned i = 0; i < 4; i++) {
3564 // A[i]
3565 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3566 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3567 .addDef(AElt)
3568 .addUse(GR.getSPIRVTypeID(ResType))
3569 .addUse(X)
3570 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull))
3571 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3572 .constrainAllUses(TII, TRI, RBI);
3573
3574 // B[i]
3575 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3576 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3577 .addDef(BElt)
3578 .addUse(GR.getSPIRVTypeID(ResType))
3579 .addUse(Y)
3580 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull))
3581 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3582 .constrainAllUses(TII, TRI, RBI);
3583
3584 // A[i] * B[i]
3585 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3586 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS))
3587 .addDef(Mul)
3588 .addUse(GR.getSPIRVTypeID(ResType))
3589 .addUse(AElt)
3590 .addUse(BElt)
3591 .constrainAllUses(TII, TRI, RBI);
3592
3593 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
3594 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3595 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3596 .addDef(MaskMul)
3597 .addUse(GR.getSPIRVTypeID(ResType))
3598 .addUse(Mul)
3599 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII, ZeroAsNull))
3600 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3601 .constrainAllUses(TII, TRI, RBI);
3602
3603 // Acc = Acc + A[i] * B[i]
3604 Register Sum =
3605 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg;
3606 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3607 .addDef(Sum)
3608 .addUse(GR.getSPIRVTypeID(ResType))
3609 .addUse(Acc)
3610 .addUse(MaskMul)
3611 .constrainAllUses(TII, TRI, RBI);
3612
3613 Acc = Sum;
3614 }
3615
3616 return true;
3617}
3618
3619/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
3620/// does not have a saturate builtin.
3621bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
3622 SPIRVTypeInst ResType,
3623 MachineInstr &I) const {
3624 assert(I.getNumOperands() == 3);
3625 assert(I.getOperand(2).isReg());
3626 MachineBasicBlock &BB = *I.getParent();
3627 Register VZero = buildZerosValF(ResType, I);
3628 Register VOne = buildOnesValF(ResType, I);
3629
3630 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
3631 .addDef(ResVReg)
3632 .addUse(GR.getSPIRVTypeID(ResType))
3633 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3634 .addImm(GL::FClamp)
3635 .addUse(I.getOperand(2).getReg())
3636 .addUse(VZero)
3637 .addUse(VOne)
3638 .constrainAllUses(TII, TRI, RBI);
3639 return true;
3640}
3641
3642bool SPIRVInstructionSelector::selectSign(Register ResVReg,
3643 SPIRVTypeInst ResType,
3644 MachineInstr &I) const {
3645 assert(I.getNumOperands() == 3);
3646 assert(I.getOperand(2).isReg());
3647 MachineBasicBlock &BB = *I.getParent();
3648 Register InputRegister = I.getOperand(2).getReg();
3649 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3650 auto &DL = I.getDebugLoc();
3651
3652 if (!InputType)
3653 return diagnoseUnsupported(I, "Input Type could not be determined.");
3654
3655 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3656
3657 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType);
3658 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType);
3659
3660 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
3661
3662 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
3663 Register SignReg = NeedsConversion
3664 ? MRI->createVirtualRegister(&SPIRV::IDRegClass)
3665 : ResVReg;
3666
3667 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst))
3668 .addDef(SignReg)
3669 .addUse(GR.getSPIRVTypeID(InputType))
3670 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3671 .addImm(SignOpcode)
3672 .addUse(InputRegister)
3673 .constrainAllUses(TII, TRI, RBI);
3674
3675 if (NeedsConversion) {
3676 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
3677 BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode))
3678 .addDef(ResVReg)
3679 .addUse(GR.getSPIRVTypeID(ResType))
3680 .addUse(SignReg)
3681 .constrainAllUses(TII, TRI, RBI);
3682 }
3683
3684 return true;
3685}
3686
3687bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
3688 SPIRVTypeInst ResType,
3689 MachineInstr &I,
3690 unsigned Opcode) const {
3691 MachineBasicBlock &BB = *I.getParent();
3692 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3693
3694 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3695 .addDef(ResVReg)
3696 .addUse(GR.getSPIRVTypeID(ResType))
3697 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I,
3698 IntTy, TII, !STI.isShader()));
3699
3700 for (unsigned J = 2; J < I.getNumOperands(); J++) {
3701 BMI.addUse(I.getOperand(J).getReg());
3702 }
3703
3704 BMI.constrainAllUses(TII, TRI, RBI);
3705 return true;
3706}
3707
3708bool SPIRVInstructionSelector::selectBarrierInst(MachineInstr &I,
3709 unsigned Scope,
3710 unsigned MemSem,
3711 bool WithGroupSync) const {
3712 auto BarrierType =
3713 WithGroupSync ? SPIRV::OpControlBarrier : SPIRV::OpMemoryBarrier;
3714
3715 MemSem |= SPIRV::MemorySemantics::AcquireRelease;
3716
3717 assert(((Scope != SPIRV::Scope::Workgroup) ||
3718 ((MemSem & SPIRV::MemorySemantics::WorkgroupMemory) > 0)) &&
3719 "Workgroup Scope must set WorkGroupMemory semantic "
3720 "in Barrier instruction");
3721
3722 assert(((Scope != SPIRV::Scope::Device) ||
3723 ((MemSem & SPIRV::MemorySemantics::UniformMemory) > 0 &&
3724 (MemSem & SPIRV::MemorySemantics::ImageMemory) > 0)) &&
3725 "Device Scope must set UniformMemory and ImageMemory semantic "
3726 "in Barrier instruction");
3727
3728 MachineBasicBlock &BB = *I.getParent();
3729 auto MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType));
3730
3731 // OpControlBarrier needs to also set Execution Scope
3732 if (WithGroupSync) {
3733 Register ExecReg = buildI32Constant(SPIRV::Scope::Workgroup, I);
3734 MI.addUse(ExecReg);
3735 }
3736
3737 Register ScopeReg = buildI32Constant(Scope, I);
3738 Register MemSemReg = buildI32Constant(MemSem, I);
3739
3740 MI.addUse(ScopeReg).addUse(MemSemReg).constrainAllUses(TII, TRI, RBI);
3741 return true;
3742}
3743
3744bool SPIRVInstructionSelector::selectWaveActiveCountBits(
3745 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3746
3747 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3748 SPIRVTypeInst BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII);
3749 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType));
3750 if (!selectWaveOpInst(BallotReg, BallotType, I,
3751 SPIRV::OpGroupNonUniformBallot))
3752 return false;
3753
3754 MachineBasicBlock &BB = *I.getParent();
3755 BuildMI(BB, I, I.getDebugLoc(),
3756 TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3757 .addDef(ResVReg)
3758 .addUse(GR.getSPIRVTypeID(ResType))
3759 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
3760 !STI.isShader()))
3761 .addImm(SPIRV::GroupOperation::Reduce)
3762 .addUse(BallotReg)
3763 .constrainAllUses(TII, TRI, RBI);
3764
3765 return true;
3766}
3767
3768bool SPIRVInstructionSelector::selectWaveActiveAllEqual(Register ResVReg,
3769 SPIRVTypeInst ResType,
3770 MachineInstr &I) const {
3771 MachineBasicBlock &BB = *I.getParent();
3772 const DebugLoc &DL = I.getDebugLoc();
3773
3774 // Input to the intrinsic
3775 Register InputReg = I.getOperand(2).getReg();
3776 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputReg);
3777
3778 // Determine if input is vector
3779 unsigned NumElems = GR.getScalarOrVectorComponentCount(InputType);
3780 bool IsVector = NumElems > 1 ||
3781 (InputType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
3782 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector));
3783
3784 // Determine element types
3785 SPIRVTypeInst ElemInputType = GR.getScalarOrVectorComponentType(InputType);
3786 SPIRVTypeInst ElemBoolType = GR.getScalarOrVectorComponentType(ResType);
3787
3788 // Subgroup scope constant
3789 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3790 Register ScopeConst = GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy,
3791 TII, !STI.isShader());
3792
3793 // Scalar case
3794 if (!IsVector) {
3795 return selectWaveOpInst(ResVReg, ElemBoolType, I,
3796 SPIRV::OpGroupNonUniformAllEqual);
3797 }
3798
3799 // Vector case
3800 SmallVector<Register, 4> ElementResults;
3801 ElementResults.reserve(NumElems);
3802
3803 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
3804 // Extract element
3805 Register ElemInput = InputReg;
3806 Register Extracted =
3807 MRI->createVirtualRegister(GR.getRegClass(ElemInputType));
3808
3809 BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeExtract))
3810 .addDef(Extracted)
3811 .addUse(GR.getSPIRVTypeID(ElemInputType))
3812 .addUse(InputReg)
3813 .addImm(Idx)
3814 .constrainAllUses(TII, TRI, RBI);
3815
3816 ElemInput = Extracted;
3817
3818 // Emit per-element AllEqual
3819 Register ElemResult =
3820 MRI->createVirtualRegister(GR.getRegClass(ElemBoolType));
3821
3822 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformAllEqual))
3823 .addDef(ElemResult)
3824 .addUse(GR.getSPIRVTypeID(ElemBoolType))
3825 .addUse(ScopeConst)
3826 .addUse(ElemInput)
3827 .constrainAllUses(TII, TRI, RBI);
3828
3829 ElementResults.push_back(ElemResult);
3830 }
3831
3832 // Reconstruct vector<bool>
3833 auto MIB = BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeConstruct))
3834 .addDef(ResVReg)
3835 .addUse(GR.getSPIRVTypeID(ResType));
3836 for (Register R : ElementResults)
3837 MIB.addUse(R);
3838
3839 MIB.constrainAllUses(TII, TRI, RBI);
3840
3841 return true;
3842}
3843
3844bool SPIRVInstructionSelector::selectWavePrefixBitCount(Register ResVReg,
3845 SPIRVTypeInst ResType,
3846 MachineInstr &I) const {
3847
3848 assert(I.getNumOperands() == 3);
3849
3850 auto Op = I.getOperand(2);
3851 assert(Op.isReg());
3852
3853 MachineBasicBlock &BB = *I.getParent();
3854 DebugLoc DL = I.getDebugLoc();
3855
3856 Register InputRegister = Op.getReg();
3857 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3858
3859 if (!InputType)
3860 return diagnoseUnsupported(I, "Input Type could not be determined.");
3861
3862 if (InputType->getOpcode() != SPIRV::OpTypeBool)
3863 return diagnoseUnsupported(I, "WavePrefixBitCount requires boolean input");
3864
3865 // Types
3866 SPIRVTypeInst Int32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3867
3868 // Ballot result type: vector<uint32>
3869 // Match DXC: %v4uint for Subgroup size
3870 SPIRVTypeInst BallotTy = GR.getOrCreateSPIRVVectorType(Int32Ty, 4, I, TII);
3871
3872 // Create a vreg for the ballot result
3873 Register BallotVReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3874
3875 // 1. OpGroupNonUniformBallot
3876 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallot))
3877 .addDef(BallotVReg)
3878 .addUse(GR.getSPIRVTypeID(BallotTy))
3879 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII))
3880 .addUse(InputRegister)
3881 .constrainAllUses(TII, TRI, RBI);
3882
3883 // 2. OpGroupNonUniformBallotBitCount
3884 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3885 .addDef(ResVReg)
3886 .addUse(GR.getSPIRVTypeID(ResType))
3887 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII))
3888 .addImm(SPIRV::GroupOperation::ExclusiveScan)
3889 .addUse(BallotVReg)
3890 .constrainAllUses(TII, TRI, RBI);
3891
3892 return true;
3893}
3894
3895bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
3896 SPIRVTypeInst ResType,
3897 MachineInstr &I,
3898 bool IsUnsigned) const {
3899 return selectWaveReduce(
3900 ResVReg, ResType, I, IsUnsigned,
3901 [&](Register InputRegister, bool IsUnsigned) {
3902 const bool IsFloatTy =
3903 GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3904 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax
3905 : SPIRV::OpGroupNonUniformSMax;
3906 return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp;
3907 });
3908}
3909
3910bool SPIRVInstructionSelector::selectWaveReduceMin(Register ResVReg,
3911 SPIRVTypeInst ResType,
3912 MachineInstr &I,
3913 bool IsUnsigned) const {
3914 return selectWaveReduce(
3915 ResVReg, ResType, I, IsUnsigned,
3916 [&](Register InputRegister, bool IsUnsigned) {
3917 const bool IsFloatTy =
3918 GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3919 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin
3920 : SPIRV::OpGroupNonUniformSMin;
3921 return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp;
3922 });
3923}
3924
3925bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
3926 SPIRVTypeInst ResType,
3927 MachineInstr &I) const {
3928 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3929 [&](Register InputRegister, bool IsUnsigned) {
3930 bool IsFloatTy = GR.isScalarOrVectorOfType(
3931 InputRegister, SPIRV::OpTypeFloat);
3932 return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
3933 : SPIRV::OpGroupNonUniformIAdd;
3934 });
3935}
3936
3937bool SPIRVInstructionSelector::selectWaveReduceProduct(Register ResVReg,
3938 SPIRVTypeInst ResType,
3939 MachineInstr &I) const {
3940 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3941 [&](Register InputRegister, bool IsUnsigned) {
3942 bool IsFloatTy = GR.isScalarOrVectorOfType(
3943 InputRegister, SPIRV::OpTypeFloat);
3944 return IsFloatTy ? SPIRV::OpGroupNonUniformFMul
3945 : SPIRV::OpGroupNonUniformIMul;
3946 });
3947}
3948
3949template <typename PickOpcodeFn>
3950bool SPIRVInstructionSelector::selectWaveReduce(
3951 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
3952 PickOpcodeFn &&PickOpcode) const {
3953 assert(I.getNumOperands() == 3);
3954 assert(I.getOperand(2).isReg());
3955 MachineBasicBlock &BB = *I.getParent();
3956 Register InputRegister = I.getOperand(2).getReg();
3957 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3958
3959 if (!InputType)
3960 return diagnoseUnsupported(I, "Input Type could not be determined.");
3961
3962 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3963 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
3964 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3965 .addDef(ResVReg)
3966 .addUse(GR.getSPIRVTypeID(ResType))
3967 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
3968 !STI.isShader()))
3969 .addImm(SPIRV::GroupOperation::Reduce)
3970 .addUse(I.getOperand(2).getReg())
3971 .constrainAllUses(TII, TRI, RBI);
3972 return true;
3973}
3974
3975bool SPIRVInstructionSelector::selectWaveReduceOp(Register ResVReg,
3976 SPIRVTypeInst ResType,
3977 MachineInstr &I,
3978 unsigned Opcode) const {
3979 return selectWaveReduce(
3980 ResVReg, ResType, I, false,
3981 [&](Register InputRegister, bool IsUnsigned) { return Opcode; });
3982}
3983
3984bool SPIRVInstructionSelector::selectWaveExclusiveScanSum(
3985 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3986 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
3987 [&](Register InputRegister, bool IsUnsigned) {
3988 bool IsFloatTy = GR.isScalarOrVectorOfType(
3989 InputRegister, SPIRV::OpTypeFloat);
3990 return IsFloatTy
3991 ? SPIRV::OpGroupNonUniformFAdd
3992 : SPIRV::OpGroupNonUniformIAdd;
3993 });
3994}
3995
3996bool SPIRVInstructionSelector::selectWaveExclusiveScanProduct(
3997 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3998 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
3999 [&](Register InputRegister, bool IsUnsigned) {
4000 bool IsFloatTy = GR.isScalarOrVectorOfType(
4001 InputRegister, SPIRV::OpTypeFloat);
4002 return IsFloatTy
4003 ? SPIRV::OpGroupNonUniformFMul
4004 : SPIRV::OpGroupNonUniformIMul;
4005 });
4006}
4007
4008template <typename PickOpcodeFn>
4009bool SPIRVInstructionSelector::selectWaveExclusiveScan(
4010 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
4011 PickOpcodeFn &&PickOpcode) const {
4012 assert(I.getNumOperands() == 3);
4013 assert(I.getOperand(2).isReg());
4014 MachineBasicBlock &BB = *I.getParent();
4015 Register InputRegister = I.getOperand(2).getReg();
4016 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
4017
4018 if (!InputType)
4019 return diagnoseUnsupported(I, "Input Type could not be determined.");
4020
4021 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4022 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
4023 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4024 .addDef(ResVReg)
4025 .addUse(GR.getSPIRVTypeID(ResType))
4026 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
4027 !STI.isShader()))
4028 .addImm(SPIRV::GroupOperation::ExclusiveScan)
4029 .addUse(I.getOperand(2).getReg())
4030 .constrainAllUses(TII, TRI, RBI);
4031 return true;
4032}
4033
4034bool SPIRVInstructionSelector::selectQuadSwap(Register ResVReg,
4035 SPIRVTypeInst ResType,
4036 MachineInstr &I,
4037 unsigned Direction) const {
4038 assert(I.getNumOperands() == 3);
4039 assert(I.getOperand(2).isReg());
4040 MachineBasicBlock &BB = *I.getParent();
4041 Register InputRegister = I.getOperand(2).getReg();
4042
4043 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4044 bool ZeroAsNull = !STI.isShader();
4045 Register DirectionReg =
4046 GR.getOrCreateConstInt(Direction, I, IntTy, TII, ZeroAsNull);
4047 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGroupNonUniformQuadSwap))
4048 .addDef(ResVReg)
4049 .addUse(GR.getSPIRVTypeID(ResType))
4050 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
4051 ZeroAsNull))
4052 .addUse(InputRegister)
4053 .addUse(DirectionReg)
4054 .constrainAllUses(TII, TRI, RBI);
4055 return true;
4056}
4057
4058bool SPIRVInstructionSelector::selectBitreverseViaI32(Register ResVReg,
4059 SPIRVTypeInst ResType,
4060 MachineInstr &I,
4061 Register Op) const {
4062 SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4063 const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4064 Register ShiftConst =
4065 GR.getOrCreateConstInt(32 - BitWidth, I, Int32Type, TII);
4066 unsigned ShiftOp = SPIRV::OpShiftRightLogicalS;
4067
4068 const unsigned N = GR.getScalarOrVectorComponentCount(ResType);
4069 const unsigned ExtendOpcode = GR.isScalarOrVectorSigned(ResType)
4070 ? SPIRV::OpSConvert
4071 : SPIRV::OpUConvert;
4072
4073 if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4074 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) {
4075 Int32Type = GR.getOrCreateSPIRVVectorType(Int32Type, N, I, TII);
4076 ShiftOp = SPIRV::OpShiftRightLogicalV;
4077
4078 // Vector shifts require a composite constant
4079 const Register CompositeReg =
4080 MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4081 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4082 TII.get(SPIRV::OpConstantComposite))
4083 .addDef(CompositeReg)
4084 .addUse(GR.getSPIRVTypeID(Int32Type));
4085 for (unsigned It = 0; It < N; ++It)
4086 MIB.addUse(ShiftConst);
4087 MIB.constrainAllUses(TII, TRI, RBI);
4088
4089 ShiftConst = CompositeReg;
4090 }
4091
4092 // Converts the input to i32 (or vector of i32)
4093 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4094 if (!selectOpWithSrcs(ExtReg, Int32Type, I, {Op}, ExtendOpcode))
4095 return false;
4096
4097 // Perform bitreverse on the i32 value
4098 Register BitrevReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4099 if (!selectBitreverseNative(BitrevReg, Int32Type, I, ExtReg))
4100 return false;
4101
4102 // Shift the bit-reversed value to get the final result.
4103 Register ShiftReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4104 if (!selectOpWithSrcs(ShiftReg, Int32Type, I, {BitrevReg, ShiftConst},
4105 ShiftOp))
4106 return false;
4107
4108 // Finally, convert the result back.
4109 return selectOpWithSrcs(ResVReg, ResType, I, {ShiftReg}, ExtendOpcode);
4110}
4111
4112bool SPIRVInstructionSelector::handle64BitOverflow(
4113 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
4114 unsigned int Opcode,
4115 std::function<bool(Register, SPIRVTypeInst, MachineInstr &, Register,
4116 unsigned)>
4117 CallbackFunction) const {
4118
4119 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType);
4120 assert(BaseType->getOpcode() == SPIRV::OpTypeInt &&
4121 "handle64BitOverflow should only be used for integer types");
4122 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
4123 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
4124
4125 MachineIRBuilder MIRBuilder(I);
4126 SPIRVTypeInst I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder);
4127 SPIRVTypeInst I64x2Type =
4128 GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder, false);
4129 SPIRVTypeInst Vec2ResType =
4130 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder, false);
4131
4132 std::vector<Register> PartialRegs;
4133
4134 unsigned CurrentComponent = 0;
4135 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
4136 Register PopCountResult =
4137 MRI->createVirtualRegister(GR.getRegClass(I64x2Type));
4138
4139 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4140 TII.get(SPIRV::OpVectorShuffle))
4141 .addDef(PopCountResult)
4142 .addUse(GR.getSPIRVTypeID(I64x2Type))
4143 .addUse(SrcReg)
4144 .addUse(SrcReg)
4145 .addImm(CurrentComponent)
4146 .addImm(CurrentComponent + 1);
4147
4148 MIB.constrainAllUses(TII, TRI, RBI);
4149
4150 Register SubVecReg =
4151 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType));
4152
4153 if (!CallbackFunction(SubVecReg, Vec2ResType, I, PopCountResult, Opcode))
4154 return false;
4155
4156 PartialRegs.push_back(SubVecReg);
4157 }
4158 // On odd component counts we need to handle one more component
4159 if (CurrentComponent != ComponentCount) {
4160 bool ZeroAsNull = !STI.isShader();
4161 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type));
4162 Register ConstIntLastIdx = GR.getOrCreateConstInt(
4163 ComponentCount - 1, I, BaseType, TII, ZeroAsNull);
4164
4165 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx},
4166 SPIRV::OpVectorExtractDynamic))
4167 return false;
4168
4169 Register FinalElemResReg =
4170 MRI->createVirtualRegister(GR.getRegClass(BaseType));
4171
4172 if (!CallbackFunction(FinalElemResReg, BaseType, I, FinalElemReg, Opcode))
4173 return false;
4174
4175 PartialRegs.push_back(FinalElemResReg);
4176 }
4177 // Join all the resulting registers back into the return type in order
4178 // (ie i32x2, i32x2, i32x1 -> i32x5)
4179 return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs,
4180 SPIRV::OpCompositeConstruct);
4181}
4182
4183bool SPIRVInstructionSelector::selectBitreverse64(Register ResVReg,
4184 SPIRVTypeInst ResType,
4185 MachineInstr &I,
4186 Register SrcReg) const {
4187 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
4188 if (ComponentCount > 2)
4189 return handle64BitOverflow(
4190 ResVReg, ResType, I, SrcReg, SPIRV::OpBitReverse,
4191 [this](Register R, SPIRVTypeInst T, MachineInstr &I, Register S,
4192 unsigned O) { return this->selectBitreverse64(R, T, I, S); });
4193
4194 MachineIRBuilder MIRBuilder(I);
4195
4196 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
4197 SPIRVTypeInst VecI32Type = GR.getOrCreateSPIRVVectorType(
4198 I32Type, 2 * ComponentCount, MIRBuilder, /*IsSigned=*/false);
4199
4200 // Converts 64 bit into and array of 32 bit, containing 2 elements.
4201 Register Vec32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
4202 if (!selectOpWithSrcs(Vec32, VecI32Type, I, {SrcReg}, SPIRV::OpBitcast))
4203 return false;
4204
4205 // Apply bitreverse on each 32 bit lane
4206 Register Reverse32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
4207 if (!selectBitreverseNative(Reverse32, VecI32Type, I, Vec32))
4208 return false;
4209
4210 // Reversing a 64-bit value = reverse each 32-bit half AND swap them,
4211 // so the old High word becomes lane 0 (low) and old Low becomes lane 1
4212 // (high).
4213 Register SwappedVec = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
4214 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4215 TII.get(SPIRV::OpVectorShuffle))
4216 .addDef(SwappedVec)
4217 .addUse(GR.getSPIRVTypeID(VecI32Type))
4218 .addUse(Reverse32)
4219 .addUse(Reverse32);
4220 for (unsigned J = 0; J < ComponentCount; ++J) {
4221 MIB.addImm(2 * J + 1);
4222 MIB.addImm(2 * J);
4223 }
4224 MIB.constrainAllUses(TII, TRI, RBI);
4225
4226 // Groups 32 bit vector back to 64 bit scalar.
4227 return selectOpWithSrcs(ResVReg, ResType, I, {SwappedVec}, SPIRV::OpBitcast);
4228}
4229
4230bool SPIRVInstructionSelector::selectBitreverseNative(Register ResVReg,
4231 SPIRVTypeInst ResType,
4232 MachineInstr &I,
4233 Register Op) const {
4234 MachineBasicBlock &BB = *I.getParent();
4235 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
4236 .addDef(ResVReg)
4237 .addUse(GR.getSPIRVTypeID(ResType))
4238 .addUse(Op)
4239 .constrainAllUses(TII, TRI, RBI);
4240 return true;
4241}
4242
4243bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
4244 SPIRVTypeInst ResType,
4245 MachineInstr &I) const {
4246 Register OpReg = I.getOperand(1).getReg();
4247
4248 // TODO: Fix shader behavior in case of VK_KHR_maintenance9 extension is set
4249 if (STI.isShader()) {
4250 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
4251 switch (GR.getScalarOrVectorBitWidth(OpType)) {
4252 case 8:
4253 case 16:
4254 case 24:
4255 return selectBitreverseViaI32(ResVReg, ResType, I, OpReg);
4256 case 32:
4257 return selectBitreverseNative(ResVReg, ResType, I, OpReg);
4258 case 64:
4259 return selectBitreverse64(ResVReg, ResType, I, OpReg);
4260 }
4261 return SPIRVInstructionSelector::diagnoseUnsupported(
4262 I, "G_BITREVERSE only support 16,32,64 bits.");
4263 }
4264
4265 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions))
4266 return selectBitreverseNative(ResVReg, ResType, I, OpReg);
4267
4268 // Expansion bitreverse using bit manipulation operations
4269 // Algo: https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
4270 const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4271 // TODO: add support for any bit width and bitwidth more than 64.
4272 if (BitWidth > 64 || !isPowerOf2_32(BitWidth))
4273 return false;
4274
4275 const unsigned N = GR.getScalarOrVectorComponentCount(ResType);
4276
4277 unsigned AndOp = SPIRV::OpBitwiseAndS;
4278 unsigned OrOp = SPIRV::OpBitwiseOrS;
4279 unsigned ShlOp = SPIRV::OpShiftLeftLogicalS;
4280 unsigned ShrOp = SPIRV::OpShiftRightLogicalS;
4281 if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4282 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) {
4283 AndOp = SPIRV::OpBitwiseAndV;
4284 OrOp = SPIRV::OpBitwiseOrV;
4285 ShlOp = SPIRV::OpShiftLeftLogicalV;
4286 ShrOp = SPIRV::OpShiftRightLogicalV;
4287 }
4288
4289 // Helper, one swap per step: ((input>>shift)&mask)|((input&mask)<<shift),
4290 // RPN: input shift >> mask & input mask & shift << |
4291 auto SwapBits = [&](const Register Input, const uint64_t Mask,
4292 const unsigned Shift) -> Register {
4293 auto CreateConst = [&](const uint64_t Value) -> Register {
4294 if (N == 1 &&
4295 (ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT ||
4296 !STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
4297 return GR.getOrCreateConstInt(
4298 Value, I, GR.retrieveScalarOrVectorIntType(ResType), TII);
4299 return GR.getOrCreateConstVector(Value, I, ResType, TII);
4300 };
4301
4302 Register MaskReg = CreateConst(Mask);
4303 Register ShiftReg = CreateConst(Shift);
4304 Register T1 = MRI->createVirtualRegister(GR.getRegClass(ResType));
4305 Register T2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
4306 Register T3 = MRI->createVirtualRegister(GR.getRegClass(ResType));
4307 Register T4 = MRI->createVirtualRegister(GR.getRegClass(ResType));
4308 Register Result = MRI->createVirtualRegister(GR.getRegClass(ResType));
4309
4310 if (!selectOpWithSrcs(T1, ResType, I, {Input, ShiftReg}, ShrOp) ||
4311 !selectOpWithSrcs(T2, ResType, I, {T1, MaskReg}, AndOp) ||
4312 !selectOpWithSrcs(T3, ResType, I, {Input, MaskReg}, AndOp) ||
4313 !selectOpWithSrcs(T4, ResType, I, {T3, ShiftReg}, ShlOp) ||
4314 !selectOpWithSrcs(Result, ResType, I, {T2, T4}, OrOp))
4315 return Register();
4316
4317 return Result;
4318 };
4319
4320 unsigned Shift = BitWidth;
4321 Register Result = OpReg;
4322 uint64_t Mask = ~0ull;
4323 while ((Shift >>= 1) > 0) {
4324 Mask ^= (Mask << Shift);
4325 Result = SwapBits(Result, Mask, Shift);
4326 if (!Result.isValid())
4327 return false;
4328 }
4329
4330 return BuildCOPY(ResVReg, Result, I);
4331}
4332
4333bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
4334 SPIRVTypeInst ResType,
4335 MachineInstr &I) const {
4336 assert(I.getOperand(0).isReg() && I.getOperand(1).isReg() &&
4337 "G_FREEZE must define and use a register");
4338 Register OpReg = I.getOperand(1).getReg();
4339
4340 // With SPV_KHR_poison_freeze, lower `freeze` to OpFreezeKHR.
4341 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_poison_freeze)) {
4342 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFreezeKHR))
4343 .addDef(ResVReg)
4344 .addUse(GR.getSPIRVTypeID(ResType))
4345 .addUse(OpReg)
4346 .constrainAllUses(TII, TRI, RBI);
4347 return true;
4348 }
4349
4350 // There is no way to implement `freeze` correctly without support on SPIR-V
4351 // standard side, but we may at least address a simple (static) case when
4352 // undef/poison value presence is obvious. The main benefit of even
4353 // incomplete `freeze` support is preventing of translation from crashing due
4354 // to lack of support on legalization and instruction selection steps.
4355 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) {
4356 if (Def->getOpcode() == TargetOpcode::COPY)
4357 Def = MRI->getVRegDef(Def->getOperand(1).getReg());
4358 Register Reg;
4359 switch (Def->getOpcode()) {
4360 case SPIRV::ASSIGN_TYPE:
4361 if (MachineInstr *AssignToDef =
4362 MRI->getVRegDef(Def->getOperand(1).getReg())) {
4363 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
4364 Reg = Def->getOperand(2).getReg();
4365 }
4366 break;
4367 case SPIRV::OpUndef:
4368 Reg = Def->getOperand(1).getReg();
4369 break;
4370 }
4371 unsigned DestOpCode;
4372 if (Reg.isValid()) {
4373 DestOpCode = SPIRV::OpConstantNull;
4374 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. freeze of a "
4375 "static undef/poison lowered to OpConstantNull\n");
4376 } else {
4377 DestOpCode = TargetOpcode::COPY;
4378 Reg = OpReg;
4379 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. freeze "
4380 "skipped, lowered as a copy of the operand\n");
4381 }
4382 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode))
4383 .addDef(I.getOperand(0).getReg())
4384 .addUse(Reg)
4385 .constrainAllUses(TII, TRI, RBI);
4386 return true;
4387 }
4388 return false;
4389}
4390
4391bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
4392 SPIRVTypeInst ResType,
4393 MachineInstr &I) const {
4394 unsigned N = 0;
4395 if (isVectorType(ResType))
4396 N = GR.getScalarOrVectorComponentCount(ResType);
4397 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
4398 N = getArrayComponentCount(MRI, ResType);
4399 else
4400 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result");
4401 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
4402 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent");
4403
4404 // check if we may construct a constant vector
4405 bool IsConst = true;
4406 for (unsigned i = I.getNumExplicitDefs();
4407 i < I.getNumExplicitOperands() && IsConst; ++i)
4408 if (!isConstReg(MRI, I.getOperand(i).getReg()))
4409 IsConst = false;
4410
4411 if (!IsConst && (N < 2 && ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT))
4412 return diagnoseUnsupported(
4413 I, "There must be at least two constituent operands in a vector");
4414
4415 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
4416
4417 bool IsNullVector = IsConst && !STI.isShader();
4418 for (unsigned i = I.getNumExplicitDefs();
4419 i < I.getNumExplicitOperands() && IsNullVector; ++i) {
4420 MachineInstr *Def = getDef(I.getOperand(i), MRI);
4421 IsNullVector = Def && isNullOrNullSplat(*Def, *MRI);
4422 }
4423
4424 if (IsNullVector) {
4425 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
4426 .addDef(ResVReg)
4427 .addUse(GR.getSPIRVTypeID(ResType))
4428 .constrainAllUses(TII, TRI, RBI);
4429 return true;
4430 }
4431
4432 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4433 TII.get(IsConst ? SPIRV::OpConstantComposite
4434 : SPIRV::OpCompositeConstruct))
4435 .addDef(ResVReg)
4436 .addUse(GR.getSPIRVTypeID(ResType));
4437 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
4438 MIB.addUse(I.getOperand(i).getReg());
4439 MIB.constrainAllUses(TII, TRI, RBI);
4440 return true;
4441}
4442
4443bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
4444 SPIRVTypeInst ResType,
4445 MachineInstr &I) const {
4446 unsigned N = 0;
4447 if (isVectorType(ResType))
4448 N = GR.getScalarOrVectorComponentCount(ResType);
4449 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
4450 N = getArrayComponentCount(MRI, ResType);
4451 else
4452 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result");
4453
4454 unsigned OpIdx = I.getNumExplicitDefs();
4455 if (!I.getOperand(OpIdx).isReg())
4456 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR");
4457
4458 // check if we may construct a constant vector
4459 Register OpReg = I.getOperand(OpIdx).getReg();
4460 bool IsConst = isConstReg(MRI, OpReg);
4461
4462 if (!IsConst && (N < 2 && ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT))
4463 return diagnoseUnsupported(
4464 I, "There must be at least two constituent operands in a vector");
4465
4466 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
4467 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4468 TII.get(IsConst ? SPIRV::OpConstantComposite
4469 : SPIRV::OpCompositeConstruct))
4470 .addDef(ResVReg)
4471 .addUse(GR.getSPIRVTypeID(ResType));
4472 for (unsigned i = 0; i < N; ++i)
4473 MIB.addUse(OpReg);
4474 MIB.constrainAllUses(TII, TRI, RBI);
4475 return true;
4476}
4477
4478bool SPIRVInstructionSelector::selectConcatVectors(Register ResVReg,
4479 SPIRVTypeInst ResType,
4480 MachineInstr &I) const {
4481 // Implement G_CONCAT_VECTORS using OpCompositeConstruct, which allows vector
4482 // constituents that share the result's component type to be
4483 // concatenated in operand order.
4484 if (!isVectorType(ResType))
4485 report_fatal_error(
4486 "Cannot select G_CONCAT_VECTORS with a non-vector result");
4487
4488 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4489 TII.get(SPIRV::OpCompositeConstruct))
4490 .addDef(ResVReg)
4491 .addUse(GR.getSPIRVTypeID(ResType));
4492 for (unsigned OpIdx = I.getNumExplicitDefs();
4493 OpIdx < I.getNumExplicitOperands(); ++OpIdx)
4494 MIB.addUse(I.getOperand(OpIdx).getReg());
4495 MIB.constrainAllUses(TII, TRI, RBI);
4496 return true;
4497}
4498
4499bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
4500 SPIRVTypeInst ResType,
4501 MachineInstr &I) const {
4502
4503 unsigned Opcode;
4504 MachineBasicBlock &BB = *I.getParent();
4505
4506 if (STI.canUseExtension(
4507 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
4508 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) {
4509 Opcode = SPIRV::OpDemoteToHelperInvocation;
4510 } else {
4511 Opcode = SPIRV::OpKill;
4512 // OpKill must be the last operation of any basic block.
4513 for (MachineInstr &ToErase : make_early_inc_range(
4514 make_range(std::next(I.getIterator()), BB.instr_end()))) {
4515 GR.invalidateMachineInstr(&ToErase);
4516 ToErase.eraseFromParent();
4517 }
4518 }
4519
4520 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4521 .constrainAllUses(TII, TRI, RBI);
4522 return true;
4523}
4524
4525bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
4526 SPIRVTypeInst ResType, unsigned CmpOpc,
4527 MachineInstr &I) const {
4528 Register Cmp0 = I.getOperand(2).getReg();
4529 Register Cmp1 = I.getOperand(3).getReg();
4530 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
4531 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
4532 "CMP operands should have the same type");
4533 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
4534 .addDef(ResVReg)
4535 .addUse(GR.getSPIRVTypeID(ResType))
4536 .addUse(Cmp0)
4537 .addUse(Cmp1)
4538 .setMIFlags(I.getFlags())
4539 .constrainAllUses(TII, TRI, RBI);
4540 return true;
4541}
4542
4543bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
4544 SPIRVTypeInst ResType,
4545 MachineInstr &I) const {
4546 auto Pred = I.getOperand(1).getPredicate();
4547 unsigned CmpOpc;
4548
4549 Register CmpOperand = I.getOperand(2).getReg();
4550 SPIRVTypeInst CmpOperandType = GR.getSPIRVTypeForVReg(CmpOperand);
4551 bool IsPtrCmp = CmpOperandType && CmpOperandType.isPointer();
4552 if (IsPtrCmp) {
4553 CmpOpc = getPtrCmpOpcode(Pred);
4554 // OpPtrEqual/OpPtrNotEqual require both operands to share an identical
4555 // pointer type. If they are not OpBitcast is inserted.
4556 Register Op1 = I.getOperand(3).getReg();
4557 SPIRVTypeInst Ty0 = GR.getSPIRVTypeForVReg(CmpOperand);
4558 if (Ty0 != GR.getSPIRVTypeForVReg(Op1)) {
4559 Register NewOp1 = createVirtualRegister(Ty0, &GR, MRI, MRI->getMF());
4560 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
4561 .addDef(NewOp1)
4562 .addUse(GR.getSPIRVTypeID(Ty0))
4563 .addUse(Op1)
4564 .constrainAllUses(TII, TRI, RBI);
4565 I.getOperand(3).setReg(NewOp1);
4566 }
4567 } else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
4568 CmpOpc = getBoolCmpOpcode(Pred);
4569 else
4570 CmpOpc = getICmpOpcode(Pred);
4571 return selectCmp(ResVReg, ResType, CmpOpc, I);
4572}
4573
4574Register
4575SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
4576 SPIRVTypeInst ResType) const {
4577 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
4578 SPIRVTypeInst SpvI32Ty =
4579 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
4580 // Find a constant in DT or build a new one.
4581 auto ConstInt = ConstantInt::get(LLVMTy, Val);
4582 Register NewReg = GR.find(ConstInt, GR.CurMF);
4583 if (!NewReg.isValid()) {
4584 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
4585 MachineBasicBlock &BB = *I.getParent();
4586 MachineInstr *MI =
4587 Val == 0
4588 ? BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
4589 .addDef(NewReg)
4590 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
4591 : BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
4592 .addDef(NewReg)
4593 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
4594 .addImm(APInt(32, Val).getZExtValue());
4595 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
4596 GR.add(ConstInt, MI);
4597 }
4598 return NewReg;
4599}
4600
4601// Like buildI32Constant, but always inserts the constant definition in the
4602// entry block so it dominates all uses regardless of block ordering.
4603Register SPIRVInstructionSelector::buildI32ConstantInEntryBlock(
4604 uint32_t Val, MachineInstr &I, SPIRVTypeInst ResType) const {
4605 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
4606 SPIRVTypeInst SpvI32Ty =
4607 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
4608 auto *ConstInt = ConstantInt::get(LLVMTy, Val);
4609 Register NewReg = GR.find(ConstInt, GR.CurMF);
4610 if (!NewReg.isValid()) {
4611 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
4612 auto InsertIt = getOpVariableMBBIt(*I.getMF());
4613 MachineBasicBlock &EntryBB = *InsertIt->getParent();
4614 MachineInstr *MI = nullptr;
4615 Register TypeReg = GR.getSPIRVTypeID(SpvI32Ty);
4616 DebugLoc DbgLoc = I.getDebugLoc();
4617 if (Val == 0) {
4618 MI = BuildMI(EntryBB, InsertIt, DbgLoc, TII.get(SPIRV::OpConstantNull))
4619 .addDef(NewReg)
4620 .addUse(TypeReg);
4621 } else {
4622 uint64_t ImmVal = APInt(32, Val).getZExtValue();
4623 MI = BuildMI(EntryBB, InsertIt, DbgLoc, TII.get(SPIRV::OpConstantI))
4624 .addDef(NewReg)
4625 .addUse(TypeReg)
4626 .addImm(ImmVal);
4627 }
4628 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
4629 GR.add(ConstInt, MI);
4630 }
4631 return NewReg;
4632}
4633
4634bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
4635 SPIRVTypeInst ResType,
4636 MachineInstr &I) const {
4637 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
4638 return selectCmp(ResVReg, ResType, CmpOp, I);
4639}
4640
4641bool SPIRVInstructionSelector::selectExp10(Register ResVReg,
4642 SPIRVTypeInst ResType,
4643 MachineInstr &I) const {
4644 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
4645 return selectExtInst(ResVReg, ResType, I, CL::exp10);
4646 }
4647
4648 if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
4649 /// There is no exp10 in GLSL. Use exp10(x) = exp2(x * log2(10)) instead
4650 /// log2(10) ~= 3.3219280948874l
4651
4652 if (!isVectorType(ResType) && !ResType.isAnyTypeFloat())
4653 return false;
4654
4655 MachineIRBuilder MIRBuilder(I);
4656
4657 SPIRVTypeInst SpirvScalarType = GR.getScalarOrVectorComponentType(ResType);
4658
4659 // Match the literal precision to the scalar type so the OpConstant
4660 // literal does not contain non-zero high-order bits that would fail
4661 // SPIR-V validation when the type is narrower than 32 bits (e.g. half).
4662 APFloat ConstVal(3.3219280948873623);
4663 bool LosesInfo;
4664 ConstVal.convert(
4665 getZeroFP(GR.getTypeForSPIRVType(SpirvScalarType)).getSemantics(),
4666 APFloat::rmNearestTiesToEven, &LosesInfo);
4667 Register ConstReg =
4668 GR.buildConstantFP(ConstVal, MIRBuilder, SpirvScalarType);
4669 Register ArgReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
4670 auto Opcode =
4671 isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS;
4672
4673 if (!selectOpWithSrcs(ArgReg, ResType, I,
4674 {I.getOperand(1).getReg(), ConstReg}, Opcode))
4675 return false;
4676 if (!selectExtInst(ResVReg, ResType, I,
4677 {{SPIRV::InstructionSet::GLSL_std_450, GL::Exp2}}, false,
4678 false, {ArgReg}))
4679 return false;
4680
4681 return true;
4682 }
4683
4684 return false;
4685}
4686
4687Register SPIRVInstructionSelector::buildZerosVal(SPIRVTypeInst ResType,
4688 MachineInstr &I) const {
4689 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
4690 bool ZeroAsNull = !STI.isShader();
4691 if (isVectorType(ResType))
4692 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
4693 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
4694}
4695
4696bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero(
4697 Register Reg) const {
4698 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(Reg);
4699 if (!Type)
4700 return false;
4701 SPIRVTypeInst CompType = GR.getScalarOrVectorComponentType(Type);
4702 if (!CompType || CompType->getOpcode() != SPIRV::OpTypeInt)
4703 return false;
4704
4705 auto IsZero = [this](Register Reg) {
4706 MachineInstr *Def = getDefInstrMaybeConstant(Reg, MRI);
4707 if (!Def)
4708 return false;
4709
4710 if (Def->getOpcode() == SPIRV::OpConstantNull)
4711 return true;
4712
4713 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
4714 Def->getOpcode() == SPIRV::OpConstantI)
4715 return getIConstVal(Reg, MRI) == 0;
4716
4717 return false;
4718 };
4719
4720 if (IsZero(Reg))
4721 return true;
4722
4723 MachineInstr *Def = MRI->getVRegDef(Reg);
4724 if (!Def)
4725 return false;
4726
4727 if (Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
4728 (Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
4729 cast<GIntrinsic>(Def)->getIntrinsicID() ==
4730 Intrinsic::spv_const_composite)) {
4731 unsigned StartOp = Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2;
4732 for (unsigned i = StartOp; i < Def->getNumOperands(); ++i) {
4733 if (!IsZero(Def->getOperand(i).getReg()))
4734 return false;
4735 }
4736 return true;
4737 }
4738
4739 return false;
4740}
4741
4742Register SPIRVInstructionSelector::buildZerosValF(SPIRVTypeInst ResType,
4743 MachineInstr &I) const {
4744 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
4745 bool ZeroAsNull = !STI.isShader();
4746 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType));
4747 if (isVectorType(ResType))
4748 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
4749 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull);
4750}
4751
4752Register SPIRVInstructionSelector::buildOnesValF(SPIRVTypeInst ResType,
4753 MachineInstr &I) const {
4754 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
4755 bool ZeroAsNull = !STI.isShader();
4756 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType));
4757 if (isVectorType(ResType))
4758 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull);
4759 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull);
4760}
4761
4762Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
4763 SPIRVTypeInst ResType,
4764 MachineInstr &I) const {
4765 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4766 APInt One =
4767 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
4768 if (isVectorType(ResType))
4769 return GR.getOrCreateConstVector(One, I, ResType, TII);
4770 return GR.getOrCreateConstInt(One, I, ResType, TII);
4771}
4772
4773bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
4774 SPIRVTypeInst ResType,
4775 MachineInstr &I) const {
4776 Register SelectFirstArg = I.getOperand(2).getReg();
4777 Register SelectSecondArg = I.getOperand(3).getReg();
4778 assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) &&
4779 ResType == GR.getSPIRVTypeForVReg(SelectSecondArg));
4780
4781 bool IsFloatTy =
4782 GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypeFloat);
4783 bool IsPtrTy =
4784 GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypePointer);
4785
4786 bool IsScalarBool =
4787 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
4788 unsigned Opcode;
4789 if (isVectorType(GR.getSPIRVTypeForVReg(SelectFirstArg))) {
4790 if (IsFloatTy) {
4791 Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
4792 } else if (IsPtrTy) {
4793 Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
4794 } else {
4795 Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
4796 }
4797 } else {
4798 assert(IsScalarBool && "OpSelect with a scalar result requires a scalar "
4799 "boolean condition");
4800 if (IsFloatTy) {
4801 Opcode = SPIRV::OpSelectSFSCond;
4802 } else if (IsPtrTy) {
4803 Opcode = SPIRV::OpSelectSPSCond;
4804 } else {
4805 Opcode = SPIRV::OpSelectSISCond;
4806 }
4807 }
4808 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
4809 .addDef(ResVReg)
4810 .addUse(GR.getSPIRVTypeID(ResType))
4811 .addUse(I.getOperand(1).getReg())
4812 .addUse(SelectFirstArg)
4813 .addUse(SelectSecondArg)
4814 .constrainAllUses(TII, TRI, RBI);
4815 return true;
4816}
4817
4818// This function is used to extend a bool or a vector of bools into an integer
4819// or vector of integers.
4820bool SPIRVInstructionSelector::selectBoolToInt(Register ResVReg,
4821 SPIRVTypeInst ResType,
4822 Register BooleanVReg,
4823 MachineInstr &InsertAt,
4824 bool IsSigned) const {
4825 // To extend a bool, we need to use OpSelect between constants.
4826 Register ZeroReg = buildZerosVal(ResType, InsertAt);
4827 Register OneReg = buildOnesVal(IsSigned, ResType, InsertAt);
4828 bool IsScalarBool = GR.isScalarOfType(BooleanVReg, SPIRV::OpTypeBool);
4829 unsigned Opcode =
4830 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
4831 BuildMI(*InsertAt.getParent(), InsertAt, InsertAt.getDebugLoc(),
4832 TII.get(Opcode))
4833 .addDef(ResVReg)
4834 .addUse(GR.getSPIRVTypeID(ResType))
4835 .addUse(BooleanVReg)
4836 .addUse(OneReg)
4837 .addUse(ZeroReg)
4838 .constrainAllUses(TII, TRI, RBI);
4839 return true;
4840}
4841
4842bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
4843 SPIRVTypeInst ResType,
4844 MachineInstr &I, bool IsSigned,
4845 unsigned Opcode) const {
4846 Register SrcReg = I.getOperand(1).getReg();
4847 // We can convert bool value directly to float type without OpConvert*ToF,
4848 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
4849 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
4850 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4851 SPIRVTypeInst TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
4852 if (isVectorType(ResType)) {
4853 const unsigned NumElts = GR.getScalarOrVectorComponentCount(ResType);
4854 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
4855 }
4856 SrcReg = createVirtualRegister(TmpType, &GR, MRI, MRI->getMF());
4857 selectBoolToInt(SrcReg, TmpType, I.getOperand(1).getReg(), I, IsSigned);
4858 }
4859 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
4860}
4861
4862bool SPIRVInstructionSelector::selectExt(Register ResVReg,
4863 SPIRVTypeInst ResType, MachineInstr &I,
4864 bool IsSigned) const {
4865 Register SrcReg = I.getOperand(1).getReg();
4866 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool))
4867 return selectBoolToInt(ResVReg, ResType, I.getOperand(1).getReg(), I,
4868 IsSigned);
4869
4870 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg);
4871 if (ResType == SrcType)
4872 return BuildCOPY(ResVReg, SrcReg, I);
4873
4874 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
4875 return selectUnOp(ResVReg, ResType, I, Opcode);
4876}
4877
4878bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
4879 SPIRVTypeInst ResType,
4880 MachineInstr &I,
4881 bool IsSigned) const {
4882 MachineIRBuilder MIRBuilder(I);
4883 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
4884 MachineBasicBlock &BB = *I.getParent();
4885 // Ensure we have bool.
4886 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
4887 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
4888 if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4889 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
4890 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
4891 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
4892 // Build less-than-equal and less-than.
4893 Register IsLessEqReg =
4894 createVirtualRegister(BoolType, &GR, MRI, MIRBuilder.getMF());
4895 BuildMI(BB, I, I.getDebugLoc(),
4896 TII.get(IsSigned ? SPIRV::OpSLessThanEqual : SPIRV::OpULessThanEqual))
4897 .addDef(IsLessEqReg)
4898 .addUse(BoolTypeReg)
4899 .addUse(I.getOperand(1).getReg())
4900 .addUse(I.getOperand(2).getReg())
4901 .constrainAllUses(TII, TRI, RBI);
4902 Register IsLessReg =
4903 createVirtualRegister(BoolType, &GR, MRI, MIRBuilder.getMF());
4904 BuildMI(BB, I, I.getDebugLoc(),
4905 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
4906 .addDef(IsLessReg)
4907 .addUse(BoolTypeReg)
4908 .addUse(I.getOperand(1).getReg())
4909 .addUse(I.getOperand(2).getReg())
4910 .constrainAllUses(TII, TRI, RBI);
4911 // Build selects.
4912 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
4913 Register NegOneOrZeroReg =
4914 MRI->createVirtualRegister(GR.getRegClass(ResType));
4915 MRI->setType(NegOneOrZeroReg, LLT::scalar(64));
4916 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF());
4917 unsigned SelectOpcode =
4918 (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4919 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
4920 ? SPIRV::OpSelectVIVCond
4921 : SPIRV::OpSelectSISCond;
4922 BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
4923 .addDef(NegOneOrZeroReg)
4924 .addUse(ResTypeReg)
4925 .addUse(IsLessReg)
4926 .addUse(buildOnesVal(true, ResType, I)) // -1
4927 .addUse(buildZerosVal(ResType, I))
4928 .constrainAllUses(TII, TRI, RBI);
4929 BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
4930 .addDef(ResVReg)
4931 .addUse(ResTypeReg)
4932 .addUse(IsLessEqReg)
4933 .addUse(NegOneOrZeroReg) // -1 or 0
4934 .addUse(buildOnesVal(false, ResType, I))
4935 .constrainAllUses(TII, TRI, RBI);
4936 return true;
4937}
4938
4939bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
4940 Register ResVReg,
4941 MachineInstr &I,
4942 SPIRVTypeInst IntTy,
4943 SPIRVTypeInst BoolTy) const {
4944 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
4945 Register BitIntReg = createVirtualRegister(IntTy, &GR, MRI, MRI->getMF());
4946 unsigned Opcode =
4947 isVectorType(IntTy) ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
4948 Register Zero = buildZerosVal(IntTy, I);
4949 Register One = buildOnesVal(false, IntTy, I);
4950 MachineBasicBlock &BB = *I.getParent();
4951 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4952 .addDef(BitIntReg)
4953 .addUse(GR.getSPIRVTypeID(IntTy))
4954 .addUse(IntReg)
4955 .addUse(One)
4956 .constrainAllUses(TII, TRI, RBI);
4957 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
4958 .addDef(ResVReg)
4959 .addUse(GR.getSPIRVTypeID(BoolTy))
4960 .addUse(BitIntReg)
4961 .addUse(Zero)
4962 .constrainAllUses(TII, TRI, RBI);
4963 return true;
4964}
4965
4966bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
4967 SPIRVTypeInst ResType,
4968 MachineInstr &I) const {
4969 Register IntReg = I.getOperand(1).getReg();
4970 const SPIRVTypeInst ArgType = GR.getSPIRVTypeForVReg(IntReg);
4971 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool))
4972 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
4973 if (ArgType == ResType)
4974 return BuildCOPY(ResVReg, IntReg, I);
4975 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
4976 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
4977 return selectUnOp(ResVReg, ResType, I, Opcode);
4978}
4979
4980bool SPIRVInstructionSelector::selectConst(Register ResVReg,
4981 SPIRVTypeInst ResType,
4982 MachineInstr &I) const {
4983 unsigned Opcode = I.getOpcode();
4984 unsigned TpOpcode = ResType->getOpcode();
4985 Register Reg;
4986 if (ResType.isPointer() || TpOpcode == SPIRV::OpTypeEvent) {
4987 assert(Opcode == TargetOpcode::G_CONSTANT &&
4988 I.getOperand(1).getCImm()->isZero());
4989 MachineBasicBlock &DepMBB = I.getMF()->front();
4990 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
4991 Reg = GR.getOrCreateConstNullPtr(MIRBuilder, ResType);
4992 } else if (TpOpcode == SPIRV::OpTypeVectorIdEXT) {
4993 // We ended up here coming from a splat on a <1 x T> type, which
4994 // IRTranslator translated into a scalar, so we have to restore the
4995 // vectorness.
4996 assert(GR.getScalarOrVectorComponentCount(ResType) == 1 &&
4997 "Expected <1 x T> Vector!");
4998 if (Opcode == TargetOpcode::G_FCONSTANT)
4999 Reg = GR.getOrCreateConstVector(I.getOperand(1).getFPImm()->getValue(), I,
5000 ResType, TII);
5001 else // We handle vector of pointer here as well.
5002 Reg = GR.getOrCreateConstVector(I.getOperand(1).getCImm()->getValue(), I,
5003 ResType, TII);
5004 } else if (Opcode == TargetOpcode::G_FCONSTANT) {
5005 Reg = GR.getOrCreateConstFP(I.getOperand(1).getFPImm()->getValue(), I,
5006 ResType, TII, !STI.isShader());
5007
5008 } else {
5009 Reg = GR.getOrCreateConstInt(I.getOperand(1).getCImm()->getValue(), I,
5010 ResType, TII, !STI.isShader());
5011 }
5012 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I);
5013}
5014
5015bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
5016 SPIRVTypeInst ResType,
5017 MachineInstr &I) const {
5018 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
5019 .addDef(ResVReg)
5020 .addUse(GR.getSPIRVTypeID(ResType))
5021 .constrainAllUses(TII, TRI, RBI);
5022 return true;
5023}
5024
5025bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
5026 SPIRVTypeInst ResType,
5027 MachineInstr &I) const {
5028 MachineBasicBlock &BB = *I.getParent();
5029 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
5030 .addDef(ResVReg)
5031 .addUse(GR.getSPIRVTypeID(ResType))
5032 // object to insert
5033 .addUse(I.getOperand(3).getReg())
5034 // composite to insert into
5035 .addUse(I.getOperand(2).getReg());
5036 for (unsigned i = 4; i < I.getNumOperands(); i++)
5037 MIB.addImm(foldImm(I.getOperand(i), MRI));
5038 MIB.constrainAllUses(TII, TRI, RBI);
5039 return true;
5040}
5041
5042bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
5043 SPIRVTypeInst ResType,
5044 MachineInstr &I) const {
5045 Type *MaybeResTy = nullptr;
5046 StringRef ResName;
5047 if (GR.findValueAttrs(&I, MaybeResTy, ResName) &&
5048 MaybeResTy != GR.getTypeForSPIRVType(ResType)) {
5049 assert((!MaybeResTy || MaybeResTy->isAggregateType()) &&
5050 "Expected aggregate type for extractv instruction");
5051 ResType = GR.getOrCreateSPIRVType(MaybeResTy, I,
5052 SPIRV::AccessQualifier::ReadWrite, false);
5053 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *I.getMF());
5054 }
5055 MachineBasicBlock &BB = *I.getParent();
5056 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
5057 .addDef(ResVReg)
5058 .addUse(GR.getSPIRVTypeID(ResType))
5059 .addUse(I.getOperand(2).getReg());
5060 for (unsigned i = 3; i < I.getNumOperands(); i++)
5061 MIB.addImm(foldImm(I.getOperand(i), MRI));
5062 MIB.constrainAllUses(TII, TRI, RBI);
5063 return true;
5064}
5065
5066bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
5067 SPIRVTypeInst ResType,
5068 MachineInstr &I) const {
5069 if (getImm(I.getOperand(4), MRI))
5070 return selectInsertVal(ResVReg, ResType, I);
5071 MachineBasicBlock &BB = *I.getParent();
5072 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
5073 .addDef(ResVReg)
5074 .addUse(GR.getSPIRVTypeID(ResType))
5075 .addUse(I.getOperand(2).getReg())
5076 .addUse(I.getOperand(3).getReg())
5077 .addUse(I.getOperand(4).getReg())
5078 .constrainAllUses(TII, TRI, RBI);
5079 return true;
5080}
5081
5082bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
5083 SPIRVTypeInst ResType,
5084 MachineInstr &I) const {
5085 if (getImm(I.getOperand(3), MRI))
5086 return selectExtractVal(ResVReg, ResType, I);
5087 MachineBasicBlock &BB = *I.getParent();
5088 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
5089 .addDef(ResVReg)
5090 .addUse(GR.getSPIRVTypeID(ResType))
5091 .addUse(I.getOperand(2).getReg())
5092 .addUse(I.getOperand(3).getReg())
5093 .constrainAllUses(TII, TRI, RBI);
5094 return true;
5095}
5096
5097bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
5098 SPIRVTypeInst ResType,
5099 MachineInstr &I) const {
5100 const bool IsGEPInBounds = I.getOperand(2).getImm();
5101 // Pointers to opaque types stay typed even with the extension on, so emit the
5102 // untyped variant only when the result is actually an untyped pointer.
5103 const bool UseUntypedPointers =
5104 ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
5105
5106 // Determine the opcode based on pointer type and bounds checking.
5107 // When using untyped pointers, use OpUntyped*AccessChainKHR variants.
5108 unsigned Opcode;
5109 if (UseUntypedPointers) {
5110 if (STI.isLogicalSPIRV()) {
5111 Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsAccessChainKHR
5112 : SPIRV::OpUntypedAccessChainKHR;
5113 } else {
5114 Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsPtrAccessChainKHR
5115 : SPIRV::OpUntypedPtrAccessChainKHR;
5116 }
5117 } else {
5118 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator
5119 // only relies on PtrAccessChain, so we'll try not to deviate. For Vulkan
5120 // however, we have to use Op[InBounds]AccessChain.
5121 // FIXME: fix llvm-spirv.
5122 if (STI.isLogicalSPIRV()) {
5123 Opcode =
5124 IsGEPInBounds ? SPIRV::OpInBoundsAccessChain : SPIRV::OpAccessChain;
5125 } else {
5126 Opcode = IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
5127 : SPIRV::OpPtrAccessChain;
5128 }
5129 }
5130
5131 Register BaseReg = I.getOperand(3).getReg();
5132 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
5133 .addDef(ResVReg)
5134 .addUse(GR.getSPIRVTypeID(ResType));
5135
5136 // For untyped access chains, we need to add the base type operand.
5137 if (UseUntypedPointers) {
5138 // Get the element type from the base pointer register.
5139 // For untyped pointers, this was stored when processing
5140 // spv_assign_ptr_type.
5141 SPIRVTypeInst BaseType = GR.getUntypedPtrElementType(BaseReg);
5142 if (!BaseType) {
5143 // Otherwise try the pointee type for mixed typed-pointer usage.
5144 SPIRVTypeInst BasePtrType = GR.getSPIRVTypeForVReg(BaseReg);
5145 BaseType = BasePtrType ? GR.getPointeeType(BasePtrType) : nullptr;
5146 }
5147 if (!BaseType) {
5148 // The base may be a not-yet-selected global. Read its value type from
5149 // the defining G_GLOBAL_VALUE, following copies.
5150 Register DefReg = BaseReg;
5151 MachineInstr *Def = MRI->getVRegDef(DefReg);
5152 while (Def && Def->getOpcode() == TargetOpcode::COPY &&
5153 Def->getOperand(1).isReg())
5154 Def = MRI->getVRegDef(Def->getOperand(1).getReg());
5155 if (Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE)
5156 if (const auto *GVar =
5157 dyn_cast<GlobalVariable>(Def->getOperand(1).getGlobal()))
5158 BaseType = GR.getOrCreateSPIRVType(GVar->getValueType(), I,
5159 SPIRV::AccessQualifier::ReadWrite,
5160 /*EmitIR=*/false);
5161 }
5162 if (!BaseType)
5163 return diagnoseUnsupported(
5164 I, "could not deduce the base type of an untyped access chain");
5165 Res.addUse(GR.getSPIRVTypeID(BaseType));
5166 }
5167
5168 // Object to get a pointer to.
5169 Res.addUse(BaseReg);
5170
5171 const bool IsAccessChainOpcode =
5172 (Opcode == SPIRV::OpAccessChain ||
5173 Opcode == SPIRV::OpInBoundsAccessChain ||
5174 Opcode == SPIRV::OpUntypedAccessChainKHR ||
5175 Opcode == SPIRV::OpUntypedInBoundsAccessChainKHR);
5176
5177 assert((!IsAccessChainOpcode || (getImm(I.getOperand(4), MRI) &&
5178 foldImm(I.getOperand(4), MRI) == 0)) &&
5179 "Cannot translate GEP to OpAccessChain.");
5180
5181 // Adding indices.
5182 const unsigned StartingIndex = IsAccessChainOpcode ? 5 : 4;
5183 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
5184 Res.addUse(I.getOperand(i).getReg());
5185 Res.constrainAllUses(TII, TRI, RBI);
5186
5187 // IRTranslator doesn't like <1 x T> vectors, and treats them as scalars. This
5188 // creates an information loss issue, and also broken code where a
5189 // OpCompositeExtract is applied to the scalar return of an OpPtrAccessChain,
5190 // when it was meant to be applied to the OpCompositeInsert created <1 x T>.
5191 // TODO: should we also re-create the index <1 x T>?
5192 if (MRI->hasOneUse(ResVReg)) {
5193 MachineInstr &Extract = *MRI->use_instr_begin(ResVReg);
5194 if (Extract.getOpcode() == SPIRV::OpCompositeExtract) {
5195 SPIRVTypeInst V = GR.getOrCreateSPIRVVectorType(ResType, 1, Extract, TII);
5196 // We cannot use GlobalRegistry::getOrCreateUndef directly here because it
5197 // tries to use UndefVal::get, which does not work for TypedPointerType,
5198 // which we can get here if we're dealing with <1 x T*>.
5199 Register Tmp = createVirtualRegister(V, &GR, MRI, *I.getMF());
5200 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
5201 .addDef(Tmp)
5202 .addUse(GR.getSPIRVTypeID(V))
5203 .constrainAllUses(TII, TRI, RBI);
5204 Register InsertReg = createVirtualRegister(V, &GR, MRI, *I.getMF());
5205 BuildMI(*I.getParent(), Extract, Extract.getDebugLoc(),
5206 TII.get(SPIRV::OpCompositeInsert))
5207 .addDef(InsertReg)
5208 .addUse(GR.getSPIRVTypeID(V))
5209 .addUse(ResVReg)
5210 .addUse(Tmp)
5211 .addImm(0)
5212 .constrainAllUses(TII, TRI, RBI);
5213 Extract.substituteRegister(ResVReg, InsertReg, 0, TRI);
5214 }
5215 }
5216 return true;
5217}
5218
5219// Maybe wrap a value into OpSpecConstantOp
5220bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
5221 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
5222 unsigned Lim = I.getNumExplicitOperands();
5223 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
5224 Register OpReg = I.getOperand(i).getReg();
5225 MachineInstr *OpDefine = MRI->getVRegDef(OpReg);
5226 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
5227 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) ||
5228 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
5229 OpDefine->getOpcode() == TargetOpcode::G_INTTOPTR ||
5230 GR.isAggregateType(OpType)) {
5231 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
5232 // by selectAddrSpaceCast(), and G_INTTOPTR is processed by selectUnOp()
5233 CompositeArgs.push_back(OpReg);
5234 continue;
5235 }
5236 MachineFunction *MF = I.getMF();
5237 Register WrapReg = GR.find(OpDefine, MF);
5238 if (WrapReg.isValid()) {
5239 CompositeArgs.push_back(WrapReg);
5240 continue;
5241 }
5242 SPIRVTypeInst WrapType = OpType;
5243 if (OpType->getOpcode() == SPIRV::OpTypePointer &&
5244 GR.getPointerStorageClass(OpType) ==
5245 SPIRV::StorageClass::CodeSectionINTEL) {
5246 WrapType = GR.changePointerStorageClass(OpType,
5247 SPIRV::StorageClass::Function, I);
5248 }
5249 WrapReg = MRI->createVirtualRegister(GR.getRegClass(WrapType));
5250 CompositeArgs.push_back(WrapReg);
5251 // Decorate the wrapper register and generate a new instruction
5252 MRI->setType(WrapReg, LLT::pointer(0, 64));
5253 GR.assignSPIRVTypeToVReg(WrapType, WrapReg, *MF);
5254 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
5255 TII.get(SPIRV::OpSpecConstantOp))
5256 .addDef(WrapReg)
5257 .addUse(GR.getSPIRVTypeID(WrapType))
5258 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
5259 .addUse(OpReg);
5260 GR.add(OpDefine, MIB);
5261 MIB.constrainAllUses(TII, TRI, RBI);
5262 }
5263 return true;
5264}
5265
5266bool SPIRVInstructionSelector::selectDerivativeInst(
5267 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
5268 const unsigned DPdOpCode) const {
5269 // TODO: This should check specifically for Fragment Execution Model, but STI
5270 // doesn't provide that information yet. See #167562
5271 if (!errorIfInstrOutsideShader(I))
5272 return false;
5273
5274 // If the arg/result types are half then we need to wrap the instr in
5275 // conversions to float
5276 // This case occurs because a half arg/result is legal in HLSL but not spirv.
5277 Register SrcReg = I.getOperand(2).getReg();
5278 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg);
5279 unsigned BitWidth = std::min(GR.getScalarOrVectorBitWidth(SrcType),
5280 GR.getScalarOrVectorBitWidth(ResType));
5281 if (BitWidth == 32)
5282 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode))
5283 .addDef(ResVReg)
5284 .addUse(GR.getSPIRVTypeID(ResType))
5285 .addUse(I.getOperand(2).getReg());
5286
5287 MachineIRBuilder MIRBuilder(I);
5288 unsigned componentCount = GR.getScalarOrVectorComponentCount(SrcType);
5289 SPIRVTypeInst F32ConvertTy = GR.getOrCreateSPIRVFloatType(32, I, TII);
5290 if (componentCount != 1)
5291 F32ConvertTy = GR.getOrCreateSPIRVVectorType(F32ConvertTy, componentCount,
5292 MIRBuilder, false);
5293
5294 const TargetRegisterClass *RegClass = GR.getRegClass(SrcType);
5295 Register ConvertToVReg = MRI->createVirtualRegister(RegClass);
5296 Register DpdOpVReg = MRI->createVirtualRegister(RegClass);
5297
5298 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert))
5299 .addDef(ConvertToVReg)
5300 .addUse(GR.getSPIRVTypeID(F32ConvertTy))
5301 .addUse(SrcReg)
5302 .constrainAllUses(TII, TRI, RBI);
5303 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode))
5304 .addDef(DpdOpVReg)
5305 .addUse(GR.getSPIRVTypeID(F32ConvertTy))
5306 .addUse(ConvertToVReg)
5307 .constrainAllUses(TII, TRI, RBI);
5308 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert))
5309 .addDef(ResVReg)
5310 .addUse(GR.getSPIRVTypeID(ResType))
5311 .addUse(DpdOpVReg)
5312 .constrainAllUses(TII, TRI, RBI);
5313 return true;
5314}
5315
5316bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
5317 SPIRVTypeInst ResType,
5318 MachineInstr &I) const {
5319 MachineBasicBlock &BB = *I.getParent();
5320 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID();
5321 switch (IID) {
5322 case Intrinsic::spv_load:
5323 return selectLoad(ResVReg, ResType, I);
5324 case Intrinsic::spv_atomic_load:
5325 return selectAtomicLoad(ResVReg, ResType, I);
5326 case Intrinsic::spv_store:
5327 return selectStore(I);
5328 case Intrinsic::spv_atomic_store:
5329 return selectAtomicStore(I);
5330 case Intrinsic::spv_extractv:
5331 return selectExtractVal(ResVReg, ResType, I);
5332 case Intrinsic::spv_insertv:
5333 return selectInsertVal(ResVReg, ResType, I);
5334 case Intrinsic::spv_extractelt:
5335 return selectExtractElt(ResVReg, ResType, I);
5336 case Intrinsic::spv_insertelt:
5337 return selectInsertElt(ResVReg, ResType, I);
5338 case Intrinsic::spv_gep:
5339 return selectGEP(ResVReg, ResType, I);
5340 case Intrinsic::spv_bitcast: {
5341 Register OpReg = I.getOperand(2).getReg();
5342 SPIRVTypeInst OpType =
5343 OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
5344 if (!GR.isBitcastCompatible(ResType, OpType))
5345 report_fatal_error("incompatible result and operand types in a bitcast");
5346 return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
5347 }
5348 case Intrinsic::spv_unref_global:
5349 case Intrinsic::spv_init_global: {
5350 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
5351 MachineInstr *Init = I.getNumExplicitOperands() > 2
5352 ? MRI->getVRegDef(I.getOperand(2).getReg())
5353 : nullptr;
5354 assert(MI);
5355 Register GVarVReg = MI->getOperand(0).getReg();
5356 if (!selectGlobalValue(GVarVReg, *MI, Init))
5357 return false;
5358 // We violate SSA form by inserting OpVariable and still having a gMIR
5359 // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing
5360 // the duplicated definition.
5361 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
5362 GR.invalidateMachineInstr(MI);
5363 MI->eraseFromParent();
5364 }
5365 return true;
5366 }
5367 case Intrinsic::spv_undef: {
5368 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
5369 .addDef(ResVReg)
5370 .addUse(GR.getSPIRVTypeID(ResType));
5371 MIB.constrainAllUses(TII, TRI, RBI);
5372 return true;
5373 }
5374 case Intrinsic::spv_poison:
5375 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpPoisonKHR))
5376 .addDef(ResVReg)
5377 .addUse(GR.getSPIRVTypeID(ResType))
5378 .constrainAllUses(TII, TRI, RBI);
5379 return true;
5380 case Intrinsic::spv_freeze:
5381 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpFreezeKHR))
5382 .addDef(ResVReg)
5383 .addUse(GR.getSPIRVTypeID(ResType))
5384 .addUse(I.getOperand(2).getReg())
5385 .constrainAllUses(TII, TRI, RBI);
5386 return true;
5387 case Intrinsic::spv_named_boolean_spec_constant: {
5388 auto Opcode = I.getOperand(3).getImm() ? SPIRV::OpSpecConstantTrue
5389 : SPIRV::OpSpecConstantFalse;
5390
5391 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
5392 .addDef(I.getOperand(0).getReg())
5393 .addUse(GR.getSPIRVTypeID(ResType));
5394 MIB.constrainAllUses(TII, TRI, RBI);
5395 unsigned SpecId = I.getOperand(2).getImm();
5396 buildOpDecorate(I.getOperand(0).getReg(), *++MIB->getIterator(), TII,
5397 SPIRV::Decoration::SpecId, {SpecId});
5398
5399 return true;
5400 }
5401 case Intrinsic::spv_const_composite: {
5402 // If no values are attached, the composite is null constant.
5403 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
5404 SmallVector<Register> CompositeArgs;
5405 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
5406
5407 // skip type MD node we already used when generated assign.type for this
5408 if (!IsNull) {
5409 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
5410 return false;
5411 std::function<bool(Register)> HasSpecConstOperand =
5412 [&](Register Reg) -> bool {
5413 MachineInstr *Def = MRI->getVRegDef(Reg);
5414 if (!Def)
5415 return false;
5416 if (!isConstReg(MRI, Def))
5417 return true;
5418 // Recurse into not-yet-selected spv_const_composite intrinsics
5419 // to detect transitive spec constant operands.
5420 if (isSpvIntrinsic(*Def, Intrinsic::spv_const_composite)) {
5421 for (unsigned J = Def->getNumExplicitDefs() + 1;
5422 J < Def->getNumExplicitOperands(); ++J) {
5423 if (Def->getOperand(J).isReg() &&
5424 HasSpecConstOperand(Def->getOperand(J).getReg()))
5425 return true;
5426 }
5427 }
5428 return false;
5429 };
5430 bool HasSpecConst = llvm::any_of(CompositeArgs, HasSpecConstOperand);
5431 unsigned CompositeOpc = HasSpecConst ? SPIRV::OpSpecConstantComposite
5432 : SPIRV::OpConstantComposite;
5433 unsigned ContinuedOpc = HasSpecConst
5434 ? SPIRV::OpSpecConstantCompositeContinuedINTEL
5435 : SPIRV::OpConstantCompositeContinuedINTEL;
5436 MachineIRBuilder MIR(I);
5437 SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions(
5438 MIR, CompositeOpc, 3, ContinuedOpc, CompositeArgs, ResVReg,
5439 GR.getSPIRVTypeID(ResType));
5440 for (auto *Instr : Instructions) {
5441 Instr->setDebugLoc(I.getDebugLoc());
5442 constrainSelectedInstRegOperands(*Instr, TII, TRI, RBI);
5443 }
5444 return true;
5445 } else {
5446 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
5447 .addDef(ResVReg)
5448 .addUse(GR.getSPIRVTypeID(ResType));
5449 MIB.constrainAllUses(TII, TRI, RBI);
5450 return true;
5451 }
5452 }
5453 case Intrinsic::spv_assign_name: {
5454 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
5455 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
5456 for (unsigned i = I.getNumExplicitDefs() + 2;
5457 i < I.getNumExplicitOperands(); ++i) {
5458 MIB.addImm(I.getOperand(i).getImm());
5459 }
5460 MIB.constrainAllUses(TII, TRI, RBI);
5461 return true;
5462 }
5463 case Intrinsic::spv_switch: {
5464 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
5465 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
5466 if (I.getOperand(i).isReg())
5467 MIB.addReg(I.getOperand(i).getReg());
5468 else if (I.getOperand(i).isCImm())
5469 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
5470 else if (I.getOperand(i).isMBB())
5471 MIB.addMBB(I.getOperand(i).getMBB());
5472 else
5473 llvm_unreachable("Unexpected OpSwitch operand");
5474 }
5475 MIB.constrainAllUses(TII, TRI, RBI);
5476 return true;
5477 }
5478 case Intrinsic::spv_loop_merge: {
5479 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge));
5480 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
5481 if (I.getOperand(i).isMBB())
5482 MIB.addMBB(I.getOperand(i).getMBB());
5483 else
5484 MIB.addImm(foldImm(I.getOperand(i), MRI));
5485 }
5486 MIB.constrainAllUses(TII, TRI, RBI);
5487 return true;
5488 }
5489 case Intrinsic::spv_loop_control_intel: {
5490 auto MIB =
5491 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopControlINTEL));
5492 for (unsigned J = 1; J < I.getNumExplicitOperands(); ++J)
5493 MIB.addImm(foldImm(I.getOperand(J), MRI));
5494 MIB.constrainAllUses(TII, TRI, RBI);
5495 return true;
5496 }
5497 case Intrinsic::spv_selection_merge: {
5498 auto MIB =
5499 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge));
5500 assert(I.getOperand(1).isMBB() &&
5501 "operand 1 to spv_selection_merge must be a basic block");
5502 MIB.addMBB(I.getOperand(1).getMBB());
5503 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm()));
5504 MIB.constrainAllUses(TII, TRI, RBI);
5505 return true;
5506 }
5507 case Intrinsic::spv_cmpxchg:
5508 return selectAtomicCmpXchg(ResVReg, ResType, I);
5509 case Intrinsic::spv_unreachable:
5510 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable))
5511 .constrainAllUses(TII, TRI, RBI);
5512 return true;
5513 case Intrinsic::spv_abort:
5514 return selectAbort(I);
5515 case Intrinsic::spv_alloca:
5516 return selectFrameIndex(ResVReg, ResType, I);
5517 case Intrinsic::spv_alloca_array:
5518 return selectAllocaArray(ResVReg, ResType, I);
5519 case Intrinsic::spv_assume:
5520 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
5521 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
5522 .addUse(I.getOperand(1).getReg())
5523 .constrainAllUses(TII, TRI, RBI);
5524 return true;
5525 }
5526 break;
5527 case Intrinsic::spv_expect:
5528 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
5529 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
5530 .addDef(ResVReg)
5531 .addUse(GR.getSPIRVTypeID(ResType))
5532 .addUse(I.getOperand(2).getReg())
5533 .addUse(I.getOperand(3).getReg())
5534 .constrainAllUses(TII, TRI, RBI);
5535 return true;
5536 }
5537 break;
5538 case Intrinsic::arithmetic_fence:
5539 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) {
5540 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT))
5541 .addDef(ResVReg)
5542 .addUse(GR.getSPIRVTypeID(ResType))
5543 .addUse(I.getOperand(2).getReg())
5544 .constrainAllUses(TII, TRI, RBI);
5545 return true;
5546 } else
5547 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I);
5548 break;
5549 case Intrinsic::spv_thread_id:
5550 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
5551 // intrinsic in LLVM IR for SPIR-V backend.
5552 //
5553 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
5554 // `GlobalInvocationId` builtin variable
5555 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
5556 ResType, I);
5557 case Intrinsic::spv_thread_id_in_group:
5558 // The HLSL SV_GroupThreadId semantic is lowered to
5559 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
5560 //
5561 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
5562 // translated to a `LocalInvocationId` builtin variable
5563 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg,
5564 ResType, I);
5565 case Intrinsic::spv_group_id:
5566 // The HLSL SV_GroupId semantic is lowered to
5567 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
5568 //
5569 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
5570 // builtin variable
5571 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
5572 I);
5573 case Intrinsic::spv_flattened_thread_id_in_group:
5574 // The HLSL SV_GroupIndex semantic is lowered to
5575 // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V
5576 // backend.
5577 //
5578 // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to
5579 // a `LocalInvocationIndex` builtin variable
5580 return loadBuiltinInputID(SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
5581 ResType, I);
5582 case Intrinsic::spv_workgroup_size:
5583 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupSize, ResVReg,
5584 ResType, I);
5585 case Intrinsic::spv_global_size:
5586 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalSize, ResVReg, ResType,
5587 I);
5588 case Intrinsic::spv_global_offset:
5589 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalOffset, ResVReg,
5590 ResType, I);
5591 case Intrinsic::spv_num_workgroups:
5592 return loadVec3BuiltinInputID(SPIRV::BuiltIn::NumWorkgroups, ResVReg,
5593 ResType, I);
5594 case Intrinsic::spv_subgroup_size:
5595 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType,
5596 I);
5597 case Intrinsic::spv_num_subgroups:
5598 return loadBuiltinInputID(SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType,
5599 I);
5600 case Intrinsic::spv_subgroup_id:
5601 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupId, ResVReg, ResType, I);
5602 case Intrinsic::spv_subgroup_local_invocation_id:
5603 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupLocalInvocationId,
5604 ResVReg, ResType, I);
5605 case Intrinsic::spv_subgroup_max_size:
5606 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType,
5607 I);
5608 case Intrinsic::spv_fdot:
5609 return selectFloatDot(ResVReg, ResType, I);
5610 case Intrinsic::spv_udot:
5611 case Intrinsic::spv_sdot:
5612 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5613 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
5614 return selectIntegerDot(ResVReg, ResType, I,
5615 /*Signed=*/IID == Intrinsic::spv_sdot);
5616 return selectIntegerDotExpansion(ResVReg, ResType, I);
5617 case Intrinsic::spv_dot4add_i8packed:
5618 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5619 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
5620 return selectDot4AddPacked<true>(ResVReg, ResType, I);
5621 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
5622 case Intrinsic::spv_dot4add_u8packed:
5623 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5624 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
5625 return selectDot4AddPacked<false>(ResVReg, ResType, I);
5626 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
5627 case Intrinsic::spv_all:
5628 return selectAll(ResVReg, ResType, I);
5629 case Intrinsic::spv_any:
5630 return selectAny(ResVReg, ResType, I);
5631 case Intrinsic::spv_distance:
5632 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance);
5633 case Intrinsic::spv_lerp:
5634 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix);
5635 case Intrinsic::spv_length:
5636 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length);
5637 case Intrinsic::spv_degrees:
5638 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees);
5639 case Intrinsic::spv_faceforward:
5640 return selectExtInst(ResVReg, ResType, I, GL::FaceForward);
5641 case Intrinsic::spv_frac:
5642 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract);
5643 case Intrinsic::spv_isinf:
5644 return selectOpIsInf(ResVReg, ResType, I);
5645 case Intrinsic::spv_isnan:
5646 return selectOpIsNan(ResVReg, ResType, I);
5647 case Intrinsic::spv_isfinite:
5648 return selectOpIsFinite(ResVReg, ResType, I);
5649 case Intrinsic::spv_isnormal:
5650 return selectOpIsNormal(ResVReg, ResType, I);
5651 case Intrinsic::spv_normalize:
5652 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize);
5653 case Intrinsic::spv_refract:
5654 return selectExtInst(ResVReg, ResType, I, GL::Refract);
5655 case Intrinsic::spv_reflect:
5656 return selectExtInst(ResVReg, ResType, I, GL::Reflect);
5657 case Intrinsic::spv_rsqrt:
5658 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt);
5659 case Intrinsic::spv_sign:
5660 return selectSign(ResVReg, ResType, I);
5661 case Intrinsic::spv_smoothstep:
5662 return selectExtInst(ResVReg, ResType, I, CL::smoothstep, GL::SmoothStep);
5663 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
5664 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
5665 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
5666 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
5667 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
5668 return selectFirstBitLow(ResVReg, ResType, I);
5669 case Intrinsic::spv_all_memory_barrier:
5670 return selectBarrierInst(I, SPIRV::Scope::Device,
5671 SPIRV::MemorySemantics::UniformMemory |
5672 SPIRV::MemorySemantics::ImageMemory |
5673 SPIRV::MemorySemantics::WorkgroupMemory,
5674 /*WithGroupSync*/ false);
5675 case Intrinsic::spv_all_memory_barrier_with_group_sync:
5676 return selectBarrierInst(I, SPIRV::Scope::Device,
5677 SPIRV::MemorySemantics::UniformMemory |
5678 SPIRV::MemorySemantics::ImageMemory |
5679 SPIRV::MemorySemantics::WorkgroupMemory,
5680 /*WithGroupSync*/ true);
5681 case Intrinsic::spv_device_memory_barrier:
5682 return selectBarrierInst(I, SPIRV::Scope::Device,
5683 SPIRV::MemorySemantics::UniformMemory |
5684 SPIRV::MemorySemantics::ImageMemory,
5685 /*WithGroupSync*/ false);
5686 case Intrinsic::spv_device_memory_barrier_with_group_sync:
5687 return selectBarrierInst(I, SPIRV::Scope::Device,
5688 SPIRV::MemorySemantics::UniformMemory |
5689 SPIRV::MemorySemantics::ImageMemory,
5690 /*WithGroupSync*/ true);
5691 case Intrinsic::spv_group_memory_barrier:
5692 return selectBarrierInst(I, SPIRV::Scope::Workgroup,
5693 SPIRV::MemorySemantics::WorkgroupMemory,
5694 /*WithGroupSync*/ false);
5695 case Intrinsic::spv_group_memory_barrier_with_group_sync:
5696 return selectBarrierInst(I, SPIRV::Scope::Workgroup,
5697 SPIRV::MemorySemantics::WorkgroupMemory,
5698 /*WithGroupSync*/ true);
5699 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
5700 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 1).getReg();
5701 SPIRV::StorageClass::StorageClass ResSC =
5702 GR.getPointerStorageClass(ResType);
5703 if (!isGenericCastablePtr(ResSC))
5704 return diagnoseUnsupported(I, "The target storage class is not castable "
5705 "from the Generic storage class");
5706 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGenericCastToPtrExplicit))
5707 .addDef(ResVReg)
5708 .addUse(GR.getSPIRVTypeID(ResType))
5709 .addUse(PtrReg)
5710 .addImm(ResSC)
5711 .constrainAllUses(TII, TRI, RBI);
5712 return true;
5713 }
5714 case Intrinsic::spv_lifetime_start:
5715 case Intrinsic::spv_lifetime_end: {
5716 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
5717 : SPIRV::OpLifetimeStop;
5718 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm();
5719 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg();
5720 if (Size == -1)
5721 Size = 0;
5722 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op))
5723 .addUse(PtrReg)
5724 .addImm(Size)
5725 .constrainAllUses(TII, TRI, RBI);
5726 return true;
5727 }
5728 case Intrinsic::spv_saturate:
5729 return selectSaturate(ResVReg, ResType, I);
5730 case Intrinsic::spv_nclamp:
5731 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp);
5732 case Intrinsic::spv_uclamp:
5733 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp);
5734 case Intrinsic::spv_sclamp:
5735 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp);
5736 case Intrinsic::spv_subgroup_prefix_bit_count:
5737 return selectWavePrefixBitCount(ResVReg, ResType, I);
5738 case Intrinsic::spv_wave_active_countbits:
5739 return selectWaveActiveCountBits(ResVReg, ResType, I);
5740 case Intrinsic::spv_wave_all_equal:
5741 return selectWaveActiveAllEqual(ResVReg, ResType, I);
5742 case Intrinsic::spv_wave_all:
5743 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll);
5744 case Intrinsic::spv_wave_any:
5745 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
5746 case Intrinsic::spv_subgroup_ballot:
5747 return selectWaveOpInst(ResVReg, ResType, I,
5748 SPIRV::OpGroupNonUniformBallot);
5749 case Intrinsic::spv_wave_is_first_lane:
5750 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect);
5751 case Intrinsic::spv_wave_reduce_or:
5752 return selectWaveReduceOp(ResVReg, ResType, I,
5753 SPIRV::OpGroupNonUniformBitwiseOr);
5754 case Intrinsic::spv_wave_reduce_xor:
5755 return selectWaveReduceOp(ResVReg, ResType, I,
5756 SPIRV::OpGroupNonUniformBitwiseXor);
5757 case Intrinsic::spv_wave_reduce_and:
5758 return selectWaveReduceOp(ResVReg, ResType, I,
5759 SPIRV::OpGroupNonUniformBitwiseAnd);
5760 case Intrinsic::spv_wave_reduce_umax:
5761 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
5762 case Intrinsic::spv_wave_reduce_max:
5763 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
5764 case Intrinsic::spv_wave_reduce_umin:
5765 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ true);
5766 case Intrinsic::spv_wave_reduce_min:
5767 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ false);
5768 case Intrinsic::spv_wave_reduce_sum:
5769 return selectWaveReduceSum(ResVReg, ResType, I);
5770 case Intrinsic::spv_wave_product:
5771 return selectWaveReduceProduct(ResVReg, ResType, I);
5772 case Intrinsic::spv_wave_readlane:
5773 return selectWaveOpInst(ResVReg, ResType, I,
5774 SPIRV::OpGroupNonUniformShuffle);
5775 case Intrinsic::spv_wave_prefix_sum:
5776 return selectWaveExclusiveScanSum(ResVReg, ResType, I);
5777 case Intrinsic::spv_wave_prefix_product:
5778 return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
5779 case Intrinsic::spv_quad_read_across_x: {
5780 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 0);
5781 }
5782 case Intrinsic::spv_quad_read_across_y: {
5783 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 1);
5784 }
5785 case Intrinsic::spv_quad_read_across_diagonal: {
5786 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 2);
5787 }
5788 case Intrinsic::spv_radians:
5789 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians);
5790 // Discard intrinsics which we do not expect to actually represent code after
5791 // lowering or intrinsics which are not implemented but should not crash when
5792 // found in a customer's LLVM IR input.
5793 case Intrinsic::instrprof_increment:
5794 case Intrinsic::instrprof_increment_step:
5795 case Intrinsic::instrprof_value_profile:
5796 break;
5797 // Discard internal intrinsics.
5798 case Intrinsic::spv_value_md:
5799 break;
5800 case Intrinsic::spv_resource_handlefrombinding: {
5801 return selectHandleFromBinding(ResVReg, ResType, I);
5802 }
5803 case Intrinsic::spv_resource_counterhandlefrombinding:
5804 return selectCounterHandleFromBinding(ResVReg, ResType, I);
5805 case Intrinsic::spv_resource_updatecounter:
5806 return selectUpdateCounter(ResVReg, ResType, I);
5807 case Intrinsic::spv_resource_store_typedbuffer: {
5808 return selectImageWriteIntrinsic(I);
5809 }
5810 case Intrinsic::spv_resource_load_typedbuffer: {
5811 return selectReadImageIntrinsic(ResVReg, ResType, I);
5812 }
5813 case Intrinsic::spv_resource_load_level: {
5814 return selectLoadLevelIntrinsic(ResVReg, ResType, I);
5815 }
5816 case Intrinsic::spv_resource_getdimensions_x:
5817 case Intrinsic::spv_resource_getdimensions_xy:
5818 case Intrinsic::spv_resource_getdimensions_xyz: {
5819 return selectGetDimensionsIntrinsic(ResVReg, ResType, I);
5820 }
5821 case Intrinsic::spv_resource_getdimensions_levels_x:
5822 case Intrinsic::spv_resource_getdimensions_levels_xy:
5823 case Intrinsic::spv_resource_getdimensions_levels_xyz: {
5824 return selectGetDimensionsLevelsIntrinsic(ResVReg, ResType, I);
5825 }
5826 case Intrinsic::spv_resource_getdimensions_ms_xy:
5827 case Intrinsic::spv_resource_getdimensions_ms_xyz: {
5828 return selectGetDimensionsMSIntrinsic(ResVReg, ResType, I);
5829 }
5830 case Intrinsic::spv_resource_calculate_lod:
5831 case Intrinsic::spv_resource_calculate_lod_unclamped:
5832 return selectCalculateLodIntrinsic(ResVReg, ResType, I);
5833 case Intrinsic::spv_resource_sample:
5834 case Intrinsic::spv_resource_sample_clamp:
5835 return selectSampleBasicIntrinsic(ResVReg, ResType, I);
5836 case Intrinsic::spv_resource_samplebias:
5837 case Intrinsic::spv_resource_samplebias_clamp:
5838 return selectSampleBiasIntrinsic(ResVReg, ResType, I);
5839 case Intrinsic::spv_resource_samplegrad:
5840 case Intrinsic::spv_resource_samplegrad_clamp:
5841 return selectSampleGradIntrinsic(ResVReg, ResType, I);
5842 case Intrinsic::spv_resource_samplelevel:
5843 return selectSampleLevelIntrinsic(ResVReg, ResType, I);
5844 case Intrinsic::spv_resource_samplecmp:
5845 case Intrinsic::spv_resource_samplecmp_clamp:
5846 return selectSampleCmpIntrinsic(ResVReg, ResType, I);
5847 case Intrinsic::spv_resource_samplecmplevelzero:
5848 return selectSampleCmpLevelZeroIntrinsic(ResVReg, ResType, I);
5849 case Intrinsic::spv_resource_gather:
5850 case Intrinsic::spv_resource_gather_cmp:
5851 return selectGatherIntrinsic(ResVReg, ResType, I);
5852 case Intrinsic::spv_resource_getbasepointer:
5853 case Intrinsic::spv_resource_getpointer: {
5854 return selectResourceGetPointer(ResVReg, ResType, I);
5855 }
5856 case Intrinsic::spv_pushconstant_getpointer: {
5857 return selectPushConstantGetPointer(ResVReg, ResType, I);
5858 }
5859 case Intrinsic::spv_discard: {
5860 return selectDiscard(ResVReg, ResType, I);
5861 }
5862 case Intrinsic::spv_resource_nonuniformindex: {
5863 return selectResourceNonUniformIndex(ResVReg, ResType, I);
5864 }
5865 case Intrinsic::spv_unpackhalf2x16: {
5866 return selectExtInst(ResVReg, ResType, I, GL::UnpackHalf2x16);
5867 }
5868 case Intrinsic::spv_packhalf2x16: {
5869 return selectExtInst(ResVReg, ResType, I, GL::PackHalf2x16);
5870 }
5871 case Intrinsic::spv_ddx:
5872 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdx);
5873 case Intrinsic::spv_ddy:
5874 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdy);
5875 case Intrinsic::spv_ddx_coarse:
5876 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxCoarse);
5877 case Intrinsic::spv_ddy_coarse:
5878 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyCoarse);
5879 case Intrinsic::spv_ddx_fine:
5880 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxFine);
5881 case Intrinsic::spv_ddy_fine:
5882 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyFine);
5883 case Intrinsic::spv_fwidth:
5884 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpFwidth);
5885 case Intrinsic::spv_masked_gather:
5886 if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
5887 return selectMaskedGather(ResVReg, ResType, I);
5888 return diagnoseUnsupported(
5889 I, "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
5890 case Intrinsic::spv_masked_scatter:
5891 if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
5892 return selectMaskedScatter(I);
5893 return diagnoseUnsupported(
5894 I, "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
5895 case Intrinsic::returnaddress:
5896 case Intrinsic::frameaddress: {
5897 // SPIR-V does not have a stack or return address. Lower to null.
5898 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
5899 .addDef(ResVReg)
5900 .addUse(GR.getSPIRVTypeID(ResType));
5901 MIB.constrainAllUses(TII, TRI, RBI);
5902 return true;
5903 }
5904 default:
5905 return diagnoseUnsupported(I, "intrinsic selection not implemented.");
5906 }
5907 return true;
5908}
5909
5910bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
5911 SPIRVTypeInst ResType,
5912 MachineInstr &I) const {
5913 // The images need to be loaded in the same basic block as their use. We defer
5914 // loading the image to the intrinsic that uses it.
5915 if (ResType->getOpcode() == SPIRV::OpTypeImage)
5916 return true;
5917
5918 return loadHandleBeforePosition(ResVReg, GR.getSPIRVTypeForVReg(ResVReg),
5919 *cast<GIntrinsic>(&I), I);
5920}
5921
5922bool SPIRVInstructionSelector::selectCounterHandleFromBinding(
5923 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
5924 auto &Intr = cast<GIntrinsic>(I);
5925 assert(Intr.getIntrinsicID() ==
5926 Intrinsic::spv_resource_counterhandlefrombinding);
5927
5928 // Extract information from the intrinsic call.
5929 Register MainHandleReg = Intr.getOperand(2).getReg();
5930 auto *MainHandleDef = cast<GIntrinsic>(getVRegDef(*MRI, MainHandleReg));
5931 assert(MainHandleDef->getIntrinsicID() ==
5932 Intrinsic::spv_resource_handlefrombinding);
5933
5934 uint32_t Set = getIConstVal(Intr.getOperand(4).getReg(), MRI);
5935 uint32_t Binding = getIConstVal(Intr.getOperand(3).getReg(), MRI);
5936 uint32_t ArraySize = getIConstVal(MainHandleDef->getOperand(4).getReg(), MRI);
5937 Register IndexReg = MainHandleDef->getOperand(5).getReg();
5938 std::string CounterName =
5939 getStringValueFromReg(MainHandleDef->getOperand(6).getReg(), *MRI) +
5940 ".counter";
5941
5942 // Create the counter variable.
5943 MachineIRBuilder MIRBuilder(I);
5944 Register CounterVarReg =
5945 buildPointerToResource(SPIRVTypeInst(GR.getPointeeType(ResType)),
5946 GR.getPointerStorageClass(ResType), Set, Binding,
5947 ArraySize, IndexReg, CounterName, MIRBuilder);
5948
5949 return BuildCOPY(ResVReg, CounterVarReg, I);
5950}
5951
5952bool SPIRVInstructionSelector::selectUpdateCounter(Register &ResVReg,
5953 SPIRVTypeInst ResType,
5954 MachineInstr &I) const {
5955 auto &Intr = cast<GIntrinsic>(I);
5956 assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter);
5957
5958 Register CounterHandleReg = Intr.getOperand(2).getReg();
5959 Register IncrReg = Intr.getOperand(3).getReg();
5960
5961 // The counter handle is a pointer to the counter variable (which is a struct
5962 // containing an i32). We need to get a pointer to that i32 member to do the
5963 // atomic operation.
5964#ifndef NDEBUG
5965 SPIRVTypeInst CounterVarType = GR.getSPIRVTypeForVReg(CounterHandleReg);
5966 SPIRVTypeInst CounterVarPointeeType = GR.getPointeeType(CounterVarType);
5967 assert(CounterVarPointeeType &&
5968 CounterVarPointeeType->getOpcode() == SPIRV::OpTypeStruct &&
5969 "Counter variable must be a struct");
5970 assert(GR.getPointerStorageClass(CounterVarType) ==
5971 SPIRV::StorageClass::StorageBuffer &&
5972 "Counter variable must be in the storage buffer storage class");
5973 assert(CounterVarPointeeType->getNumOperands() == 2 &&
5974 "Counter variable must have exactly 1 member in the struct");
5975 const SPIRVTypeInst MemberType =
5976 GR.getSPIRVTypeForVReg(CounterVarPointeeType->getOperand(1).getReg());
5977 assert(MemberType->getOpcode() == SPIRV::OpTypeInt &&
5978 "Counter variable struct must have a single i32 member");
5979#endif
5980
5981 // The struct has a single i32 member.
5982 MachineIRBuilder MIRBuilder(I);
5983 const Type *LLVMIntType =
5984 Type::getInt32Ty(I.getMF()->getFunction().getContext());
5985
5986 SPIRVTypeInst IntPtrType = GR.getOrCreateSPIRVPointerType(
5987 LLVMIntType, MIRBuilder, SPIRV::StorageClass::StorageBuffer);
5988
5989 Register Zero = buildI32Constant(0, I);
5990
5991 Register PtrToCounter =
5992 MRI->createVirtualRegister(GR.getRegClass(IntPtrType));
5993 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain))
5994 .addDef(PtrToCounter)
5995 .addUse(GR.getSPIRVTypeID(IntPtrType))
5996 .addUse(CounterHandleReg)
5997 .addUse(Zero)
5998 .constrainAllUses(TII, TRI, RBI);
5999
6000 // For UAV/SSBO counters, the scope is Device. The counter variable is not
6001 // used as a flag. So the memory semantics can be None.
6002 Register Scope = buildI32Constant(SPIRV::Scope::Device, I);
6003 Register Semantics = buildI32Constant(SPIRV::MemorySemantics::None, I);
6004
6005 int64_t IncrVal = getIConstValSext(IncrReg, MRI);
6006 Register Incr = buildI32Constant(static_cast<uint32_t>(IncrVal), I);
6007
6008 Register AtomicRes = MRI->createVirtualRegister(GR.getRegClass(ResType));
6009 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAtomicIAdd))
6010 .addDef(AtomicRes)
6011 .addUse(GR.getSPIRVTypeID(ResType))
6012 .addUse(PtrToCounter)
6013 .addUse(Scope)
6014 .addUse(Semantics)
6015 .addUse(Incr)
6016 .constrainAllUses(TII, TRI, RBI);
6017 if (IncrVal >= 0) {
6018 return BuildCOPY(ResVReg, AtomicRes, I);
6019 }
6020
6021 // In HLSL, IncrementCounter returns the value *before* the increment, while
6022 // DecrementCounter returns the value *after* the decrement. Both are lowered
6023 // to the same atomic intrinsic which returns the value *before* the
6024 // operation. So for decrements (negative IncrVal), we must subtract the
6025 // increment value from the result to get the post-decrement value.
6026 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
6027 .addDef(ResVReg)
6028 .addUse(GR.getSPIRVTypeID(ResType))
6029 .addUse(AtomicRes)
6030 .addUse(Incr)
6031 .constrainAllUses(TII, TRI, RBI);
6032 return true;
6033}
6034bool SPIRVInstructionSelector::selectReadImageIntrinsic(Register &ResVReg,
6035 SPIRVTypeInst ResType,
6036 MachineInstr &I) const {
6037
6038 // If the load of the image is in a different basic block, then
6039 // this will generate invalid code. A proper solution is to move
6040 // the OpLoad from selectHandleFromBinding here. However, to do
6041 // that we will need to change the return type of the intrinsic.
6042 // We will do that when we can, but for now trying to move forward with other
6043 // issues.
6044 Register ImageReg = I.getOperand(2).getReg();
6045 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6046 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6047 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6048 *ImageDef, I)) {
6049 return false;
6050 }
6051
6052 Register IdxReg = I.getOperand(3).getReg();
6053 DebugLoc Loc = I.getDebugLoc();
6054 MachineInstr &Pos = I;
6055
6056 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, IdxReg, Loc,
6057 Pos);
6058}
6059
6060bool SPIRVInstructionSelector::generateSampleImage(
6061 Register ResVReg, SPIRVTypeInst ResType, Register ImageReg,
6062 Register SamplerReg, Register CoordinateReg, const ImageOperands &ImOps,
6063 DebugLoc Loc, MachineInstr &Pos) const {
6064 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6065 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6066 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6067 *ImageDef, Pos)) {
6068 return false;
6069 }
6070
6071 auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
6072 Register NewSamplerReg =
6073 MRI->createVirtualRegister(MRI->getRegClass(SamplerReg));
6074 if (!loadHandleBeforePosition(NewSamplerReg,
6075 GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef,
6076 Pos)) {
6077 return false;
6078 }
6079
6080 MachineIRBuilder MIRBuilder(Pos);
6081 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
6082 GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder);
6083 Register SampledImageReg =
6084 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
6085
6086 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpSampledImage))
6087 .addDef(SampledImageReg)
6088 .addUse(GR.getSPIRVTypeID(SampledImageType))
6089 .addUse(NewImageReg)
6090 .addUse(NewSamplerReg)
6091 .constrainAllUses(TII, TRI, RBI);
6092
6093 bool IsExplicitLod = ImOps.GradX.has_value() || ImOps.GradY.has_value() ||
6094 ImOps.Lod.has_value();
6095 unsigned Opcode = IsExplicitLod ? SPIRV::OpImageSampleExplicitLod
6096 : SPIRV::OpImageSampleImplicitLod;
6097 if (ImOps.Compare)
6098 Opcode = IsExplicitLod ? SPIRV::OpImageSampleDrefExplicitLod
6099 : SPIRV::OpImageSampleDrefImplicitLod;
6100
6101 auto MIB = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(Opcode))
6102 .addDef(ResVReg)
6103 .addUse(GR.getSPIRVTypeID(ResType))
6104 .addUse(SampledImageReg)
6105 .addUse(CoordinateReg);
6106
6107 if (ImOps.Compare)
6108 MIB.addUse(*ImOps.Compare);
6109
6110 uint32_t ImageOperands = 0;
6111 if (ImOps.Bias)
6112 ImageOperands |= SPIRV::ImageOperand::Bias;
6113 if (ImOps.Lod)
6114 ImageOperands |= SPIRV::ImageOperand::Lod;
6115 if (ImOps.GradX && ImOps.GradY)
6116 ImageOperands |= SPIRV::ImageOperand::Grad;
6117 if (ImOps.Offset && !isScalarOrVectorIntConstantZero(*ImOps.Offset)) {
6118 if (isConstReg(MRI, *ImOps.Offset))
6119 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
6120 else {
6121 Pos.emitGenericError(
6122 "Non-constant offsets are not supported in sample instructions.");
6123 return false;
6124 }
6125 }
6126 if (ImOps.MinLod)
6127 ImageOperands |= SPIRV::ImageOperand::MinLod;
6128
6129 if (ImageOperands != 0) {
6130 MIB.addImm(ImageOperands);
6131 if (ImageOperands & SPIRV::ImageOperand::Bias)
6132 MIB.addUse(*ImOps.Bias);
6133 if (ImageOperands & SPIRV::ImageOperand::Lod)
6134 MIB.addUse(*ImOps.Lod);
6135 if (ImageOperands & SPIRV::ImageOperand::Grad) {
6136 MIB.addUse(*ImOps.GradX);
6137 MIB.addUse(*ImOps.GradY);
6138 }
6139 if (ImageOperands &
6140 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
6141 MIB.addUse(*ImOps.Offset);
6142 if (ImageOperands & SPIRV::ImageOperand::MinLod)
6143 MIB.addUse(*ImOps.MinLod);
6144 }
6145
6146 MIB.constrainAllUses(TII, TRI, RBI);
6147 return true;
6148}
6149
6150bool SPIRVInstructionSelector::selectImageQuerySize(
6151 Register ImageReg, Register &ResVReg, MachineInstr &I,
6152 std::optional<Register> LodReg) const {
6153 unsigned Opcode =
6154 LodReg ? SPIRV::OpImageQuerySizeLod : SPIRV::OpImageQuerySize;
6155 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
6156 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
6157 "ImageReg is not an image type.");
6158
6159 auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm());
6160 bool IsArray = ImageType->getOperand(4).getImm() != 0;
6161 unsigned NumComponents = 0;
6162 switch (Dim) {
6163 case SPIRV::Dim::DIM_1D:
6164 case SPIRV::Dim::DIM_Buffer:
6165 NumComponents = IsArray ? 2 : 1;
6166 break;
6167 case SPIRV::Dim::DIM_2D:
6168 case SPIRV::Dim::DIM_Cube:
6169 case SPIRV::Dim::DIM_Rect:
6170 NumComponents = IsArray ? 3 : 2;
6171 break;
6172 case SPIRV::Dim::DIM_3D:
6173 NumComponents = 3;
6174 break;
6175 default:
6176 I.emitGenericError("Unsupported image dimension for OpImageQuerySize.");
6177 return false;
6178 }
6179
6180 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
6181 SPIRVTypeInst ResType =
6182 NumComponents == 1
6183 ? I32Ty
6184 : GR.getOrCreateSPIRVVectorType(I32Ty, NumComponents, I, TII);
6185
6186 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
6187 .addDef(ResVReg)
6188 .addUse(GR.getSPIRVTypeID(ResType))
6189 .addUse(ImageReg);
6190 if (LodReg)
6191 MIB.addUse(*LodReg);
6192 MIB.constrainAllUses(TII, TRI, RBI);
6193 return true;
6194}
6195
6196bool SPIRVInstructionSelector::selectGetDimensionsIntrinsic(
6197 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6198 Register ImageReg = I.getOperand(2).getReg();
6199 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6200 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6201 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6202 *ImageDef, I)) {
6203 return false;
6204 }
6205 return selectImageQuerySize(NewImageReg, ResVReg, I);
6206}
6207
6208bool SPIRVInstructionSelector::selectGetDimensionsLevelsIntrinsic(
6209 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6210 Register ImageReg = I.getOperand(2).getReg();
6211 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6212 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6213 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6214 *ImageDef, I)) {
6215 return false;
6216 }
6217
6218 Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6219 Register LodReg = I.getOperand(3).getReg();
6220
6221 assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(6).getImm() == 1 &&
6222 "OpImageQuerySizeLod and OpImageQueryLevels require a sampled image");
6223
6224 if (!selectImageQuerySize(NewImageReg, SizeReg, I, LodReg)) {
6225 return false;
6226 }
6227
6228 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
6229 Register LevelsReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6230 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6231 TII.get(SPIRV::OpImageQueryLevels))
6232 .addDef(LevelsReg)
6233 .addUse(GR.getSPIRVTypeID(I32Ty))
6234 .addUse(NewImageReg)
6235 .constrainAllUses(TII, TRI, RBI);
6236
6237 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6238 TII.get(SPIRV::OpCompositeConstruct))
6239 .addDef(ResVReg)
6240 .addUse(GR.getSPIRVTypeID(ResType))
6241 .addUse(SizeReg)
6242 .addUse(LevelsReg)
6243 .constrainAllUses(TII, TRI, RBI);
6244
6245 return true;
6246}
6247
6248bool SPIRVInstructionSelector::selectGetDimensionsMSIntrinsic(
6249 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6250 Register ImageReg = I.getOperand(2).getReg();
6251 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6252 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6253 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6254 *ImageDef, I)) {
6255 return false;
6256 }
6257
6258 Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6259
6260 assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(5).getImm() == 1 &&
6261 "OpImageQuerySamples requires a multisampled image");
6262
6263 if (!selectImageQuerySize(NewImageReg, SizeReg, I)) {
6264 return false;
6265 }
6266
6267 Register SamplesReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6268
6269 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
6270 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6271 TII.get(SPIRV::OpImageQuerySamples))
6272 .addDef(SamplesReg)
6273 .addUse(GR.getSPIRVTypeID(I32Ty))
6274 .addUse(NewImageReg)
6275 .constrainAllUses(TII, TRI, RBI);
6276
6277 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6278 TII.get(SPIRV::OpCompositeConstruct))
6279 .addDef(ResVReg)
6280 .addUse(GR.getSPIRVTypeID(ResType))
6281 .addUse(SizeReg)
6282 .addUse(SamplesReg)
6283 .constrainAllUses(TII, TRI, RBI);
6284
6285 return true;
6286}
6287
6288bool SPIRVInstructionSelector::selectCalculateLodIntrinsic(
6289 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6290 Register ImageReg = I.getOperand(2).getReg();
6291 Register SamplerReg = I.getOperand(3).getReg();
6292 Register CoordinateReg = I.getOperand(4).getReg();
6293
6294 auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6295 if (!ImageDef)
6296 return false;
6297 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6298 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6299 *ImageDef, I)) {
6300 return false;
6301 }
6302
6303 auto *SamplerDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
6304 if (!SamplerDef)
6305 return false;
6306 Register NewSamplerReg =
6307 MRI->createVirtualRegister(MRI->getRegClass(SamplerReg));
6308 if (!loadHandleBeforePosition(
6309 NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) {
6310 return false;
6311 }
6312
6313 MachineIRBuilder MIRBuilder(I);
6314 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
6315 GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder);
6316 Register SampledImageReg =
6317 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
6318
6319 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage))
6320 .addDef(SampledImageReg)
6321 .addUse(GR.getSPIRVTypeID(SampledImageType))
6322 .addUse(NewImageReg)
6323 .addUse(NewSamplerReg)
6324 .constrainAllUses(TII, TRI, RBI);
6325
6326 SPIRVTypeInst Vec2Ty = GR.getOrCreateSPIRVVectorType(ResType, 2, I, TII);
6327 Register QueryResultReg = MRI->createVirtualRegister(GR.getRegClass(Vec2Ty));
6328
6329 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageQueryLod))
6330 .addDef(QueryResultReg)
6331 .addUse(GR.getSPIRVTypeID(Vec2Ty))
6332 .addUse(SampledImageReg)
6333 .addUse(CoordinateReg)
6334 .constrainAllUses(TII, TRI, RBI);
6335
6336 unsigned ExtractedIndex =
6337 cast<GIntrinsic>(I).getIntrinsicID() ==
6338 Intrinsic::spv_resource_calculate_lod_unclamped
6339 ? 1
6340 : 0;
6341
6342 MachineInstrBuilder MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
6343 TII.get(SPIRV::OpCompositeExtract))
6344 .addDef(ResVReg)
6345 .addUse(GR.getSPIRVTypeID(ResType))
6346 .addUse(QueryResultReg)
6347 .addImm(ExtractedIndex);
6348
6349 MIB.constrainAllUses(TII, TRI, RBI);
6350 return true;
6351}
6352
6353bool SPIRVInstructionSelector::selectSampleBasicIntrinsic(
6354 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6355 Register ImageReg = I.getOperand(2).getReg();
6356 Register SamplerReg = I.getOperand(3).getReg();
6357 Register CoordinateReg = I.getOperand(4).getReg();
6358 ImageOperands ImOps;
6359 if (I.getNumOperands() > 5)
6360 ImOps.Offset = I.getOperand(5).getReg();
6361 if (I.getNumOperands() > 6)
6362 ImOps.MinLod = I.getOperand(6).getReg();
6363 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6364 CoordinateReg, ImOps, I.getDebugLoc(), I);
6365}
6366
6367bool SPIRVInstructionSelector::selectSampleBiasIntrinsic(
6368 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6369 Register ImageReg = I.getOperand(2).getReg();
6370 Register SamplerReg = I.getOperand(3).getReg();
6371 Register CoordinateReg = I.getOperand(4).getReg();
6372 ImageOperands ImOps;
6373 ImOps.Bias = I.getOperand(5).getReg();
6374 if (I.getNumOperands() > 6)
6375 ImOps.Offset = I.getOperand(6).getReg();
6376 if (I.getNumOperands() > 7)
6377 ImOps.MinLod = I.getOperand(7).getReg();
6378 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6379 CoordinateReg, ImOps, I.getDebugLoc(), I);
6380}
6381
6382bool SPIRVInstructionSelector::selectSampleGradIntrinsic(
6383 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6384 Register ImageReg = I.getOperand(2).getReg();
6385 Register SamplerReg = I.getOperand(3).getReg();
6386 Register CoordinateReg = I.getOperand(4).getReg();
6387 ImageOperands ImOps;
6388 ImOps.GradX = I.getOperand(5).getReg();
6389 ImOps.GradY = I.getOperand(6).getReg();
6390 if (I.getNumOperands() > 7)
6391 ImOps.Offset = I.getOperand(7).getReg();
6392 if (I.getNumOperands() > 8)
6393 ImOps.MinLod = I.getOperand(8).getReg();
6394 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6395 CoordinateReg, ImOps, I.getDebugLoc(), I);
6396}
6397
6398bool SPIRVInstructionSelector::selectSampleLevelIntrinsic(
6399 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6400 Register ImageReg = I.getOperand(2).getReg();
6401 Register SamplerReg = I.getOperand(3).getReg();
6402 Register CoordinateReg = I.getOperand(4).getReg();
6403 ImageOperands ImOps;
6404 ImOps.Lod = I.getOperand(5).getReg();
6405 if (I.getNumOperands() > 6)
6406 ImOps.Offset = I.getOperand(6).getReg();
6407 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6408 CoordinateReg, ImOps, I.getDebugLoc(), I);
6409}
6410
6411bool SPIRVInstructionSelector::selectSampleCmpIntrinsic(Register &ResVReg,
6412 SPIRVTypeInst ResType,
6413 MachineInstr &I) const {
6414 Register ImageReg = I.getOperand(2).getReg();
6415 Register SamplerReg = I.getOperand(3).getReg();
6416 Register CoordinateReg = I.getOperand(4).getReg();
6417 ImageOperands ImOps;
6418 ImOps.Compare = I.getOperand(5).getReg();
6419 if (I.getNumOperands() > 6)
6420 ImOps.Offset = I.getOperand(6).getReg();
6421 if (I.getNumOperands() > 7)
6422 ImOps.MinLod = I.getOperand(7).getReg();
6423 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6424 CoordinateReg, ImOps, I.getDebugLoc(), I);
6425}
6426
6427bool SPIRVInstructionSelector::selectLoadLevelIntrinsic(Register &ResVReg,
6428 SPIRVTypeInst ResType,
6429 MachineInstr &I) const {
6430 Register ImageReg = I.getOperand(2).getReg();
6431 Register CoordinateReg = I.getOperand(3).getReg();
6432 Register LodReg = I.getOperand(4).getReg();
6433
6434 ImageOperands ImOps;
6435 ImOps.Lod = LodReg;
6436 if (I.getNumOperands() > 5)
6437 ImOps.Offset = I.getOperand(5).getReg();
6438
6439 auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6440 if (!ImageDef)
6441 return false;
6442
6443 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6444 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6445 *ImageDef, I)) {
6446 return false;
6447 }
6448
6449 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, CoordinateReg,
6450 I.getDebugLoc(), I, &ImOps);
6451}
6452
6453bool SPIRVInstructionSelector::selectSampleCmpLevelZeroIntrinsic(
6454 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6455 Register ImageReg = I.getOperand(2).getReg();
6456 Register SamplerReg = I.getOperand(3).getReg();
6457 Register CoordinateReg = I.getOperand(4).getReg();
6458 ImageOperands ImOps;
6459 ImOps.Compare = I.getOperand(5).getReg();
6460 if (I.getNumOperands() > 6)
6461 ImOps.Offset = I.getOperand(6).getReg();
6462 SPIRVTypeInst FloatTy = GR.getOrCreateSPIRVFloatType(32, I, TII);
6463 ImOps.Lod = GR.getOrCreateConstFP(APFloat(0.0f), I, FloatTy, TII);
6464 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6465 CoordinateReg, ImOps, I.getDebugLoc(), I);
6466}
6467
6468bool SPIRVInstructionSelector::selectGatherIntrinsic(Register &ResVReg,
6469 SPIRVTypeInst ResType,
6470 MachineInstr &I) const {
6471 Register ImageReg = I.getOperand(2).getReg();
6472 Register SamplerReg = I.getOperand(3).getReg();
6473 Register CoordinateReg = I.getOperand(4).getReg();
6474 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
6475 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
6476 "ImageReg is not an image type.");
6477
6478 Register ComponentOrCompareReg;
6479 Register OffsetReg;
6480
6481 ComponentOrCompareReg = I.getOperand(5).getReg();
6482 OffsetReg = I.getOperand(6).getReg();
6483 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6484 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6485 if (!loadHandleBeforePosition(NewImageReg, ImageType, *ImageDef, I)) {
6486 return false;
6487 }
6488
6489 auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm());
6490 if (Dim != SPIRV::Dim::DIM_2D && Dim != SPIRV::Dim::DIM_Cube &&
6491 Dim != SPIRV::Dim::DIM_Rect) {
6492 I.emitGenericError(
6493 "Gather operations are only supported for 2D, Cube, and Rect images.");
6494 return false;
6495 }
6496
6497 auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
6498 Register NewSamplerReg =
6499 MRI->createVirtualRegister(MRI->getRegClass(SamplerReg));
6500 if (!loadHandleBeforePosition(
6501 NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) {
6502 return false;
6503 }
6504
6505 MachineIRBuilder MIRBuilder(I);
6506 SPIRVTypeInst SampledImageType =
6507 GR.getOrCreateOpTypeSampledImage(ImageType, MIRBuilder);
6508 Register SampledImageReg =
6509 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
6510
6511 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage))
6512 .addDef(SampledImageReg)
6513 .addUse(GR.getSPIRVTypeID(SampledImageType))
6514 .addUse(NewImageReg)
6515 .addUse(NewSamplerReg)
6516 .constrainAllUses(TII, TRI, RBI);
6517
6518 auto IntrId = cast<GIntrinsic>(I).getIntrinsicID();
6519 bool IsGatherCmp = IntrId == Intrinsic::spv_resource_gather_cmp;
6520 unsigned Opcode =
6521 IsGatherCmp ? SPIRV::OpImageDrefGather : SPIRV::OpImageGather;
6522
6523 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
6524 .addDef(ResVReg)
6525 .addUse(GR.getSPIRVTypeID(ResType))
6526 .addUse(SampledImageReg)
6527 .addUse(CoordinateReg)
6528 .addUse(ComponentOrCompareReg);
6529
6530 uint32_t ImageOperands = 0;
6531 if (OffsetReg && !isScalarOrVectorIntConstantZero(OffsetReg)) {
6532 if (Dim == SPIRV::Dim::DIM_Cube) {
6533 I.emitGenericError(
6534 "Gather operations with offset are not supported for Cube images.");
6535 return false;
6536 }
6537 if (isConstReg(MRI, OffsetReg))
6538 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
6539 else {
6540 ImageOperands |= SPIRV::ImageOperand::Offset;
6541 }
6542 }
6543
6544 if (ImageOperands != 0) {
6545 MIB.addImm(ImageOperands);
6546 if (ImageOperands &
6547 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
6548 MIB.addUse(OffsetReg);
6549 }
6550
6551 MIB.constrainAllUses(TII, TRI, RBI);
6552 return true;
6553}
6554
6555bool SPIRVInstructionSelector::generateImageReadOrFetch(
6556 Register &ResVReg, SPIRVTypeInst ResType, Register ImageReg,
6557 Register IdxReg, DebugLoc Loc, MachineInstr &Pos,
6558 const ImageOperands *ImOps) const {
6559 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
6560 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
6561 "ImageReg is not an image type.");
6562
6563 bool IsSignedInteger =
6564 sampledTypeIsSignedInteger(GR.getTypeForSPIRVType(ImageType));
6565 // Check if the "sampled" operand of the image type is 1.
6566 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpImageFetch
6567 auto SampledOp = ImageType->getOperand(6);
6568 bool IsFetch = (SampledOp.getImm() == 1);
6569
6570 auto AddOperands = [&](MachineInstrBuilder &MIB) {
6571 uint32_t ImageOperandsMask = 0;
6572 if (IsSignedInteger)
6573 ImageOperandsMask |= 0x1000; // SignExtend
6574
6575 if (IsFetch && ImOps) {
6576 if (ImOps->Lod)
6577 ImageOperandsMask |= SPIRV::ImageOperand::Lod;
6578 if (ImOps->Offset && !isScalarOrVectorIntConstantZero(*ImOps->Offset)) {
6579 if (isConstReg(MRI, *ImOps->Offset))
6580 ImageOperandsMask |= SPIRV::ImageOperand::ConstOffset;
6581 else
6582 ImageOperandsMask |= SPIRV::ImageOperand::Offset;
6583 }
6584 }
6585
6586 if (ImageOperandsMask != 0) {
6587 MIB.addImm(ImageOperandsMask);
6588 if (IsFetch && ImOps) {
6589 if (ImOps->Lod)
6590 MIB.addUse(*ImOps->Lod);
6591 if (ImOps->Offset &&
6592 (ImageOperandsMask &
6593 (SPIRV::ImageOperand::Offset | SPIRV::ImageOperand::ConstOffset)))
6594 MIB.addUse(*ImOps->Offset);
6595 }
6596 }
6597 };
6598
6599 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
6600
6601 // A wide element (e.g. int64_t2) is emulated with a narrower packed image, so
6602 // its sampled type is different from the result.
6603 SPIRVTypeInst SampledType =
6604 GR.getSPIRVTypeForVReg(ImageType->getOperand(1).getReg());
6605 bool IsPacked = SampledType != GR.getScalarOrVectorComponentType(ResType);
6606 SPIRVTypeInst ReadType =
6607 widenTypeToVec4(IsPacked ? SampledType : ResType, Pos);
6608 bool ReadTypeMatchesResult = ReadType == ResType;
6609 // Read directly into the result, or into a temporary to bitcast/extract.
6610 Register ReadReg = ReadTypeMatchesResult
6611 ? ResVReg
6612 : MRI->createVirtualRegister(GR.getRegClass(ReadType));
6613
6614 auto BMI =
6615 BuildMI(*Pos.getParent(), Pos, Loc,
6616 TII.get(IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
6617 .addDef(ReadReg)
6618 .addUse(GR.getSPIRVTypeID(ReadType))
6619 .addUse(ImageReg)
6620 .addUse(IdxReg);
6621 AddOperands(BMI);
6622 BMI.constrainAllUses(TII, TRI, RBI);
6623
6624 if (ReadTypeMatchesResult)
6625 return true;
6626
6627 if (IsPacked) {
6628 // Reinterpret the packed vector as the wide result type.
6629 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpBitcast))
6630 .addDef(ResVReg)
6631 .addUse(GR.getSPIRVTypeID(ResType))
6632 .addUse(ReadReg)
6633 .constrainAllUses(TII, TRI, RBI);
6634 return true;
6635 }
6636
6637 if (ResultSize == 1) {
6638 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpCompositeExtract))
6639 .addDef(ResVReg)
6640 .addUse(GR.getSPIRVTypeID(ResType))
6641 .addUse(ReadReg)
6642 .addImm(0)
6643 .constrainAllUses(TII, TRI, RBI);
6644 return true;
6645 }
6646 return extractSubvector(ResVReg, ResType, ReadReg, Pos);
6647}
6648
6649bool SPIRVInstructionSelector::selectResourceGetPointer(Register &ResVReg,
6650 SPIRVTypeInst ResType,
6651 MachineInstr &I) const {
6652 Register ResourcePtr = I.getOperand(2).getReg();
6653 SPIRVTypeInst RegType = GR.getSPIRVTypeForVReg(ResourcePtr, I.getMF());
6654 if (RegType->getOpcode() == SPIRV::OpTypeImage) {
6655 // For texel buffers, the index into the image is part of the OpImageRead or
6656 // OpImageWrite instructions. So we will do nothing in this case. This
6657 // intrinsic will be combined with the load or store when selecting the load
6658 // or store.
6659 return true;
6660 }
6661
6662 assert(ResType->getOpcode() == SPIRV::OpTypePointer);
6663 MachineIRBuilder MIRBuilder(I);
6664
6665 Register ZeroReg =
6666 buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
6667 auto MIB =
6668 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain))
6669 .addDef(ResVReg)
6670 .addUse(GR.getSPIRVTypeID(ResType))
6671 .addUse(ResourcePtr)
6672 .addUse(ZeroReg);
6673
6674 if (I.getNumExplicitOperands() > 3) {
6675 Register IndexReg = I.getOperand(3).getReg();
6676 MIB.addUse(IndexReg);
6677 }
6678 MIB.constrainAllUses(TII, TRI, RBI);
6679 return true;
6680}
6681
6682bool SPIRVInstructionSelector::selectPushConstantGetPointer(
6683 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6684 MRI->replaceRegWith(ResVReg, I.getOperand(2).getReg());
6685 return true;
6686}
6687
6688bool SPIRVInstructionSelector::selectResourceNonUniformIndex(
6689 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6690 Register ObjReg = I.getOperand(2).getReg();
6691 if (!BuildCOPY(ResVReg, ObjReg, I))
6692 return false;
6693
6694 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {});
6695 // Check for the registers that use the index marked as non-uniform
6696 // and recursively mark them as non-uniform.
6697 // Per the spec, it's necessary that the final argument used for
6698 // load/store/sample/atomic must be decorated, so we need to propagate the
6699 // decoration through access chains and copies.
6700 // https://docs.vulkan.org/samples/latest/samples/extensions/descriptor_indexing/README.html#_when_to_use_non_uniform_indexing_qualifier
6701 decorateUsesAsNonUniform(ResVReg);
6702 return true;
6703}
6704
6705void SPIRVInstructionSelector::decorateUsesAsNonUniform(
6706 Register &NonUniformReg) const {
6707 llvm::SmallVector<std::pair<Register, MachineInstr *>> WorkList = {
6708 {NonUniformReg, nullptr}};
6709 llvm::SmallSet<Register, 8> Visited;
6710 while (WorkList.size() > 0) {
6711 auto [CurrentReg, DefMI] = WorkList.pop_back_val();
6712
6713 if (!Visited.insert(CurrentReg).second)
6714 continue;
6715
6716 bool IsDecorated = false;
6717 for (MachineInstr &Use : MRI->use_instructions(CurrentReg)) {
6718 if (Use.getOpcode() == SPIRV::OpDecorate &&
6719 Use.getOperand(1).getImm() == SPIRV::Decoration::NonUniformEXT) {
6720 IsDecorated = true;
6721 continue;
6722 }
6723 // Check if the instruction has the result register and add it to the
6724 // worklist.
6725 if (Use.getOperand(0).isReg() && Use.getOperand(0).isDef()) {
6726 Register ResultReg = Use.getOperand(0).getReg();
6727 if (ResultReg == CurrentReg)
6728 continue;
6729 WorkList.push_back({ResultReg, &Use});
6730 }
6731 }
6732
6733 if (!IsDecorated) {
6734 MachineBasicBlock &MBB = *DefMI->getParent();
6735 MachineInstr &InsertPt =
6736 DefMI->isPHI() ? *MBB.getFirstNonPHI() : *DefMI->getNextNode();
6737 buildOpDecorate(CurrentReg, InsertPt, TII,
6738 SPIRV::Decoration::NonUniformEXT, {});
6739 }
6740 }
6741}
6742
6743bool SPIRVInstructionSelector::extractSubvector(
6744 Register &ResVReg, SPIRVTypeInst ResType, Register &ReadReg,
6745 MachineInstr &InsertionPoint) const {
6746 SPIRVTypeInst InputType = GR.getResultType(ReadReg);
6747 [[maybe_unused]] uint64_t InputSize =
6748 GR.getScalarOrVectorComponentCount(InputType);
6749 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
6750 [[maybe_unused]] bool IsLongVectorEXT =
6751 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector);
6752 assert((InputSize > 1 || IsLongVectorEXT) && "The input must be a vector.");
6753 assert((ResultSize > 1 || IsLongVectorEXT) && "The result must be a vector.");
6754 assert(ResultSize < InputSize &&
6755 "Cannot extract more element than there are in the input.");
6756 SmallVector<Register> ComponentRegisters;
6757 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(ResType);
6758 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType);
6759 for (uint64_t I = 0; I < ResultSize; I++) {
6760 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass);
6761 BuildMI(*InsertionPoint.getParent(), InsertionPoint,
6762 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
6763 .addDef(ComponentReg)
6764 .addUse(ScalarType->getOperand(0).getReg())
6765 .addUse(ReadReg)
6766 .addImm(I)
6767 .constrainAllUses(TII, TRI, RBI);
6768 ComponentRegisters.emplace_back(ComponentReg);
6769 }
6770
6771 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint,
6772 InsertionPoint.getDebugLoc(),
6773 TII.get(SPIRV::OpCompositeConstruct))
6774 .addDef(ResVReg)
6775 .addUse(GR.getSPIRVTypeID(ResType));
6776
6777 for (Register ComponentReg : ComponentRegisters)
6778 MIB.addUse(ComponentReg);
6779 MIB.constrainAllUses(TII, TRI, RBI);
6780 return true;
6781}
6782
6783bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
6784 MachineInstr &I) const {
6785 // If the load of the image is in a different basic block, then
6786 // this will generate invalid code. A proper solution is to move
6787 // the OpLoad from selectHandleFromBinding here. However, to do
6788 // that we will need to change the return type of the intrinsic.
6789 // We will do that when we can, but for now trying to move forward with other
6790 // issues.
6791 Register ImageReg = I.getOperand(1).getReg();
6792 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6793 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6794 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6795 *ImageDef, I)) {
6796 return false;
6797 }
6798
6799 Register CoordinateReg = I.getOperand(2).getReg();
6800 Register DataReg = I.getOperand(3).getReg();
6801 assert(isVectorType(GR.getResultType(DataReg)));
6802 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4);
6803 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite))
6804 .addUse(NewImageReg)
6805 .addUse(CoordinateReg)
6806 .addUse(DataReg)
6807 .constrainAllUses(TII, TRI, RBI);
6808 return true;
6809}
6810
6811Register SPIRVInstructionSelector::buildPointerToResource(
6812 SPIRVTypeInst SpirvResType, SPIRV::StorageClass::StorageClass SC,
6813 uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
6814 StringRef Name, MachineIRBuilder MIRBuilder) const {
6815 const Type *ResType = GR.getTypeForSPIRVType(SpirvResType);
6816 if (ArraySize == 1) {
6817 SPIRVTypeInst PtrType =
6818 GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
6819 assert(GR.getPointeeType(PtrType) == SpirvResType &&
6820 "SpirvResType did not have an explicit layout.");
6821 return GR.getOrCreateGlobalVariableWithBinding(PtrType, Set, Binding, Name,
6822 MIRBuilder);
6823 }
6824
6825 const Type *VarType = ArrayType::get(const_cast<Type *>(ResType), ArraySize);
6826 SPIRVTypeInst VarPointerType =
6827 GR.getOrCreateSPIRVPointerType(VarType, MIRBuilder, SC);
6828 Register VarReg = GR.getOrCreateGlobalVariableWithBinding(
6829 VarPointerType, Set, Binding, Name, MIRBuilder);
6830
6831 SPIRVTypeInst ResPointerType =
6832 GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
6833 Register AcReg = MRI->createVirtualRegister(GR.getRegClass(ResPointerType));
6834
6835 MIRBuilder.buildInstr(SPIRV::OpAccessChain)
6836 .addDef(AcReg)
6837 .addUse(GR.getSPIRVTypeID(ResPointerType))
6838 .addUse(VarReg)
6839 .addUse(IndexReg);
6840
6841 return AcReg;
6842}
6843
6844bool SPIRVInstructionSelector::selectFirstBitSet16(
6845 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
6846 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
6847 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
6848 if (!selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()},
6849 ExtendOpcode))
6850 return false;
6851
6852 return selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode);
6853}
6854
6855bool SPIRVInstructionSelector::selectFirstBitSet32(
6856 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
6857 unsigned BitSetOpcode) const {
6858 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
6859 .addDef(ResVReg)
6860 .addUse(GR.getSPIRVTypeID(ResType))
6861 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
6862 .addImm(BitSetOpcode)
6863 .addUse(SrcReg)
6864 .constrainAllUses(TII, TRI, RBI);
6865 return true;
6866}
6867
6868bool SPIRVInstructionSelector::selectFirstBitSet64(
6869 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
6870 unsigned BitSetOpcode, bool SwapPrimarySide) const {
6871 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
6872 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType);
6873 bool ZeroAsNull = !STI.isShader();
6874 Register ConstIntZero =
6875 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull);
6876 Register ConstIntOne =
6877 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull);
6878
6879 // SPIRV doesn't support vectors with more than 4 components. Since the
6880 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
6881 // operate on vectors with 2 or less components. When largers vectors are
6882 // seen. Split them, recurse, then recombine them.
6883 // TODO: handle the case where SPV_EXT_long_vector is enabled.
6884 if (ComponentCount > 2) {
6885 auto Func = [this, SwapPrimarySide](Register ResVReg, SPIRVTypeInst ResType,
6886 MachineInstr &I, Register SrcReg,
6887 unsigned Opcode) -> bool {
6888 return this->selectFirstBitSet64(ResVReg, ResType, I, SrcReg, Opcode,
6889 SwapPrimarySide);
6890 };
6891
6892 return handle64BitOverflow(ResVReg, ResType, I, SrcReg, BitSetOpcode, Func);
6893 }
6894
6895 // 1. Split int64 into 2 pieces using a bitcast
6896 MachineIRBuilder MIRBuilder(I);
6897 SPIRVTypeInst PostCastType = GR.getOrCreateSPIRVVectorType(
6898 BaseType, 2 * ComponentCount, MIRBuilder, false);
6899 Register BitcastReg =
6900 MRI->createVirtualRegister(GR.getRegClass(PostCastType));
6901
6902 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg},
6903 SPIRV::OpBitcast))
6904 return false;
6905
6906 // 2. Find the first set bit from the primary side for all the pieces in #1
6907 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType));
6908 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode))
6909 return false;
6910
6911 // 3. Split result vector into high bits and low bits
6912 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
6913 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
6914
6915 bool IsScalarRes = !isVectorType(ResType);
6916 if (IsScalarRes) {
6917 // if scalar do a vector extract
6918 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntOne},
6919 SPIRV::OpVectorExtractDynamic))
6920 return false;
6921 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntZero},
6922 SPIRV::OpVectorExtractDynamic))
6923 return false;
6924 } else {
6925 // if vector do a shufflevector
6926 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
6927 TII.get(SPIRV::OpVectorShuffle))
6928 .addDef(HighReg)
6929 .addUse(GR.getSPIRVTypeID(ResType))
6930 .addUse(FBSReg)
6931 // Per the spec, repeat the vector if only one vec is needed
6932 .addUse(FBSReg);
6933
6934 // high bits are stored in even natural indexes. Extract them from FBSReg
6935 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
6936 MIB.addImm(J);
6937 }
6938
6939 MIB.constrainAllUses(TII, TRI, RBI);
6940
6941 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
6942 TII.get(SPIRV::OpVectorShuffle))
6943 .addDef(LowReg)
6944 .addUse(GR.getSPIRVTypeID(ResType))
6945 .addUse(FBSReg)
6946 // Per the spec, repeat the vector if only one vec is needed
6947 .addUse(FBSReg);
6948
6949 // low bits are stored in odd natural indices. Extract them from FBSReg
6950 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
6951 MIB.addImm(J);
6952 }
6953 MIB.constrainAllUses(TII, TRI, RBI);
6954 }
6955
6956 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
6957 // primary
6958 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
6959 Register NegOneReg;
6960 Register Reg0;
6961 Register Reg32;
6962 unsigned SelectOp;
6963 unsigned AddOp;
6964
6965 if (IsScalarRes) {
6966 NegOneReg =
6967 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull);
6968 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
6969 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull);
6970 SelectOp = SPIRV::OpSelectSISCond;
6971 AddOp = SPIRV::OpIAddS;
6972 } else {
6973 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount,
6974 MIRBuilder, false);
6975 NegOneReg =
6976 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull);
6977 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull);
6978 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull);
6979 SelectOp = SPIRV::OpSelectVIVCond;
6980 AddOp = SPIRV::OpIAddV;
6981 }
6982
6983 Register PrimaryReg = HighReg;
6984 Register SecondaryReg = LowReg;
6985 Register RegPrimaryOffset = Reg32;
6986 Register RegSecondaryOffset = Reg0;
6987
6988 // By default the emitted opcodes check for the set bit from the MSB side.
6989 // Setting SwapPrimarySide checks the set bit from the LSB side
6990 if (SwapPrimarySide) {
6991 PrimaryReg = LowReg;
6992 SecondaryReg = HighReg;
6993 RegPrimaryOffset = Reg0;
6994 RegSecondaryOffset = Reg32;
6995 }
6996
6997 Register RegSecondaryHasVal =
6998 MRI->createVirtualRegister(GR.getRegClass(BoolType));
6999 if (!selectOpWithSrcs(RegSecondaryHasVal, BoolType, I,
7000 {SecondaryReg, NegOneReg}, SPIRV::OpINotEqual))
7001 return false;
7002
7003 Register RegPrimaryHasVal =
7004 MRI->createVirtualRegister(GR.getRegClass(BoolType));
7005 if (!selectOpWithSrcs(RegPrimaryHasVal, BoolType, I, {PrimaryReg, NegOneReg},
7006 SPIRV::OpINotEqual))
7007 return false;
7008
7009 // Pass 1: seed with secondary (lower-priority fallback)
7010 // ReturnBits = secondaryHasVal ? SecondaryBits : -1
7011 // Add = secondaryHasVal ? SecondaryOffset : 0
7012 Register RegReturnBits = MRI->createVirtualRegister(GR.getRegClass(ResType));
7013 if (!selectOpWithSrcs(RegReturnBits, ResType, I,
7014 {RegSecondaryHasVal, SecondaryReg, NegOneReg},
7015 SelectOp))
7016 return false;
7017
7018 Register RegAdd;
7019 if (SwapPrimarySide) {
7020 RegAdd = MRI->createVirtualRegister(GR.getRegClass(ResType));
7021 if (!selectOpWithSrcs(RegAdd, ResType, I,
7022 {RegSecondaryHasVal, RegSecondaryOffset, Reg0},
7023 SelectOp))
7024 return false;
7025 } else {
7026 RegAdd = Reg0;
7027 }
7028
7029 // Pass 2: override with primary (higher priority) if it has a valid result
7030 // ReturnBits2 = primaryHasVal ? PrimaryBits : ReturnBits
7031 // Add2 = primaryHasVal ? PrimaryOffset : Add
7032 Register RegReturnBits2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
7033 if (!selectOpWithSrcs(RegReturnBits2, ResType, I,
7034 {RegPrimaryHasVal, PrimaryReg, RegReturnBits},
7035 SelectOp))
7036 return false;
7037
7038 Register RegAdd2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
7039 if (!selectOpWithSrcs(RegAdd2, ResType, I,
7040 {RegPrimaryHasVal, RegPrimaryOffset, RegAdd}, SelectOp))
7041 return false;
7042
7043 return selectOpWithSrcs(ResVReg, ResType, I, {RegReturnBits2, RegAdd2},
7044 AddOp);
7045}
7046
7047bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
7048 SPIRVTypeInst ResType,
7049 MachineInstr &I,
7050 bool IsSigned) const {
7051 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
7052 Register OpReg = I.getOperand(2).getReg();
7053 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
7054 // zero or sign extend
7055 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
7056 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
7057
7058 switch (GR.getScalarOrVectorBitWidth(OpType)) {
7059 case 16:
7060 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
7061 case 32:
7062 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
7063 case 64:
7064 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
7065 /*SwapPrimarySide=*/false);
7066 default:
7067 return diagnoseUnsupported(
7068 I,
7069 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
7070 }
7071}
7072
7073bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
7074 SPIRVTypeInst ResType,
7075 MachineInstr &I) const {
7076 // FindILsb intrinsic only supports 32 bit integers
7077 Register OpReg = I.getOperand(2).getReg();
7078 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
7079 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
7080 // to an unsigned i32. As this leaves all the least significant bits unchanged
7081 // so the first set bit from the LSB side doesn't change.
7082 unsigned ExtendOpcode = SPIRV::OpUConvert;
7083 unsigned BitSetOpcode = GL::FindILsb;
7084
7085 switch (GR.getScalarOrVectorBitWidth(OpType)) {
7086 case 16:
7087 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
7088 case 32:
7089 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
7090 case 64:
7091 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
7092 /*SwapPrimarySide=*/true);
7093 default:
7094 return diagnoseUnsupported(I,
7095 "spv_firstbitlow only supports 16,32,64 bits.");
7096 }
7097}
7098
7099bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
7100 SPIRVTypeInst ResType,
7101 MachineInstr &I) const {
7102 // there was an allocation size parameter to the allocation instruction
7103 // that is not 1
7104 MachineBasicBlock &BB = *I.getParent();
7105 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVariableLengthArrayINTEL))
7106 .addDef(ResVReg)
7107 .addUse(GR.getSPIRVTypeID(ResType))
7108 .addUse(I.getOperand(2).getReg())
7109 .constrainAllUses(TII, TRI, RBI);
7110 if (!STI.isShader()) {
7111 unsigned Alignment = I.getOperand(3).getImm();
7112 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment});
7113 }
7114 return true;
7115}
7116
7117// Returns true iff `Ty` is a concrete SPIR-V type per the SPV_KHR_abort
7118// definition: a numerical scalar (int/float), a (physical) pointer, a vector,
7119// matrix or any aggregate (array/struct) recursively containing only such
7120// types. OpTypeBool, OpTypeVoid, opaque handles and similar abstract
7121// non-concrete types are rejected.
7122static bool isConcreteSPIRVType(SPIRVTypeInst Ty,
7123 const SPIRVGlobalRegistry &GR) {
7124 SmallVector<SPIRVTypeInst, 4> Worklist{Ty};
7125 while (!Worklist.empty()) {
7126 SPIRVTypeInst T = Worklist.pop_back_val();
7127 switch (T->getOpcode()) {
7128 case SPIRV::OpTypeInt:
7129 case SPIRV::OpTypeFloat:
7130 case SPIRV::OpTypePointer:
7131 break;
7132 case SPIRV::OpTypeVector:
7133 case SPIRV::OpTypeVectorIdEXT:
7134 case SPIRV::OpTypeMatrix:
7135 case SPIRV::OpTypeArray: {
7136 Register OperandReg = T->getOperand(1).getReg();
7137 SPIRVTypeInst ElementT = GR.getSPIRVTypeForVReg(OperandReg);
7138 Worklist.push_back(ElementT);
7139 } break;
7140 case SPIRV::OpTypeStruct:
7141 for (unsigned Idx = 1, E = T->getNumOperands(); Idx < E; ++Idx) {
7142 Register OperandReg = T->getOperand(Idx).getReg();
7143 SPIRVTypeInst ElementT = GR.getSPIRVTypeForVReg(OperandReg);
7144 Worklist.push_back(ElementT);
7145 }
7146 break;
7147 default:
7148 return false;
7149 }
7150 }
7151 return true;
7152}
7153
7154bool SPIRVInstructionSelector::selectAbort(MachineInstr &I) const {
7155 assert(I.getNumExplicitOperands() == 2);
7156
7157 Register MsgReg = I.getOperand(1).getReg();
7158 SPIRVTypeInst MsgType = GR.getSPIRVTypeForVReg(MsgReg);
7159 assert(MsgType && "Message argument of llvm.spv.abort has no SPIR-V type");
7160
7161 if (!isConcreteSPIRVType(MsgType, GR))
7162 return diagnoseUnsupported(
7163 I,
7164 "llvm.spv.abort message type must be a concrete SPIR-V type (numerical "
7165 "scalar, pointer, vector, matrix, or aggregate of such types)");
7166
7167 MachineBasicBlock &BB = *I.getParent();
7168 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAbortKHR))
7169 .addUse(GR.getSPIRVTypeID(MsgType))
7170 .addUse(MsgReg)
7171 .constrainAllUses(TII, TRI, RBI);
7172 return true;
7173}
7174
7175bool SPIRVInstructionSelector::selectTrap(MachineInstr &I) const {
7176 // When the SPV_KHR_abort extension is disabled, drop the G_TRAP and
7177 // G_UBSANTRAP silently.
7178 if (!STI.canUseExtension(SPIRV::Extension::SPV_KHR_abort))
7179 return true;
7180
7181 // Use the 32-bit integer constant for the abort "message" argument:
7182 // - G_UBSANTRAP operand is zero-extended to 32 bits.
7183 // - "All ones" constant is used for G_TRAP.
7184 uint32_t MsgVal = ~0u;
7185 if (I.getOpcode() == TargetOpcode::G_UBSANTRAP)
7186 MsgVal = static_cast<uint32_t>(I.getOperand(0).getImm());
7187
7188 SPIRVTypeInst MsgType = GR.getOrCreateSPIRVIntegerType(32, I, TII);
7189 Register MsgReg = buildI32ConstantInEntryBlock(MsgVal, I, MsgType);
7190
7191 MachineBasicBlock &BB = *I.getParent();
7192 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAbortKHR))
7193 .addUse(GR.getSPIRVTypeID(MsgType))
7194 .addUse(MsgReg)
7195 .constrainAllUses(TII, TRI, RBI);
7196 return true;
7197}
7198
7199bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
7200 SPIRVTypeInst ResType,
7201 MachineInstr &I) const {
7202 // Change order of instructions if needed: all OpVariable instructions in a
7203 // function must be the first instructions in the first block
7204 auto It = getOpVariableMBBIt(*I.getMF());
7205
7206 // Pointers to opaque types stay typed even with the extension on, so emit the
7207 // untyped variant only when the result is actually an untyped pointer.
7208 bool UseUntypedPointers =
7209 ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7210 unsigned Opcode =
7211 UseUntypedPointers ? SPIRV::OpUntypedVariableKHR : SPIRV::OpVariable;
7212
7213 auto MIB = BuildMI(*It->getParent(), It, It->getDebugLoc(), TII.get(Opcode))
7214 .addDef(ResVReg)
7215 .addUse(GR.getSPIRVTypeID(ResType))
7216 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
7217
7218 // OpUntypedVariableKHR takes an extra Data Type operand right after the
7219 // storage class.
7220 if (UseUntypedPointers) {
7221 // Get the element type that was stored when processing spv_assign_ptr_type.
7222 SPIRVTypeInst DataType = GR.getUntypedPtrElementType(ResVReg);
7223 if (!DataType)
7224 return diagnoseUnsupported(
7225 I, "could not deduce the data type of an untyped variable");
7226 MIB.addUse(GR.getSPIRVTypeID(DataType));
7227 }
7228 MIB.constrainAllUses(TII, TRI, RBI);
7229
7230 if (!STI.isShader()) {
7231 unsigned Alignment = I.getOperand(2).getImm();
7232 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment,
7233 {Alignment});
7234 }
7235 return true;
7236}
7237
7238bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
7239 // InstructionSelector walks backwards through the instructions. We can use
7240 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
7241 // first, so can generate an OpBranchConditional here. If there is no
7242 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
7243 const MachineInstr *PrevI = I.getPrevNode();
7244 MachineBasicBlock &MBB = *I.getParent();
7245 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
7246 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
7247 .addUse(PrevI->getOperand(0).getReg())
7248 .addMBB(PrevI->getOperand(1).getMBB())
7249 .addMBB(I.getOperand(0).getMBB())
7250 .constrainAllUses(TII, TRI, RBI);
7251 return true;
7252 }
7253 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
7254 .addMBB(I.getOperand(0).getMBB())
7255 .constrainAllUses(TII, TRI, RBI);
7256 return true;
7257}
7258
7259bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
7260 // InstructionSelector walks backwards through the instructions. For an
7261 // explicit conditional branch with no fallthrough, we use both a G_BR and a
7262 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
7263 // generate the OpBranchConditional in selectBranch above.
7264 //
7265 // If an OpBranchConditional has been generated, we simply return, as the work
7266 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
7267 // implicit fallthrough to the next basic block, so we need to create an
7268 // OpBranchConditional with an explicit "false" argument pointing to the next
7269 // basic block that LLVM would fall through to.
7270 const MachineInstr *NextI = I.getNextNode();
7271 // Check if this has already been successfully selected.
7272 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
7273 return true;
7274 // Must be relying on implicit block fallthrough, so generate an
7275 // OpBranchConditional with the "next" basic block as the "false" target.
7276 MachineBasicBlock &MBB = *I.getParent();
7277 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
7278 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
7279 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
7280 .addUse(I.getOperand(0).getReg())
7281 .addMBB(I.getOperand(1).getMBB())
7282 .addMBB(NextMBB)
7283 .constrainAllUses(TII, TRI, RBI);
7284 return true;
7285}
7286
7287bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
7288 MachineInstr &I) const {
7289 auto MIB =
7290 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::PHI))
7291 .addDef(ResVReg);
7292 const unsigned NumOps = I.getNumOperands();
7293 for (unsigned i = 1; i < NumOps; i += 2) {
7294 MIB.addUse(I.getOperand(i + 0).getReg());
7295 MIB.addMBB(I.getOperand(i + 1).getMBB());
7296 }
7297 MIB.constrainAllUses(TII, TRI, RBI);
7298 return true;
7299}
7300
7301bool SPIRVInstructionSelector::selectGlobalValue(
7302 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
7303 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
7304 MachineIRBuilder MIRBuilder(I);
7305 const GlobalValue *GV = I.getOperand(1).getGlobal();
7306 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV));
7307
7308 std::string GlobalIdent;
7309 if (!GV->hasName()) {
7310 unsigned &ID = UnnamedGlobalIDs[GV];
7311 if (ID == 0)
7312 ID = UnnamedGlobalIDs.size();
7313 GlobalIdent = "__unnamed_" + Twine(ID).str();
7314 } else {
7315 GlobalIdent = GV->getName();
7316 }
7317
7318 // Behaviour of functions as operands depends on availability of the
7319 // corresponding extension (SPV_INTEL_function_pointers):
7320 // - If there is an extension to operate with functions as operands:
7321 // We create a proper constant operand and evaluate a correct type for a
7322 // function pointer.
7323 // - Without the required extension:
7324 // We have functions as operands in tests with blocks of instruction e.g. in
7325 // transcoding/global_block.ll. These operands are not used and should be
7326 // substituted by zero constants. Their type is expected to be always
7327 // OpTypePointer Function %uchar.
7328 if (isa<Function>(GV)) {
7329 const Constant *ConstVal = GV;
7330 MachineBasicBlock &BB = *I.getParent();
7331 Register NewReg = GR.find(ConstVal, GR.CurMF);
7332 if (!NewReg.isValid()) {
7333 const Function *GVFun =
7334 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)
7335 ? dyn_cast<Function>(GV)
7336 : nullptr;
7337 SPIRVTypeInst ResType = GR.getOrCreateSPIRVPointerType(
7338 GVType, I,
7339 GVFun ? SPIRV::StorageClass::CodeSectionINTEL
7340 : addressSpaceToStorageClass(GV->getAddressSpace(), STI));
7341 if (GVFun) {
7342 // References to a function via function pointers generate virtual
7343 // registers without a definition. We will resolve it later, during
7344 // module analysis stage.
7345 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
7346 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
7347 Register FuncVReg =
7348 MRI->createGenericVirtualRegister(GR.getRegType(ResType));
7349 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass);
7350 GR.assignSPIRVTypeToVReg(ResType, FuncVReg, *GR.CurMF);
7351 MachineInstrBuilder MIB1 =
7352 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
7353 .addDef(FuncVReg)
7354 .addUse(ResTypeReg);
7355 MachineInstrBuilder MIB2 =
7356 BuildMI(BB, I, I.getDebugLoc(),
7357 TII.get(SPIRV::OpConstantFunctionPointerINTEL))
7358 .addDef(ResVReg)
7359 .addUse(ResTypeReg)
7360 .addUse(FuncVReg);
7361 GR.add(ConstVal, MIB2);
7362 // mapping the function pointer to the used Function
7363 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun);
7364 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
7365 MIB1.constrainAllUses(TII, TRI, RBI);
7366 MIB2.constrainAllUses(TII, TRI, RBI);
7367 return true;
7368 }
7369 MachineInstrBuilder MIB3 =
7370 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
7371 .addDef(ResVReg)
7372 .addUse(GR.getSPIRVTypeID(ResType));
7373 GR.add(ConstVal, MIB3);
7374 GR.recordFunctionPointer(&MIB3.getInstr()->getOperand(0),
7375 cast<Function>(GV));
7376 MIB3.constrainAllUses(TII, TRI, RBI);
7377 return true;
7378 }
7379 assert(NewReg != ResVReg);
7380 return BuildCOPY(ResVReg, NewReg, I);
7381 }
7382 auto GlobalVar = cast<GlobalVariable>(GV);
7383 assert(GlobalVar->getName() != "llvm.global.annotations");
7384
7385 // Skip empty declaration for GVs with initializers till we get the decl with
7386 // passed initializer.
7387 if (hasInitializer(GlobalVar) && !Init)
7388 return true;
7389
7390 const std::optional<SPIRV::LinkageType::LinkageType> LnkType =
7391 getSpirvLinkageTypeFor(STI, *GV);
7392
7393 if (LnkType && *LnkType == SPIRV::LinkageType::Import)
7394 Init = nullptr;
7395
7396 const unsigned AddrSpace = GV->getAddressSpace();
7397 SPIRV::StorageClass::StorageClass StorageClass =
7398 addressSpaceToStorageClass(AddrSpace, STI);
7399 SPIRVTypeInst ResType =
7400 GR.getOrCreateSPIRVPointerType(GVType, I, StorageClass);
7401 Register Reg = GR.buildGlobalVariable(
7402 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init,
7403 GlobalVar->isConstant(), LnkType, MIRBuilder, true);
7404 // TODO: For AMDGCN, we pipe externally_initialized through via
7405 // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during
7406 // reverse translation. We should remove this once SPIR-V gains the ability to
7407 // express the concept.
7408 if (GlobalVar->isExternallyInitialized() &&
7409 STI.getTargetTriple().getVendor() == Triple::AMD) {
7410 constexpr unsigned ReadWriteINTEL = 3u;
7411 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL,
7412 {ReadWriteINTEL});
7413 MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
7414 addStringImm(GV->getName(), MIB);
7415 }
7416 return Reg.isValid();
7417}
7418
7419bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
7420 SPIRVTypeInst ResType,
7421 MachineInstr &I) const {
7422 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
7423 return selectExtInst(ResVReg, ResType, I, CL::log10);
7424 }
7425
7426 // There is no log10 instruction in the GLSL Extended Instruction set, so it
7427 // is implemented as:
7428 // log10(x) = log2(x) * (1 / log2(10))
7429 // = log2(x) * 0.30103
7430
7431 MachineIRBuilder MIRBuilder(I);
7432 MachineBasicBlock &BB = *I.getParent();
7433
7434 // Build log2(x).
7435 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
7436 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
7437 .addDef(VarReg)
7438 .addUse(GR.getSPIRVTypeID(ResType))
7439 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
7440 .addImm(GL::Log2)
7441 .add(I.getOperand(1))
7442 .constrainAllUses(TII, TRI, RBI);
7443
7444 // Build 0.30103.
7445 assert(isVectorType(ResType) || ResType.isAnyTypeFloat());
7446 // TODO: Add matrix implementation once supported by the HLSL frontend.
7447 SPIRVTypeInst SpirvScalarType = GR.getScalarOrVectorComponentType(ResType);
7448 // The literal must match the precision of the scalar type, otherwise the
7449 // OpConstant will contain non-zero high-order bits and fail SPIR-V
7450 // validation when the type is narrower than 32 bits (e.g. half).
7451 APFloat ScaleVal(0.30103);
7452 bool LosesInfo;
7453 ScaleVal.convert(
7454 getZeroFP(GR.getTypeForSPIRVType(SpirvScalarType)).getSemantics(),
7455 APFloat::rmNearestTiesToEven, &LosesInfo);
7456 Register ScaleReg = GR.buildConstantFP(ScaleVal, MIRBuilder, SpirvScalarType);
7457
7458 // Multiply log2(x) by 0.30103 to get log10(x) result.
7459 auto Opcode =
7460 isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS;
7461 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
7462 .addDef(ResVReg)
7463 .addUse(GR.getSPIRVTypeID(ResType))
7464 .addUse(VarReg)
7465 .addUse(ScaleReg)
7466 .constrainAllUses(TII, TRI, RBI);
7467 return true;
7468}
7469
7470bool SPIRVInstructionSelector::selectFpowi(Register ResVReg,
7471 SPIRVTypeInst ResType,
7472 MachineInstr &I) const {
7473 // On OpenCL targets, pown(gentype x, intn n) maps directly.
7474 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std))
7475 return selectExtInst(ResVReg, ResType, I, CL::pown);
7476
7477 // On GLSL (Vulkan) targets, there is no integer-exponent power instruction.
7478 // Lower as: Pow(base, OpConvertSToF(exp)).
7479 if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
7480 Register BaseReg = I.getOperand(1).getReg();
7481 Register ExpReg = I.getOperand(2).getReg();
7482 Register FloatExpReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
7483 if (!selectOpWithSrcs(FloatExpReg, ResType, I, {ExpReg},
7484 SPIRV::OpConvertSToF))
7485 return false;
7486 return selectExtInst(ResVReg, ResType, I, GL::Pow,
7487 /*setMIFlags=*/true, /*useMISrc=*/false,
7488 {BaseReg, FloatExpReg});
7489 }
7490 return false;
7491}
7492
7493bool SPIRVInstructionSelector::selectModf(Register ResVReg,
7494 SPIRVTypeInst ResType,
7495 MachineInstr &I) const {
7496 // llvm.modf has a single arg --the number to be decomposed-- and returns a
7497 // struct { restype, restype }, while OpenCLLIB::modf has two args --the
7498 // number to be decomposed and a pointer--, returns the fractional part and
7499 // the integral part is stored in the pointer argument. Therefore, we can't
7500 // use directly the OpenCLLIB::modf intrinsic. However, we can do some
7501 // scaffolding to make it work. The idea is to create an alloca instruction
7502 // to get a ptr, pass this ptr to OpenCL::modf, and then load the value
7503 // from this ptr to place it in the struct. llvm.modf returns the fractional
7504 // part as the first element of the result, and the integral part as the
7505 // second element of the result.
7506
7507 // At this point, the return type is not a struct anymore, but rather two
7508 // independent elements of SPIRVResType. We can get each independent element
7509 // from I.getDefs() or I.getOperands().
7510 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
7511 MachineIRBuilder MIRBuilder(I);
7512 SPIRVTypeInst FloatType =
7513 GR.getSPIRVTypeForVReg(I.getOperand(I.getNumExplicitDefs()).getReg());
7514 // Get pointer type for alloca variable.
7515 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
7516 FloatType, MIRBuilder, SPIRV::StorageClass::Function);
7517 // Create new register for the pointer type of alloca variable.
7518 Register PtrTyReg =
7519 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
7520 MIRBuilder.getMRI()->setType(
7521 PtrTyReg,
7522 LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
7523 GR.getPointerSize()));
7524
7525 // Assign SPIR-V type of the pointer type of the alloca variable to the
7526 // new register.
7527 GR.assignSPIRVTypeToVReg(PtrType, PtrTyReg, MIRBuilder.getMF());
7528 MachineBasicBlock::iterator VarPos = getOpVariableMBBIt(*I.getMF());
7529 MachineBasicBlock &EntryBB = I.getMF()->front();
7530 const bool IsUntyped =
7531 PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7532 auto AllocaMIB =
7533 BuildMI(EntryBB, VarPos, I.getDebugLoc(),
7534 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
7535 : SPIRV::OpVariable))
7536 .addDef(PtrTyReg)
7537 .addUse(GR.getSPIRVTypeID(PtrType))
7538 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
7539 if (IsUntyped)
7540 AllocaMIB.addUse(GR.getSPIRVTypeID(ResType)); // Data Type
7541 Register Variable = AllocaMIB->getOperand(0).getReg();
7542
7543 MachineBasicBlock &BB = *I.getParent();
7544 // Create the OpenCLLIB::modf instruction.
7545 auto MIB =
7546 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
7547 .addDef(ResVReg)
7548 .addUse(GR.getSPIRVTypeID(FloatType))
7549 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
7550 .addImm(CL::modf)
7551 .setMIFlags(I.getFlags())
7552 .add(I.getOperand(I.getNumExplicitDefs())) // Floating point value.
7553 .addUse(Variable); // Pointer to integral part.
7554 // Assign the integral part stored in the ptr to the second element of the
7555 // result.
7556 Register IntegralPartReg = I.getOperand(1).getReg();
7557 if (IntegralPartReg.isValid() && !MRI->use_nodbg_empty(IntegralPartReg)) {
7558 // Load the value from the pointer to integral part.
7559 auto LoadMIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
7560 .addDef(IntegralPartReg)
7561 .addUse(GR.getSPIRVTypeID(FloatType))
7562 .addUse(Variable);
7563 LoadMIB.constrainAllUses(TII, TRI, RBI);
7564 }
7565
7566 MIB.constrainAllUses(TII, TRI, RBI);
7567 return true;
7568 } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
7569 assert(false && "GLSL::Modf is deprecated.");
7570 // FIXME: GL::Modf is deprecated, use Modfstruct instead.
7571 return false;
7572 }
7573 return false;
7574}
7575
7576// Generate the instructions to load 3-element vector builtin input
7577// IDs/Indices.
7578// Like: GlobalInvocationId, LocalInvocationId, etc....
7579
7580bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
7581 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
7582 SPIRVTypeInst ResType, MachineInstr &I) const {
7583 MachineIRBuilder MIRBuilder(I);
7584 const SPIRVTypeInst Vec3Ty =
7585 GR.getOrCreateSPIRVVectorType(ResType, 3, MIRBuilder, false);
7586 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
7587 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
7588
7589 // Create new register for the input ID builtin variable.
7590 Register NewRegister =
7591 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
7592 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64));
7593 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
7594
7595 // Build global variable with the necessary decorations for the input ID
7596 // builtin variable.
7597 Register Variable = GR.buildGlobalVariable(
7598 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
7599 SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder,
7600 false);
7601
7602 // Create new register for loading value.
7603 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
7604 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
7605 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64));
7606 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF());
7607
7608 // Load v3uint value from the global variable.
7609 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
7610 .addDef(LoadedRegister)
7611 .addUse(GR.getSPIRVTypeID(Vec3Ty))
7612 .addUse(Variable);
7613
7614 // Get the input ID index. Expecting operand is a constant immediate value,
7615 // wrapped in a type assignment.
7616 assert(I.getOperand(2).isReg());
7617 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI);
7618
7619 // Extract the input ID from the loaded vector value.
7620 MachineBasicBlock &BB = *I.getParent();
7621 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
7622 .addDef(ResVReg)
7623 .addUse(GR.getSPIRVTypeID(ResType))
7624 .addUse(LoadedRegister)
7625 .addImm(ThreadId);
7626 MIB.constrainAllUses(TII, TRI, RBI);
7627 return true;
7628}
7629
7630// Generate the instructions to load 32-bit integer builtin input IDs/Indices.
7631// Like LocalInvocationIndex
7632bool SPIRVInstructionSelector::loadBuiltinInputID(
7633 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
7634 SPIRVTypeInst ResType, MachineInstr &I) const {
7635 MachineIRBuilder MIRBuilder(I);
7636 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
7637 ResType, MIRBuilder, SPIRV::StorageClass::Input);
7638
7639 // Create new register for the input ID builtin variable.
7640 Register NewRegister =
7641 MIRBuilder.getMRI()->createVirtualRegister(GR.getRegClass(PtrType));
7642 MIRBuilder.getMRI()->setType(
7643 NewRegister,
7644 LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Input),
7645 GR.getPointerSize()));
7646 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
7647
7648 // Build global variable with the necessary decorations for the input ID
7649 // builtin variable.
7650 Register Variable = GR.buildGlobalVariable(
7651 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
7652 SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder,
7653 false);
7654
7655 // Load uint value from the global variable.
7656 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
7657 .addDef(ResVReg)
7658 .addUse(GR.getSPIRVTypeID(ResType))
7659 .addUse(Variable);
7660
7661 MIB.constrainAllUses(TII, TRI, RBI);
7662 return true;
7663}
7664
7665SPIRVTypeInst SPIRVInstructionSelector::widenTypeToVec4(SPIRVTypeInst Type,
7666 MachineInstr &I) const {
7667 MachineIRBuilder MIRBuilder(I);
7668 if (!isVectorType(Type))
7669 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder, false);
7670
7671 if (GR.getScalarOrVectorComponentCount(Type) == 4)
7672 return Type;
7673
7674 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(Type);
7675 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder, false);
7676}
7677
7678bool SPIRVInstructionSelector::loadHandleBeforePosition(
7679 Register &HandleReg, SPIRVTypeInst ResType, GIntrinsic &HandleDef,
7680 MachineInstr &Pos) const {
7681
7682 assert(HandleDef.getIntrinsicID() ==
7683 Intrinsic::spv_resource_handlefrombinding);
7684 uint32_t Set = foldImm(HandleDef.getOperand(2), MRI);
7685 uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI);
7686 uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI);
7687 Register IndexReg = HandleDef.getOperand(5).getReg();
7688 std::string Name =
7689 getStringValueFromReg(HandleDef.getOperand(6).getReg(), *MRI);
7690
7691 bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
7692 MachineIRBuilder MIRBuilder(HandleDef);
7693 SPIRVTypeInst VarType = ResType;
7694 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
7695
7696 if (IsStructuredBuffer) {
7697 VarType = GR.getPointeeType(ResType);
7698 SC = GR.getPointerStorageClass(ResType);
7699 }
7700
7701 if (ResType->getOpcode() == SPIRV::OpTypeImage && ArraySize == 0)
7702 MIRBuilder.buildInstr(SPIRV::OpCapability)
7703 .addImm(SPIRV::Capability::RuntimeDescriptorArrayEXT);
7704
7705 Register VarReg =
7706 buildPointerToResource(SPIRVTypeInst(VarType), SC, Set, Binding,
7707 ArraySize, IndexReg, Name, MIRBuilder);
7708
7709 // The handle for the buffer is the pointer to the resource. For an image, the
7710 // handle is the image object. So images get an extra load.
7711 uint32_t LoadOpcode =
7712 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
7713 GR.assignSPIRVTypeToVReg(ResType, HandleReg, *Pos.getMF());
7714 BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(), TII.get(LoadOpcode))
7715 .addDef(HandleReg)
7716 .addUse(GR.getSPIRVTypeID(ResType))
7717 .addUse(VarReg)
7718 .constrainAllUses(TII, TRI, RBI);
7719 return true;
7720}
7721
7722bool SPIRVInstructionSelector::errorIfInstrOutsideShader(
7723 MachineInstr &I) const {
7724 if (!STI.isShader())
7725 return diagnoseUnsupported(
7726 I, "this instruction is only supported in shaders.");
7727 return true;
7728}
7729
7730namespace llvm {
7731InstructionSelector *
7732createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
7733 const SPIRVSubtarget &Subtarget,
7734 const RegisterBankInfo &RBI) {
7735 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
7736}
7737} // namespace llvm
UseMI
MachineInstrBuilder & UseMI
Definition AArch64ExpandPseudoInsts.cpp:125
DefMI
MachineInstrBuilder MachineInstrBuilder & DefMI
Definition AArch64ExpandPseudoInsts.cpp:126
GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_PREDICATES_INIT
GET_GLOBALISEL_TEMPORARIES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:482
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:483
APFloat.h
This file declares a class to represent arbitrary precision floating point values and provide a varie...
selectUnmergeValues
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Definition ARMInstructionSelector.cpp:264
MBB
MachineBasicBlock & MBB
Definition ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
SwapBits
static uint8_t SwapBits(uint8_t Val)
Definition BPFMCCodeEmitter.cpp:122
true
basic Basic Alias true
Definition BasicAliasAnalysis.cpp:2111
X
#define X(NUM, ENUM, NAME)
Definition ELF.h:857
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Binding
DXIL Resource Implicit Binding
Definition DXILResourceImplicitBinding.cpp:185
DEBUG_TYPE
#define DEBUG_TYPE
Definition GenericCycleImpl.h:34
GenericMachineInstrs.h
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
TII
const HexagonInstrInfo * TII
Definition HexagonCopyToCombine.cpp:118
MI
IRTranslator LLVM IR MI
Definition IRTranslator.cpp:866
LLVMIntType
LLVMTypeRef LLVMIntType(unsigned NumBits)
Definition Core.cpp:740
NumOps
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
Definition ItaniumDemangle.h:3474
Direction
Loop::LoopBounds::Direction Direction
Definition LoopInfo.cpp:253
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
Reg
Register Reg
Definition MachineSink.cpp:2130
TRI
Register const TargetRegisterInfo * TRI
Definition MachineSink.cpp:2131
Register
Promote Memory to Register
Definition Mem2Reg.cpp:110
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
High
uint64_t High
Definition NVVMIntrRange.cpp:46
II
uint64_t IntrinsicInst * II
Definition NVVMIntrRange.cpp:46
getName
static StringRef getName(Value *V)
Definition ProvenanceAnalysisEvaluator.cpp:20
getFCmpOpcode
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
Definition RISCVInstructionSelector.cpp:2020
isConcreteSPIRVType
static bool isConcreteSPIRVType(SPIRVTypeInst Ty, const SPIRVGlobalRegistry &GR)
Definition SPIRVInstructionSelector.cpp:7122
getOneFP
static APFloat getOneFP(const Type *LLVMFloatTy)
Definition SPIRVInstructionSelector.cpp:3265
isUSMStorageClass
static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVInstructionSelector.cpp:2939
isASCastInGVar
static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg)
Definition SPIRVInstructionSelector.cpp:2950
mayApplyGenericSelection
static bool mayApplyGenericSelection(unsigned Opcode)
Definition SPIRVInstructionSelector.cpp:996
getZeroFP
static APFloat getZeroFP(const Type *LLVMFloatTy)
Definition SPIRVInstructionSelector.cpp:3249
ExtInstList
std::vector< std::pair< SPIRV::InstructionSet::InstructionSet, uint32_t > > ExtInstList
Definition SPIRVInstructionSelector.cpp:49
intrinsicHasSideEffects
static bool intrinsicHasSideEffects(Intrinsic::ID ID)
Definition SPIRVInstructionSelector.cpp:708
getBoolCmpOpcode
static unsigned getBoolCmpOpcode(unsigned PredNum)
Definition SPIRVInstructionSelector.cpp:3237
getICmpOpcode
static unsigned getICmpOpcode(unsigned PredNum)
Definition SPIRVInstructionSelector.cpp:3197
isOpcodeWithNoSideEffects
static bool isOpcodeWithNoSideEffects(unsigned Opcode)
Definition SPIRVInstructionSelector.cpp:775
addMemoryOperands
static void addMemoryOperands(MachineMemOperand *MemOp, MachineInstrBuilder &MIB, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry &GR)
Definition SPIRVInstructionSelector.cpp:2031
isConstReg
static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef)
Definition SPIRVInstructionSelector.cpp:639
getPtrCmpOpcode
static unsigned getPtrCmpOpcode(unsigned Pred)
Definition SPIRVInstructionSelector.cpp:3225
isDead
bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition SPIRVInstructionSelector.cpp:810
convertPtrToInt
static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType, LegalizerHelper &Helper, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR)
Definition SPIRVLegalizerInfo.cpp:657
Msg
const char * Msg
Definition SPIRVModuleAnalysis.cpp:59
STLExtras.h
This file contains some templates that are useful if you are working with the STL at all.
BaseType
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
Definition SafepointIRVerifier.cpp:314
SmallSet.h
This file defines the SmallSet class.
StringExtras.h
This file contains some functions that are useful when dealing with strings.
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
Y
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
CallbackFunction
static ManagedStatic< cl::opt< FnT >, OptCreatorT > CallbackFunction
Definition TableGenBackend.cpp:53
Mul
BinaryOperator * Mul
Definition X86PartialReduction.cpp:77
llvm::APFloatBase::IEEEsingle
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
llvm::APFloatBase::BFloat
static const fltSemantics & BFloat()
Definition APFloat.h:303
llvm::APFloatBase::IEEEdouble
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
llvm::APFloatBase::IEEEhalf
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
llvm::APFloat::getSemantics
const fltSemantics & getSemantics() const
Definition APFloat.h:1583
llvm::APFloat::getOne
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition APFloat.h:1184
llvm::APFloat::getZero
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1175
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:231
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:226
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition BlockFrequencyInfo.h:38
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DenseMapBase::size
unsigned size() const
Definition DenseMap.h:172
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:353
llvm::GFConstant::getConstant
static LLVM_ABI std::optional< GFConstant > getConstant(Register Const, const MachineRegisterInfo &MRI)
Definition Utils.cpp:2037
llvm::GIConstant::getConstant
static LLVM_ABI std::optional< GIConstant > getConstant(Register Const, const MachineRegisterInfo &MRI)
Definition Utils.cpp:1997
llvm::GIntrinsic
Represents a call to an intrinsic.
Definition GenericMachineInstrs.h:566
llvm::GIntrinsic::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Definition GenericMachineInstrs.h:568
llvm::GlobalValue::getAddressSpace
unsigned getAddressSpace() const
Definition GlobalValue.h:211
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition GlobalValue.h:711
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition GlobalValue.h:60
llvm::IntegerType::get
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:348
llvm::LLT::scalar
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition LowLevelType.h:88
llvm::LLT::isValid
constexpr bool isValid() const
Definition LowLevelType.h:262
llvm::LLT::getNumElements
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition LowLevelType.h:350
llvm::LLT::isVector
constexpr bool isVector() const
Definition LowLevelType.h:289
llvm::LLT::pointer
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition LowLevelType.h:115
llvm::LLT::fixed_vector
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition LowLevelType.h:203
llvm::MachineBasicBlock::getNumber
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
Definition MachineBasicBlock.h:1273
llvm::MachineBasicBlock::getFirstNonPHI
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition MachineBasicBlock.cpp:203
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition MachineBasicBlock.h:325
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition MachineBasicBlock.h:343
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition MachineFunction.h:799
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition MachineFunction.h:750
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition MachineIRBuilder.h:238
llvm::MachineIRBuilder::getInsertPt
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
Definition MachineIRBuilder.h:336
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition MachineIRBuilder.h:424
llvm::MachineIRBuilder::getMF
MachineFunction & getMF()
Getter for the function we currently build.
Definition MachineIRBuilder.h:290
llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition MachineIRBuilder.h:312
llvm::MachineInstrBuilder::constrainAllUses
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
Definition MachineInstrBuilder.h:441
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
Definition MachineInstrBuilder.h:225
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Definition MachineInstrBuilder.h:199
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition MachineInstrBuilder.h:233
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition MachineInstrBuilder.h:326
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition MachineInstrBuilder.h:248
llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition MachineInstrBuilder.h:218
llvm::MachineInstrBuilder::setMIFlags
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Definition MachineInstrBuilder.h:379
llvm::MachineInstrBuilder::getInstr
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Definition MachineInstrBuilder.h:191
llvm::MachineInstr
Representation of each machine instruction.
Definition MachineInstr.h:74
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition MachineInstr.h:602
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition MachineInstr.h:374
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition MachineInstr.h:605
llvm::MachineInstr::getNumExplicitOperands
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
Definition MachineInstr.cpp:847
llvm::MachineInstr::getNumExplicitDefs
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
Definition MachineInstr.cpp:865
llvm::MachineInstr::substituteRegister
LLVM_ABI void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo)
Replace all occurrences of FromReg with ToReg:SubIdx, properly composing subreg indices where necessa...
Definition MachineInstr.cpp:1326
llvm::MachineInstr::emitGenericError
LLVM_ABI void emitGenericError(const Twine &ErrMsg) const
Definition MachineInstr.cpp:2366
llvm::MachineInstr::getMF
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
Definition MachineInstr.cpp:782
llvm::MachineInstr::getDebugLoc
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition MachineInstr.h:526
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition MachineInstr.h:610
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition MachineMemOperand.h:141
llvm::MachineMemOperand::MOVolatile
@ MOVolatile
The memory access is volatile.
Definition MachineMemOperand.h:152
llvm::MachineMemOperand::MONonTemporal
@ MONonTemporal
The memory access is non-temporal.
Definition MachineMemOperand.h:154
llvm::MachineOperand::getImm
int64_t getImm() const
Definition MachineOperand.h:560
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition MachineOperand.h:331
llvm::MachineOperand::getMBB
MachineBasicBlock * getMBB() const
Definition MachineOperand.h:575
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition MachineOperand.h:372
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition MachineRegisterInfo.h:53
llvm::MachineRegisterInfo::use_instr_iterator
defusechain_instr_iterator< true, false, false, true > use_instr_iterator
use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the specified register,...
Definition MachineRegisterInfo.h:490
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition MachineRegisterInfo.h:676
llvm::MachineRegisterInfo::getVRegDef
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition MachineRegisterInfo.cpp:404
llvm::MachineRegisterInfo::use_instr_begin
use_instr_iterator use_instr_begin(Register RegNo) const
Definition MachineRegisterInfo.h:492
llvm::MachineRegisterInfo::use_nodbg_empty
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
Definition MachineRegisterInfo.h:590
llvm::MachineRegisterInfo::def_instr_end
static def_instr_iterator def_instr_end()
Definition MachineRegisterInfo.h:416
llvm::MachineRegisterInfo::def_instr_iterator
defusechain_instr_iterator< false, true, false, true > def_instr_iterator
def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the specified register,...
Definition MachineRegisterInfo.h:411
llvm::MachineRegisterInfo::createVirtualRegister
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition MachineRegisterInfo.cpp:154
llvm::MachineRegisterInfo::def_instr_begin
def_instr_iterator def_instr_begin(Register RegNo) const
Definition MachineRegisterInfo.h:413
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition MachineRegisterInfo.h:799
llvm::MachineRegisterInfo::hasOneUse
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
Definition MachineRegisterInfo.h:525
llvm::MachineRegisterInfo::use_instr_end
static use_instr_iterator use_instr_end()
Definition MachineRegisterInfo.h:495
llvm::MachineRegisterInfo::use_nodbg_instructions
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
Definition MachineRegisterInfo.h:567
llvm::MachineRegisterInfo::setType
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
Definition MachineRegisterInfo.cpp:185
llvm::MachineRegisterInfo::getMF
const MachineFunction & getMF() const
Definition MachineRegisterInfo.h:201
llvm::MachineRegisterInfo::setRegClass
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition MachineRegisterInfo.cpp:58
llvm::MachineRegisterInfo::createGenericVirtualRegister
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Definition MachineRegisterInfo.cpp:191
llvm::MachineRegisterInfo::getRegClassOrNull
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
Definition MachineRegisterInfo.h:693
llvm::MachineRegisterInfo::use_instructions
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
Definition MachineRegisterInfo.h:500
llvm::MachineRegisterInfo::getNumVirtRegs
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
Definition MachineRegisterInfo.h:824
llvm::MachineRegisterInfo::replaceRegWith
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Definition MachineRegisterInfo.cpp:386
llvm::ProfileSummaryInfo
Analysis providing profile information.
Definition ProfileSummaryInfo.h:42
llvm::RegisterBankInfo
Holds all the information related to register banks.
Definition RegisterBankInfo.h:41
llvm::Register
Wrapper class representing virtual and physical registers.
Definition Register.h:20
llvm::Register::isValid
constexpr bool isValid() const
Definition Register.h:112
llvm::Register::isPhysical
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
llvm::SPIRVGlobalRegistry::isScalarOrVectorSigned
bool isScalarOrVectorSigned(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1547
llvm::SPIRVGlobalRegistry::getOrCreateOpTypeSampledImage
SPIRVTypeInst getOrCreateOpTypeSampledImage(SPIRVTypeInst ImageType, MachineIRBuilder &MIRBuilder)
Definition SPIRVGlobalRegistry.cpp:1791
llvm::SPIRVGlobalRegistry::assignSPIRVTypeToVReg
void assignSPIRVTypeToVReg(SPIRVTypeInst Type, Register VReg, const MachineFunction &MF)
Definition SPIRVGlobalRegistry.cpp:119
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVPointerType
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
Definition SPIRVGlobalRegistry.cpp:2064
llvm::SPIRVGlobalRegistry::getRegClass
const TargetRegisterClass * getRegClass(SPIRVTypeInst SpvType) const
Definition SPIRVGlobalRegistry.cpp:2197
llvm::SPIRVGlobalRegistry::getOrAddMemAliasingINTELInst
MachineInstr * getOrAddMemAliasingINTELInst(MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD)
Definition SPIRVGlobalRegistry.cpp:2264
llvm::SPIRVGlobalRegistry::isAggregateType
bool isAggregateType(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.h:378
llvm::SPIRVGlobalRegistry::getScalarOrVectorBitWidth
unsigned getScalarOrVectorBitWidth(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1498
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVIntegerType
SPIRVTypeInst getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
Definition SPIRVGlobalRegistry.cpp:1921
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVVectorType
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
Definition SPIRVGlobalRegistry.cpp:2020
llvm::SPIRVGlobalRegistry::buildGlobalVariable
Register buildGlobalVariable(Register Reg, SPIRVTypeInst BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)
Definition SPIRVGlobalRegistry.cpp:816
llvm::SPIRVGlobalRegistry::getResultType
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
Definition SPIRVGlobalRegistry.cpp:1378
llvm::SPIRVGlobalRegistry::getScalarOrVectorComponentCount
unsigned getScalarOrVectorComponentCount(Register VReg) const
Definition SPIRVGlobalRegistry.cpp:1470
llvm::SPIRVGlobalRegistry::getTypeForSPIRVType
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
Definition SPIRVGlobalRegistry.h:338
llvm::SPIRVGlobalRegistry::isBitcastCompatible
bool isBitcastCompatible(SPIRVTypeInst Type1, SPIRVTypeInst Type2) const
Definition SPIRVGlobalRegistry.cpp:1558
llvm::SPIRVGlobalRegistry::getOrCreateConstFP
Register getOrCreateConstFP(APFloat Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Definition SPIRVGlobalRegistry.cpp:369
llvm::SPIRVGlobalRegistry::getRegType
LLT getRegType(SPIRVTypeInst SpvType) const
Definition SPIRVGlobalRegistry.cpp:2224
llvm::SPIRVGlobalRegistry::getOpTypeVoid
SPIRVTypeInst getOpTypeVoid(MachineIRBuilder &MIRBuilder)
Definition SPIRVGlobalRegistry.cpp:214
llvm::SPIRVGlobalRegistry::invalidateMachineInstr
void invalidateMachineInstr(MachineInstr *MI)
Definition SPIRVGlobalRegistry.cpp:222
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVBoolType
SPIRVTypeInst getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder, bool EmitIR)
Definition SPIRVGlobalRegistry.cpp:1995
llvm::SPIRVGlobalRegistry::isScalarOfType
bool isScalarOfType(Register VReg, unsigned TypeOpcode) const
Definition SPIRVGlobalRegistry.cpp:1436
llvm::SPIRVGlobalRegistry::getSPIRVTypeID
Register getSPIRVTypeID(SPIRVTypeInst SpirvType) const
Definition SPIRVGlobalRegistry.cpp:1199
llvm::SPIRVGlobalRegistry::getOrCreateConstInt
Register getOrCreateConstInt(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Definition SPIRVGlobalRegistry.cpp:419
llvm::SPIRVGlobalRegistry::getOrCreateConstIntArray
Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
Definition SPIRVGlobalRegistry.cpp:682
llvm::SPIRVGlobalRegistry::findValueAttrs
bool findValueAttrs(const MachineInstr *Key, Type *&Ty, StringRef &Name)
Definition SPIRVGlobalRegistry.h:187
llvm::SPIRVGlobalRegistry::retrieveScalarOrVectorIntType
SPIRVTypeInst retrieveScalarOrVectorIntType(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1541
llvm::SPIRVGlobalRegistry::getOrCreateGlobalVariableWithBinding
Register getOrCreateGlobalVariableWithBinding(SPIRVTypeInst VarType, uint32_t Set, uint32_t Binding, StringRef Name, MachineIRBuilder &MIRBuilder)
Definition SPIRVGlobalRegistry.cpp:959
llvm::SPIRVGlobalRegistry::changePointerStorageClass
SPIRVTypeInst changePointerStorageClass(SPIRVTypeInst PtrType, SPIRV::StorageClass::StorageClass SC, MachineInstr &I)
Definition SPIRVGlobalRegistry.cpp:2086
llvm::SPIRVGlobalRegistry::getOrCreateConstVector
Register getOrCreateConstVector(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Definition SPIRVGlobalRegistry.cpp:619
llvm::SPIRVGlobalRegistry::buildConstantFP
Register buildConstantFP(APFloat Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType=nullptr)
Definition SPIRVGlobalRegistry.cpp:527
llvm::SPIRVGlobalRegistry::addGlobalObject
void addGlobalObject(const Value *V, const MachineFunction *MF, Register R)
Definition SPIRVGlobalRegistry.h:133
llvm::SPIRVGlobalRegistry::getScalarOrVectorComponentType
SPIRVTypeInst getScalarOrVectorComponentType(SPIRVTypeInst Type) const
Definition SPIRVGlobalRegistry.cpp:1486
llvm::SPIRVGlobalRegistry::recordFunctionPointer
void recordFunctionPointer(const MachineOperand *MO, const Function *F)
Definition SPIRVGlobalRegistry.h:280
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVFloatType
SPIRVTypeInst getOrCreateSPIRVFloatType(unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII)
Definition SPIRVGlobalRegistry.cpp:1974
llvm::SPIRVGlobalRegistry::getPointeeType
SPIRVTypeInst getPointeeType(SPIRVTypeInst PtrType)
Definition SPIRVGlobalRegistry.cpp:1552
llvm::SPIRVGlobalRegistry::getOrCreateSPIRVType
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
Definition SPIRVGlobalRegistry.h:324
llvm::SPIRVGlobalRegistry::isScalarOrVectorOfType
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
Definition SPIRVGlobalRegistry.cpp:1443
llvm::SPIRVGlobalRegistry::setCurrentFunc
MachineFunction * setCurrentFunc(MachineFunction &MF)
Definition SPIRVGlobalRegistry.h:371
llvm::SPIRVGlobalRegistry::getOrCreateConstNullPtr
Register getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType)
Definition SPIRVGlobalRegistry.cpp:765
llvm::SPIRVGlobalRegistry::getSPIRVTypeForVReg
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
Definition SPIRVGlobalRegistry.cpp:1367
llvm::SPIRVGlobalRegistry::getDeducedGlobalValueType
Type * getDeducedGlobalValueType(const GlobalValue *Global)
Definition SPIRVGlobalRegistry.h:233
llvm::SPIRVGlobalRegistry::getOrCreateUndef
Register getOrCreateUndef(MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
Definition SPIRVGlobalRegistry.cpp:2166
llvm::SPIRVGlobalRegistry::getPointerStorageClass
SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const
Definition SPIRVGlobalRegistry.cpp:1577
llvm::SPIRVGlobalRegistry::getUntypedPtrElementType
SPIRVTypeInst getUntypedPtrElementType(Register Reg) const
Definition SPIRVGlobalRegistry.h:228
llvm::SPIRVIRMapping::erase
bool erase(const MachineInstr *MI)
Definition SPIRVIRMapping.h:217
llvm::SPIRVIRMapping::add
bool add(SPIRV::IRHandle Handle, const MachineInstr *MI)
Definition SPIRVIRMapping.h:195
llvm::SPIRVIRMapping::find
Register find(SPIRV::IRHandle Handle, const MachineFunction *MF)
Definition SPIRVIRMapping.h:241
llvm::SPIRVSubtarget::isPhysicalSPIRV
bool isPhysicalSPIRV() const
Definition SPIRVSubtarget.h:92
llvm::SPIRVSubtarget::isAtLeastSPIRVVer
bool isAtLeastSPIRVVer(VersionTuple VerToCompareTo) const
Definition SPIRVSubtarget.cpp:144
llvm::SPIRVSubtarget::canUseExtInstSet
bool canUseExtInstSet(SPIRV::InstructionSet::InstructionSet E) const
Definition SPIRVSubtarget.cpp:131
llvm::SPIRVSubtarget::isLogicalSPIRV
bool isLogicalSPIRV() const
Definition SPIRVSubtarget.h:89
llvm::SPIRVSubtarget::canUseExtension
bool canUseExtension(SPIRV::Extension::Extension E) const
Definition SPIRVSubtarget.cpp:127
llvm::SPIRVTypeInst::isTypeIntOrFloat
bool isTypeIntOrFloat() const
Definition SPIRVTypeInst.h:57
llvm::SPIRVTypeInst::isAnyTypeFloat
bool isAnyTypeFloat() const
Definition SPIRVTypeInst.cpp:38
llvm::SmallPtrSetImpl::erase
bool erase(PtrType Ptr)
Remove pointer from the set.
Definition SmallPtrSet.h:387
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:372
llvm::SmallPtrSetImpl::contains
bool contains(ConstPtrType Ptr) const
Definition SmallPtrSet.h:452
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:518
llvm::SmallSet::insert
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:977
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition SmallVector.h:682
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1236
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:477
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::HalfTyID
@ HalfTyID
16-bit floating point type
Definition Type.h:57
llvm::Type::FloatTyID
@ FloatTyID
32-bit floating point type
Definition Type.h:59
llvm::Type::BFloatTyID
@ BFloatTyID
16-bit floating point type (7-bit significand)
Definition Type.h:58
llvm::Type::DoubleTyID
@ DoubleTyID
64-bit floating point type
Definition Type.h:60
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::isStructTy
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:276
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition Type.h:319
llvm::Type::getTypeID
TypeID getTypeID() const
Return the type id for the type.
Definition Type.h:138
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition STLFunctionalExtras.h:37
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
uint32_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
R600_InstFlag::FC
@ FC
Definition R600Defines.h:32
llvm::AMDGPU::HSAMD::Kernel::Arg::Key::IsConst
constexpr char IsConst[]
Key for Kernel::Arg::Metadata::mIsConst.
Definition AMDGPUMetadata.h:197
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition BitmaskEnum.h:126
llvm::dwarf_linker::DebugSectionKind::DebugLoc
@ DebugLoc
Definition DWARFLinkerBase.h:35
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm::rdf::Def
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
llvm::rdf::Instr
NodeAddr< InstrNode * > Instr
Definition RDFGraph.h:389
llvm::rdf::Use
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
llvm::rdf::Func
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
llvm::sframe::BaseReg
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
llvm::vputils::getOpcode
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
Definition VPlanUtils.cpp:373
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::ThreadPriority::Low
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
llvm::Offset
@ Offset
Definition DWP.cpp:578
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1739
llvm::addStringImm
void addStringImm(StringRef Str, MCInst &Inst)
Definition SPIRVUtils.cpp:163
llvm::getOpVariableMBBIt
MachineBasicBlock::iterator getOpVariableMBBIt(MachineFunction &MF)
Definition SPIRVUtils.cpp:337
llvm::getIConstValSext
int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:516
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition MachineInstrBuilder.h:449
llvm::isTypeFoldingSupported
bool isTypeFoldingSupported(unsigned Opcode)
Definition SPIRVUtils.cpp:1155
llvm::getMemSemanticsWithStorageClass
uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem, uint32_t StorageClassSem)
Definition SPIRVUtils.cpp:452
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::getDef
MachineInstr * getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1167
llvm::addNumImm
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
Definition SPIRVUtils.cpp:193
llvm::salvageDebugInfo
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition Utils.cpp:1675
llvm::constrainSelectedInstRegOperands
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition Utils.cpp:159
llvm::isPreISelGenericOpcode
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
Definition TargetOpcodes.h:30
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition iterator_range.h:67
llvm::createVirtualRegister
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
Definition SPIRVUtils.cpp:976
llvm::getArrayComponentCount
unsigned getArrayComponentCount(const MachineRegisterInfo *MRI, const MachineInstr *ResType)
Definition SPIRVUtils.cpp:1192
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::buildOpDecorate
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, ArrayRef< uint32_t > DecArgs, StringRef StrImm)
Definition SPIRVUtils.cpp:249
llvm::isNullOrNullSplat
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition Utils.cpp:1539
llvm::getMemScope
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx, SyncScope::ID Id)
Definition SPIRVUtils.cpp:460
llvm::getIConstVal
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:510
llvm::createContinuedInstructions
SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)
Definition SPIRVUtils.cpp:1046
llvm::getMemSemanticsForStorageClass
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.cpp:416
llvm::storageClassToAddressSpace
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.h:245
llvm::isVectorType
bool isVectorType(SPIRVTypeInst SPVTy)
Definition SPIRVUtils.cpp:1003
llvm::Value
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
llvm::buildOpName
void buildOpName(Register Target, StringRef Name, MachineIRBuilder &MIRBuilder)
Definition SPIRVUtils.cpp:222
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1746
llvm::getImm
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1173
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::toTypedPointer
Type * toTypedPointer(Type *Ty)
Definition SPIRVUtils.h:479
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isGenericCastablePtr
constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.h:229
llvm::SmallVector
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
Definition SmallVector.h:1162
llvm::passCopy
MachineInstr * passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1160
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::getSpirvLinkageTypeFor
std::optional< SPIRV::LinkageType::LinkageType > getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV)
Definition SPIRVUtils.cpp:1266
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::addressSpaceToStorageClass
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)
Definition SPIRVUtils.cpp:376
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::createSPIRVInstructionSelector
InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &Subtarget, const RegisterBankInfo &RBI)
Definition SPIRVInstructionSelector.cpp:7732
llvm::getStringValueFromReg
std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI)
Definition SPIRVUtils.cpp:183
llvm::foldImm
int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1182
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::getDefInstrMaybeConstant
MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:487
llvm::BitWidth
constexpr unsigned BitWidth
Definition BitmaskEnum.h:219
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::hasInitializer
bool hasInitializer(const GlobalVariable *GV)
Definition SPIRVUtils.h:364
llvm::isSpvIntrinsic
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
Definition SPIRVUtils.cpp:522
llvm::getVRegDef
MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)
Definition SPIRVUtils.cpp:927
llvm::getMemSemantics
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
Definition SPIRVUtils.cpp:434
llvm::getLinkStringForBuiltIn
std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)
Definition SPIRVBaseInfo.cpp:204
llvm::isTriviallyDead
LLVM_ABI bool isTriviallyDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Check whether an instruction MI is dead: it only defines dead virtual registers, and doesn't have oth...
Definition Utils.cpp:224
llvm::TargetRegisterClass
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
N
#define N
llvm::MemOp::isVolatile
bool isVolatile() const
Definition TargetLowering.h:187
Generated on for LLVM by doxygen 1.14.0