LLVM 24.0.0git
SPIRVInstructionSelector.cpp
Go to the documentation of this file.
1//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the InstructionSelector class for
10// SPIRV.
11// TODO: This should be generated by TableGen.
12//
13//===----------------------------------------------------------------------===//
14
17#include "SPIRV.h"
18#include "SPIRVGlobalRegistry.h"
19#include "SPIRVInstrInfo.h"
20#include "SPIRVRegisterInfo.h"
21#include "SPIRVTargetMachine.h"
22#include "SPIRVTypeInst.h"
23#include "SPIRVUtils.h"
24#include "llvm/ADT/APFloat.h"
25#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallSet.h"
37#include "llvm/IR/IntrinsicsSPIRV.h"
38#include "llvm/Support/Debug.h"
40#include <functional>
41#include <optional>
42
43#define DEBUG_TYPE "spirv-isel"
44
45using namespace llvm;
46namespace CL = SPIRV::OpenCLExtInst;
47namespace GL = SPIRV::GLSLExtInst;
48
50 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
51
52namespace {
53
54struct ImageOperands {
55 std::optional<Register> Bias;
56 std::optional<Register> Offset;
57 std::optional<Register> MinLod;
58 std::optional<Register> GradX;
59 std::optional<Register> GradY;
60 std::optional<Register> Lod;
61 std::optional<Register> Compare;
62};
63
64llvm::SPIRV::SelectionControl::SelectionControl
65getSelectionOperandForImm(int Imm) {
66 if (Imm == 2)
67 return SPIRV::SelectionControl::Flatten;
68 if (Imm == 1)
69 return SPIRV::SelectionControl::DontFlatten;
70 if (Imm == 0)
71 return SPIRV::SelectionControl::None;
72 llvm_unreachable("Invalid immediate");
73}
74
75#define GET_GLOBALISEL_PREDICATE_BITSET
76#include "SPIRVGenGlobalISel.inc"
77#undef GET_GLOBALISEL_PREDICATE_BITSET
78
79class SPIRVInstructionSelector : public InstructionSelector {
80 const SPIRVSubtarget &STI;
81 const SPIRVInstrInfo &TII;
83 const RegisterBankInfo &RBI;
86 MachineFunction *HasVRegsReset = nullptr;
87
88 /// We need to keep track of the number we give to anonymous global values to
89 /// generate the same name every time when this is needed.
90 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
92
93public:
94 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
95 const SPIRVSubtarget &ST,
96 const RegisterBankInfo &RBI);
97 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
98 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
99 BlockFrequencyInfo *BFI) override;
100 // Common selection code. Instruction-specific selection occurs in spvSelect.
101 bool select(MachineInstr &I) override;
102 static const char *getName() { return DEBUG_TYPE; }
103
104#define GET_GLOBALISEL_PREDICATES_DECL
105#include "SPIRVGenGlobalISel.inc"
106#undef GET_GLOBALISEL_PREDICATES_DECL
107
108#define GET_GLOBALISEL_TEMPORARIES_DECL
109#include "SPIRVGenGlobalISel.inc"
110#undef GET_GLOBALISEL_TEMPORARIES_DECL
111
112private:
113 void resetVRegsType(MachineFunction &MF);
114 void removeDeadInstruction(MachineInstr &MI) const;
115 void removeOpNamesForDeadMI(MachineInstr &MI) const;
116
117 // tblgen-erated 'select' implementation, used as the initial selector for
118 // the patterns that don't require complex C++.
119 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
120
121 // All instruction-specific selection that didn't happen in "select()".
122 // Is basically a large Switch/Case delegating to all other select method.
123 bool spvSelect(Register ResVReg, SPIRVTypeInst ResType,
124 MachineInstr &I) const;
125
126 bool selectFirstBitHigh(Register ResVReg, SPIRVTypeInst ResType,
127 MachineInstr &I, bool IsSigned) const;
128
129 bool selectFirstBitLow(Register ResVReg, SPIRVTypeInst ResType,
130 MachineInstr &I) const;
131
132 bool selectFirstBitSet16(Register ResVReg, SPIRVTypeInst ResType,
133 MachineInstr &I, unsigned ExtendOpcode,
134 unsigned BitSetOpcode) const;
135
136 bool selectFirstBitSet32(Register ResVReg, SPIRVTypeInst ResType,
137 MachineInstr &I, Register SrcReg,
138 unsigned BitSetOpcode) const;
139
140 bool selectFirstBitSet64(Register ResVReg, SPIRVTypeInst ResType,
141 MachineInstr &I, Register SrcReg,
142 unsigned BitSetOpcode, bool SwapPrimarySide) const;
143
144 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
145 const MachineInstr *Init = nullptr) const;
146
147 bool selectOpWithSrcs(Register ResVReg, SPIRVTypeInst ResType,
149 unsigned Opcode) const;
150
151 bool selectUnOp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
152 unsigned Opcode) const;
153
154 bool selectBitcast(Register ResVReg, SPIRVTypeInst ResType,
155 MachineInstr &I) const;
156
157 bool selectLoad(Register ResVReg, SPIRVTypeInst ResType,
158 MachineInstr &I) const;
159 bool selectAtomicLoad(Register ResVReg, SPIRVTypeInst ResType,
160 MachineInstr &I) const;
161 bool selectStore(MachineInstr &I) const;
162 bool selectAtomicStore(MachineInstr &I) const;
163
164 bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType,
165 MachineInstr &I) const;
166 bool selectStackRestore(MachineInstr &I) const;
167
168 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
169 Register getOrCreateMemSetGlobal(MachineInstr &I) const;
170 bool selectCopyMemory(MachineInstr &I, Register SrcReg) const;
171
172 bool selectAtomicRMW(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
173 unsigned NewOpcode, unsigned NegateOpcode = 0) const;
174
175 // Creates an integer-typed register with bitwidth equal to pointer size.
176 Register createPtrSizedIntReg(MachineIRBuilder &MIRBuilder) const;
177 // Emit an OpConvertPtrToU that converts the pointer value in \p PtrVal into
178 // an integer of equal bitwidth, returning the register holding the result.
179 Register convertPtrToInt(Register PtrVal, MachineIRBuilder &MIRBuilder) const;
180 // Emit an OpBitcast that reinterprets the pointer \p Ptr as a pointer to an
181 // integer of pointer size in storage class \p SC, returning the result.
182 Register castPtrToPtrToInt(Register Ptr, SPIRV::StorageClass::StorageClass SC,
183 MachineIRBuilder &MIRBuilder) const;
184 // Handle atomic loads, stores and exchanges of pointer types by casting
185 // to/from integer types as needed.
186 bool selectAtomicPtrValue(
187 Register ResVReg, SPIRVTypeInst ResType, MachineIRBuilder &MIRBuilder,
188 function_ref<Register(SPIRVTypeInst IntType)> EmitAtomic) const;
189
190 bool selectAtomicCmpXchg(Register ResVReg, SPIRVTypeInst ResType,
191 MachineInstr &I) const;
192
193 bool selectFence(MachineInstr &I) const;
194
195 bool selectAddrSpaceCast(Register ResVReg, SPIRVTypeInst ResType,
196 MachineInstr &I) const;
197
198 bool selectPtrMask(Register ResVReg, SPIRVTypeInst ResType,
199 MachineInstr &I) const;
200
201 bool selectAnyOrAll(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
202 unsigned OpType) const;
203
204 bool selectAll(Register ResVReg, SPIRVTypeInst ResType,
205 MachineInstr &I) const;
206
207 bool selectAny(Register ResVReg, SPIRVTypeInst ResType,
208 MachineInstr &I) const;
209
210 bool selectBitreverse(Register ResVReg, SPIRVTypeInst ResType,
211 MachineInstr &I) const;
212
213 bool selectBitreverseViaI32(Register ResVReg, SPIRVTypeInst ResType,
214 MachineInstr &I, Register Op) const;
215
216 bool selectBitreverse64(Register ResVReg, SPIRVTypeInst ResType,
217 MachineInstr &I, Register SrcReg) const;
218
219 bool selectBitreverseNative(Register ResVReg, SPIRVTypeInst ResType,
220 MachineInstr &I, Register Op) const;
221
222 bool selectBuildVector(Register ResVReg, SPIRVTypeInst ResType,
223 MachineInstr &I) const;
224 bool selectSplatVector(Register ResVReg, SPIRVTypeInst ResType,
225 MachineInstr &I) const;
226 bool selectConcatVectors(Register ResVReg, SPIRVTypeInst ResType,
227 MachineInstr &I) const;
228
229 bool selectCmp(Register ResVReg, SPIRVTypeInst ResType,
230 unsigned comparisonOpcode, MachineInstr &I) const;
231 bool selectDiscard(Register ResVReg, SPIRVTypeInst ResType,
232 MachineInstr &I) const;
233
234 bool selectICmp(Register ResVReg, SPIRVTypeInst ResType,
235 MachineInstr &I) const;
236 bool selectFCmp(Register ResVReg, SPIRVTypeInst ResType,
237 MachineInstr &I) const;
238
239 bool selectSign(Register ResVReg, SPIRVTypeInst ResType,
240 MachineInstr &I) const;
241
242 bool selectFloatDot(Register ResVReg, SPIRVTypeInst ResType,
243 MachineInstr &I) const;
244
245 bool selectOverflowArith(Register ResVReg, SPIRVTypeInst ResType,
246 MachineInstr &I, unsigned Opcode) const;
247 bool selectDebugTrap(Register ResVReg, SPIRVTypeInst ResType,
248 MachineInstr &I) const;
249 bool selectPrefetch(MachineInstr &I) const;
250
251 bool selectIntegerDot(Register ResVReg, SPIRVTypeInst ResType,
252 MachineInstr &I, bool Signed) const;
253
254 bool selectIntegerDotExpansion(Register ResVReg, SPIRVTypeInst ResType,
255 MachineInstr &I) const;
256
257 bool selectOpIsInf(Register ResVReg, SPIRVTypeInst ResType,
258 MachineInstr &I) const;
259
260 bool selectOpIsNan(Register ResVReg, SPIRVTypeInst ResType,
261 MachineInstr &I) const;
262
263 bool selectOpIsFinite(Register ResVReg, SPIRVTypeInst ResType,
264 MachineInstr &I) const;
265
266 bool selectOpIsNormal(Register ResVReg, SPIRVTypeInst ResType,
267 MachineInstr &I) const;
268
269 bool selectPopCount(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
270 unsigned Opcode) const;
271
272 bool selectPopCount16(Register ResVReg, SPIRVTypeInst ResType,
273 MachineInstr &I, unsigned ExtOpcode,
274 unsigned Opcode) const;
275
276 bool selectPopCount32(Register ResVReg, SPIRVTypeInst ResType,
277 MachineInstr &I, Register SrcReg,
278 unsigned Opcode) const;
279
280 bool selectPopCount64(Register ResVReg, SPIRVTypeInst ResType,
281 MachineInstr &I, Register SrcReg,
282 unsigned Opcode) const;
283
284 template <bool Signed>
285 bool selectDot4AddPacked(Register ResVReg, SPIRVTypeInst ResType,
286 MachineInstr &I) const;
287 template <bool Signed>
288 bool selectDot4AddPackedExpansion(Register ResVReg, SPIRVTypeInst ResType,
289 MachineInstr &I) const;
290
291 bool selectWavePrefixBitCount(Register ResVReg, SPIRVTypeInst ResType,
292 MachineInstr &I) const;
293
294 template <typename PickOpcodeFn>
295 bool selectWaveReduce(Register ResVReg, SPIRVTypeInst ResType,
296 MachineInstr &I, bool IsUnsigned,
297 PickOpcodeFn &&PickOpcode) const;
298
299 bool selectWaveReduceOp(Register ResVReg, SPIRVTypeInst ResType,
300 MachineInstr &I, unsigned Opcode) const;
301
302 bool selectWaveReduceMax(Register ResVReg, SPIRVTypeInst ResType,
303 MachineInstr &I, bool IsUnsigned) const;
304
305 bool selectWaveReduceMin(Register ResVReg, SPIRVTypeInst ResType,
306 MachineInstr &I, bool IsUnsigned) const;
307
308 bool selectWaveReduceSum(Register ResVReg, SPIRVTypeInst ResType,
309 MachineInstr &I) const;
310
311 bool selectWaveReduceProduct(Register ResVReg, const SPIRVTypeInst ResType,
312 MachineInstr &I) const;
313
314 template <typename PickOpcodeFn>
315 bool selectWaveExclusiveScan(Register ResVReg, SPIRVTypeInst ResType,
316 MachineInstr &I, bool IsUnsigned,
317 PickOpcodeFn &&PickOpcode) const;
318
319 bool selectWaveExclusiveScanSum(Register ResVReg, SPIRVTypeInst ResType,
320 MachineInstr &I) const;
321
322 bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType,
323 MachineInstr &I) const;
324
325 bool selectWaveMultiPrefixOp(Register ResVReg, SPIRVTypeInst ResType,
326 MachineInstr &I, unsigned Opcode,
327 StringRef BuiltinName) const;
328
329 bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
330 unsigned Direction) const;
331
332 bool selectQuadAll(Register ResVReg, SPIRVTypeInst ResType,
333 MachineInstr &I) const;
334
335 bool selectConst(Register ResVReg, SPIRVTypeInst ResType,
336 MachineInstr &I) const;
337
338 bool selectSelect(Register ResVReg, SPIRVTypeInst ResType,
339 MachineInstr &I) const;
340 bool selectBoolToInt(Register ResVReg, SPIRVTypeInst ResType,
341 Register BooleanVReg, MachineInstr &InsertAt,
342 bool IsSigned) const;
343 bool selectIToF(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
344 bool IsSigned, unsigned Opcode) const;
345 bool selectExt(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
346 bool IsSigned) const;
347
348 bool selectTrunc(Register ResVReg, SPIRVTypeInst ResType,
349 MachineInstr &I) const;
350
351 bool selectSUCmp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
352 bool IsSigned) const;
353
354 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
355 SPIRVTypeInst intTy, SPIRVTypeInst boolTy) const;
356
357 bool selectOpUndef(Register ResVReg, SPIRVTypeInst ResType,
358 MachineInstr &I) const;
359 bool selectFreeze(Register ResVReg, SPIRVTypeInst ResType,
360 MachineInstr &I) const;
361 bool selectIntrinsic(Register ResVReg, SPIRVTypeInst ResType,
362 MachineInstr &I) const;
363 bool selectExtractVal(Register ResVReg, SPIRVTypeInst ResType,
364 MachineInstr &I) const;
365 bool selectInsertVal(Register ResVReg, SPIRVTypeInst ResType,
366 MachineInstr &I) const;
367 bool selectExtractElt(Register ResVReg, SPIRVTypeInst ResType,
368 MachineInstr &I) const;
369 bool selectInsertElt(Register ResVReg, SPIRVTypeInst ResType,
370 MachineInstr &I) const;
371 bool selectGEP(Register ResVReg, SPIRVTypeInst ResType,
372 MachineInstr &I) const;
373
374 bool selectMaskedGather(Register ResVReg, SPIRVTypeInst ResType,
375 MachineInstr &I) const;
376 bool selectMaskedScatter(MachineInstr &I) const;
377
378 bool diagnoseUnsupported(const MachineInstr &I, const Twine &Msg) const;
379
380 bool selectAbort(MachineInstr &I) const;
381 bool selectTrap(MachineInstr &I) const;
382 bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType,
383 MachineInstr &I) const;
384 bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType,
385 MachineInstr &I) const;
386
387 bool selectBranch(MachineInstr &I) const;
388 bool selectBranchCond(MachineInstr &I) const;
389
390 bool selectPhi(Register ResVReg, MachineInstr &I) const;
391
392 bool selectExtInst(Register ResVReg, SPIRVTypeInst RestType, MachineInstr &I,
393 GL::GLSLExtInst GLInst, bool setMIFlags = true,
394 bool useMISrc = true,
395 ArrayRef<Register> SrcRegs = {}) const;
396 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
397 CL::OpenCLExtInst CLInst, bool setMIFlags = true,
398 bool useMISrc = true,
399 ArrayRef<Register> SrcRegs = {}) const;
400 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
401 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst,
402 bool setMIFlags = true, bool useMISrc = true,
403 ArrayRef<Register> SrcRegs = {}) const;
404 bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
405 const ExtInstList &ExtInsts, bool setMIFlags = true,
406 bool useMISrc = true,
407 ArrayRef<Register> SrcRegs = {}) const;
408
409 bool selectLog10(Register ResVReg, SPIRVTypeInst ResType,
410 MachineInstr &I) const;
411
412 bool selectFpowi(Register ResVReg, SPIRVTypeInst ResType,
413 MachineInstr &I) const;
414
415 bool selectSaturate(Register ResVReg, SPIRVTypeInst ResType,
416 MachineInstr &I) const;
417
418 bool selectWaveOpInst(Register ResVReg, SPIRVTypeInst ResType,
419 MachineInstr &I, unsigned Opcode) const;
420
421 bool selectBarrierInst(MachineInstr &I, unsigned Scope, unsigned MemSem,
422 bool WithGroupSync) const;
423
424 bool selectWaveActiveCountBits(Register ResVReg, SPIRVTypeInst ResType,
425 MachineInstr &I) const;
426
427 bool selectWaveActiveAllEqual(Register ResVReg, SPIRVTypeInst ResType,
428 MachineInstr &I) const;
429
430 bool selectUnmergeValues(MachineInstr &I) const;
431
432 bool selectHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
433 MachineInstr &I) const;
434
435 bool selectCounterHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType,
436 MachineInstr &I) const;
437
438 bool selectReadImageIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
439 MachineInstr &I) const;
440 bool selectGetDimensionsIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
441 MachineInstr &I) const;
442 bool selectGetDimensionsLevelsIntrinsic(Register &ResVReg,
443 SPIRVTypeInst ResType,
444 MachineInstr &I) const;
445 bool selectGetDimensionsMSIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
446 MachineInstr &I) const;
447 bool
448 selectImageQuerySize(Register ImageReg, Register &ResVReg, MachineInstr &I,
449 std::optional<Register> LodReg = std::nullopt) const;
450 bool selectSampleBasicIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
451 MachineInstr &I) const;
452 bool selectCalculateLodIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
453 MachineInstr &I) const;
454 bool selectSampleBiasIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
455 MachineInstr &I) const;
456 bool selectSampleGradIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
457 MachineInstr &I) const;
458 bool selectSampleLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
459 MachineInstr &I) const;
460 bool selectLoadLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
461 MachineInstr &I) const;
462 bool selectSampleCmpIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
463 MachineInstr &I) const;
464 bool selectSampleCmpLevelZeroIntrinsic(Register &ResVReg,
465 SPIRVTypeInst ResType,
466 MachineInstr &I) const;
467 bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
468 MachineInstr &I) const;
469 bool selectImageWriteIntrinsic(MachineInstr &I) const;
470 bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
471 MachineInstr &I) const;
472 bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
473 MachineInstr &I) const;
474 bool selectResourceNonUniformIndex(Register &ResVReg, SPIRVTypeInst ResType,
475 MachineInstr &I) const;
476 bool selectModf(Register ResVReg, SPIRVTypeInst ResType,
477 MachineInstr &I) const;
478 bool selectUpdateCounter(Register &ResVReg, SPIRVTypeInst ResType,
479 MachineInstr &I) const;
480 bool selectFrexp(Register ResVReg, SPIRVTypeInst ResType,
481 MachineInstr &I) const;
482
483 bool selectCopySign(Register ResVReg, SPIRVTypeInst ResType,
484 MachineInstr &I) const;
485
486 bool selectLdexp(Register ResVReg, SPIRVTypeInst ResType,
487 MachineInstr &I) const;
488 bool selectSincos(Register ResVReg, SPIRVTypeInst ResType,
489 MachineInstr &I) const;
490 bool selectExp10(Register ResVReg, SPIRVTypeInst ResType,
491 MachineInstr &I) const;
492 bool selectDerivativeInst(Register ResVReg, SPIRVTypeInst ResType,
493 MachineInstr &I, const unsigned DPdOpCode) const;
494 // Utilities
495 Register buildI32ConstantInEntryBlock(uint32_t Val, MachineInstr &I,
496 SPIRVTypeInst ResType = nullptr) const;
497
498 Register buildZerosVal(SPIRVTypeInst ResType, MachineInstr &I) const;
499 bool isScalarOrVectorIntConstantZero(Register Reg) const;
500 Register buildZerosValF(SPIRVTypeInst ResType, MachineInstr &I) const;
501 Register buildOnesVal(bool AllOnes, SPIRVTypeInst ResType,
502 MachineInstr &I) const;
503 Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &I) const;
504 Register buildVectorSplat(Register ScalarReg, unsigned NumElts,
505 MachineInstr &I) const;
506
507 bool wrapIntoSpecConstantOp(MachineInstr &I,
508 SmallVector<Register> &CompositeArgs) const;
509
510 SPIRVTypeInst getUcharPtrType(MachineInstr &I,
511 SPIRV::StorageClass::StorageClass SC) const;
512 Register getUcharPtrTypeReg(MachineInstr &I,
513 SPIRV::StorageClass::StorageClass SC) const;
514 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest,
515 Register Src, Register DestType,
516 uint32_t Opcode) const;
517 Register bitcastConstPtrToI8(MachineInstr &I, Register Src) const;
518 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
519 SPIRVTypeInst SrcPtrTy) const;
520 Register buildPointerToResource(SPIRVTypeInst ResType,
521 SPIRV::StorageClass::StorageClass SC,
522 uint32_t Set, uint32_t Binding,
523 uint32_t ArraySize, Register IndexReg,
524 StringRef Name,
525 MachineIRBuilder MIRBuilder) const;
526 SPIRVTypeInst widenTypeToVec4(SPIRVTypeInst Type, MachineInstr &I) const;
527 bool extractSubvector(Register &ResVReg, SPIRVTypeInst ResType,
528 Register &ReadReg, MachineInstr &InsertionPoint) const;
529 bool generateImageReadOrFetch(Register &ResVReg, SPIRVTypeInst ResType,
530 Register ImageReg, Register IdxReg,
531 DebugLoc Loc, MachineInstr &Pos,
532 const ImageOperands *ImOps = nullptr) const;
533 bool generateSampleImage(Register ResVReg, SPIRVTypeInst ResType,
534 Register ImageReg, Register SamplerReg,
535 Register CoordinateReg, const ImageOperands &ImOps,
536 DebugLoc Loc, MachineInstr &I) const;
537 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const;
538 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
539 Register ResVReg, SPIRVTypeInst ResType,
540 MachineInstr &I) const;
541 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
542 Register ResVReg, SPIRVTypeInst ResType,
543 MachineInstr &I) const;
544 bool loadHandleBeforePosition(Register &HandleReg, SPIRVTypeInst ResType,
545 GIntrinsic &HandleDef, MachineInstr &Pos) const;
546 void decorateUsesAsNonUniform(Register &NonUniformReg) const;
547 bool errorIfInstrOutsideShader(MachineInstr &I) const;
548
549 bool
550 handle64BitOverflow(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
551 Register SrcReg, unsigned int Opcode,
552 std::function<bool(Register, SPIRVTypeInst,
553 MachineInstr &, Register, unsigned)>
554 CallbackFunction) const;
555};
556
557bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
558 const TargetExtType *TET = cast<TargetExtType>(HandleType);
559 if (TET->getTargetExtName() == "spirv.Image") {
560 return false;
561 }
562 assert(TET->getTargetExtName() == "spirv.SignedImage");
563 return TET->getTypeParameter(0)->isIntegerTy();
564}
565} // end anonymous namespace
566
567#define GET_GLOBALISEL_IMPL
568#include "SPIRVGenGlobalISel.inc"
569#undef GET_GLOBALISEL_IMPL
570
571SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
572 const SPIRVSubtarget &ST,
573 const RegisterBankInfo &RBI)
574 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
575 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
576 MRI(nullptr),
578#include "SPIRVGenGlobalISel.inc"
581#include "SPIRVGenGlobalISel.inc"
583{
584}
585
586void SPIRVInstructionSelector::setupMF(MachineFunction &MF,
588 CodeGenCoverage *CoverageInfo,
590 BlockFrequencyInfo *BFI) {
591 MRI = &MF.getRegInfo();
592 GR.setCurrentFunc(MF);
593 InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
594}
595
596// Ensure that register classes correspond to pattern matching rules.
597void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) {
598 if (HasVRegsReset == &MF)
599 return;
600 HasVRegsReset = &MF;
601
602 MachineRegisterInfo &MRI = MF.getRegInfo();
603 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
604 Register Reg = Register::index2VirtReg(I);
605 LLT RegType = MRI.getType(Reg);
606 LLT ScalarTy =
607 RegType.isFloatOrFloatVector() ? LLT::float64() : LLT::integer(64);
608 if (RegType.isScalar())
609 MRI.setType(Reg, ScalarTy);
610 else if (RegType.isPointer())
611 MRI.setType(Reg, LLT::pointer(0, 64));
612 else if (RegType.isVector())
613 MRI.setType(Reg, LLT::fixed_vector(2, ScalarTy));
614 }
615 for (const auto &MBB : MF) {
616 for (const auto &MI : MBB) {
617 if (isPreISelGenericOpcode(MI.getOpcode()))
618 GR.erase(&MI);
619 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
620 continue;
621
622 Register DstReg = MI.getOperand(0).getReg();
623 LLT DstType = MRI.getType(DstReg);
624 Register SrcReg = MI.getOperand(1).getReg();
625 LLT SrcType = MRI.getType(SrcReg);
626 if (DstType != SrcType)
627 MRI.setType(DstReg, MRI.getType(SrcReg));
628
629 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg);
630 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg);
631 if (DstRC != SrcRC && SrcRC)
632 MRI.setRegClass(DstReg, SrcRC);
633 }
634 }
635}
636
637// Return true if the MachineInstr represents a constant register
639
640 SmallVector<MachineInstr *> Stack = {OpDef};
642
643 while (!Stack.empty()) {
644 MachineInstr *MI = Stack.pop_back_val();
645 MI = passCopy(MI, MRI);
646 if (!Visited.insert(MI).second)
647 continue;
648 switch (MI->getOpcode()) {
649 case TargetOpcode::G_INTRINSIC:
650 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
651 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
653 unsigned IntrID = GIntr->getIntrinsicID();
654 if (IntrID != Intrinsic::spv_const_composite &&
655 IntrID != Intrinsic::spv_undef && IntrID != Intrinsic::spv_poison)
656 return false;
657 continue;
658 }
659 case TargetOpcode::G_BUILD_VECTOR:
660 case TargetOpcode::G_SPLAT_VECTOR:
661 for (unsigned i = OpDef->getNumExplicitDefs();
662 i < OpDef->getNumOperands(); i++) {
663 if (!OpDef->getOperand(i).isReg())
664 continue;
665 MachineInstr *OpNestedDef =
666 MRI->getVRegDef(OpDef->getOperand(i).getReg());
667 Stack.push_back(OpNestedDef);
668 }
669 continue;
670 case TargetOpcode::G_CONSTANT:
671 case TargetOpcode::G_FCONSTANT:
672 case TargetOpcode::G_IMPLICIT_DEF:
673 case SPIRV::OpConstantTrue:
674 case SPIRV::OpConstantFalse:
675 case SPIRV::OpConstantI:
676 case SPIRV::OpConstantF:
677 case SPIRV::OpConstantComposite:
678 case SPIRV::OpConstantCompositeContinuedINTEL:
679 case SPIRV::OpConstantSampler:
680 case SPIRV::OpConstantNull:
681 case SPIRV::OpUndef:
682 case SPIRV::OpPoisonKHR:
683 case SPIRV::OpConstantFunctionPointerINTEL:
684 continue;
685 default:
686 return false;
687 }
688 }
689 return true;
690}
691
692// Return true if the virtual register represents a constant
694 if (MachineInstr *OpDef = MRI->getVRegDef(OpReg))
695 return isConstReg(MRI, OpDef);
696 return false;
697}
698
699// TODO(168736): We should make this either a flag in tabelgen
700// or reduce our dependence on the global registry, so we can remove this
701// function. It can easily be missed when new intrinsics are added.
702
703// Most SPIR-V intrinsics are considered to have side-effects in their tablegen
704// definition because they are referenced in the global registry. This is a list
705// of intrinsics that have no side effects other than their references in the
706// global registry.
708 switch (ID) {
709 // This is not an exhaustive list and may need to be updated.
710 case Intrinsic::spv_all:
711 case Intrinsic::spv_alloca:
712 case Intrinsic::spv_any:
713 case Intrinsic::spv_bitcast:
714 case Intrinsic::spv_const_composite:
715 case Intrinsic::spv_degrees:
716 case Intrinsic::spv_distance:
717 case Intrinsic::spv_extractelt:
718 case Intrinsic::spv_extractv:
719 case Intrinsic::spv_faceforward:
720 case Intrinsic::spv_fdot:
721 case Intrinsic::spv_firstbitlow:
722 case Intrinsic::spv_firstbitshigh:
723 case Intrinsic::spv_firstbituhigh:
724 case Intrinsic::spv_frac:
725 case Intrinsic::spv_gep:
726 case Intrinsic::spv_global_offset:
727 case Intrinsic::spv_global_size:
728 case Intrinsic::spv_group_id:
729 case Intrinsic::spv_insertelt:
730 case Intrinsic::spv_insertv:
731 case Intrinsic::spv_isinf:
732 case Intrinsic::spv_isnan:
733 case Intrinsic::spv_isfinite:
734 case Intrinsic::spv_isnormal:
735 case Intrinsic::spv_lerp:
736 case Intrinsic::spv_length:
737 case Intrinsic::spv_normalize:
738 case Intrinsic::spv_num_subgroups:
739 case Intrinsic::spv_num_workgroups:
740 case Intrinsic::spv_ptrcast:
741 case Intrinsic::spv_radians:
742 case Intrinsic::spv_reflect:
743 case Intrinsic::spv_refract:
744 case Intrinsic::spv_resource_getbasepointer:
745 case Intrinsic::spv_resource_getpointer:
746 case Intrinsic::spv_resource_handlefrombinding:
747 case Intrinsic::spv_resource_handlefromimplicitbinding:
748 case Intrinsic::spv_resource_nonuniformindex:
749 case Intrinsic::spv_resource_sample:
750 case Intrinsic::spv_rsqrt:
751 case Intrinsic::spv_saturate:
752 case Intrinsic::spv_sdot:
753 case Intrinsic::spv_sign:
754 case Intrinsic::spv_smoothstep:
755 case Intrinsic::spv_subgroup_id:
756 case Intrinsic::spv_subgroup_local_invocation_id:
757 case Intrinsic::spv_subgroup_max_size:
758 case Intrinsic::spv_subgroup_size:
759 case Intrinsic::spv_thread_id:
760 case Intrinsic::spv_thread_id_in_group:
761 case Intrinsic::spv_udot:
762 case Intrinsic::spv_undef:
763 case Intrinsic::spv_value_md:
764 case Intrinsic::spv_workgroup_size:
765 return false;
766 default:
767 return true;
768 }
769}
770
771// TODO(168736): We should make this either a flag in tabelgen
772// or reduce our dependence on the global registry, so we can remove this
773// function. It can easily be missed when new intrinsics are added.
774static bool isOpcodeWithNoSideEffects(unsigned Opcode) {
775 switch (Opcode) {
776 case SPIRV::OpTypeVoid:
777 case SPIRV::OpTypeBool:
778 case SPIRV::OpTypeInt:
779 case SPIRV::OpTypeFloat:
780 case SPIRV::OpTypeVector:
781 case SPIRV::OpTypeVectorIdEXT:
782 case SPIRV::OpTypeMatrix:
783 case SPIRV::OpTypeImage:
784 case SPIRV::OpTypeSampler:
785 case SPIRV::OpTypeSampledImage:
786 case SPIRV::OpTypeArray:
787 case SPIRV::OpTypeRuntimeArray:
788 case SPIRV::OpTypeStruct:
789 case SPIRV::OpTypeOpaque:
790 case SPIRV::OpTypePointer:
791 case SPIRV::OpTypeFunction:
792 case SPIRV::OpTypeEvent:
793 case SPIRV::OpTypeDeviceEvent:
794 case SPIRV::OpTypeReserveId:
795 case SPIRV::OpTypeQueue:
796 case SPIRV::OpTypePipe:
797 case SPIRV::OpTypeForwardPointer:
798 case SPIRV::OpTypePipeStorage:
799 case SPIRV::OpTypeNamedBarrier:
800 case SPIRV::OpTypeAccelerationStructureNV:
801 case SPIRV::OpTypeCooperativeMatrixNV:
802 case SPIRV::OpTypeCooperativeMatrixKHR:
803 return true;
804 default:
805 return false;
806 }
807}
808
810 // If there are no definitions, then assume there is some other
811 // side-effect that makes this instruction live.
812 if (MI.getNumDefs() == 0)
813 return false;
814
815 for (const auto &MO : MI.all_defs()) {
816 Register Reg = MO.getReg();
817 if (Reg.isPhysical()) {
818 LLVM_DEBUG(dbgs() << "Not dead: def of physical register " << Reg);
819 return false;
820 }
821 for (const auto &UseMI : MRI.use_nodbg_instructions(Reg)) {
822 if (UseMI.getOpcode() != SPIRV::OpName) {
823 LLVM_DEBUG(dbgs() << "Not dead: def " << MO << " has use in " << UseMI);
824 return false;
825 }
826 }
827 }
828
829 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() ||
830 MI.isLifetimeMarker()) {
832 dbgs()
833 << "Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n");
834 return false;
835 }
836 if (MI.isPHI()) {
837 LLVM_DEBUG(dbgs() << "Dead: Phi instruction with no uses.\n");
838 return true;
839 }
840
841 // It is possible that the only side effect is that the instruction is
842 // referenced in the global registry. If that is the only side effect, the
843 // intrinsic is dead.
844 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
845 MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
846 const auto &Intr = cast<GIntrinsic>(MI);
847 if (!intrinsicHasSideEffects(Intr.getIntrinsicID())) {
848 LLVM_DEBUG(dbgs() << "Dead: Intrinsic with no real side effects.\n");
849 return true;
850 }
851 }
852
853 if (MI.mayStore() || MI.isCall() ||
854 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() ||
855 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) {
856 LLVM_DEBUG(dbgs() << "Not dead: instruction has side effects.\n");
857 return false;
858 }
859
860 if (isPreISelGenericOpcode(MI.getOpcode())) {
861 // TODO: Is there a generic way to check if the opcode has side effects?
862 LLVM_DEBUG(dbgs() << "Dead: Generic opcode with no uses.\n");
863 return true;
864 }
865
866 if (isOpcodeWithNoSideEffects(MI.getOpcode())) {
867 LLVM_DEBUG(dbgs() << "Dead: known opcode with no side effects\n");
868 return true;
869 }
870
871 return false;
872}
873
874void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &MI) const {
875 // Delete the OpName that uses the result if there is one.
876 for (const auto &MO : MI.all_defs()) {
877 Register Reg = MO.getReg();
878 if (Reg.isPhysical())
879 continue;
880 SmallVector<MachineInstr *, 4> UselessOpNames;
881 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
882 assert(UseMI.getOpcode() == SPIRV::OpName &&
883 "There is still a use of the dead function.");
884 UselessOpNames.push_back(&UseMI);
885 }
886 for (MachineInstr *OpNameMI : UselessOpNames) {
887 GR.invalidateMachineInstr(OpNameMI);
888 OpNameMI->eraseFromParent();
889 }
890 }
891}
892
893void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &MI) const {
896 removeOpNamesForDeadMI(MI);
897 MI.eraseFromParent();
898}
899
900bool SPIRVInstructionSelector::select(MachineInstr &I) {
901 resetVRegsType(*I.getParent()->getParent());
902
903 assert(I.getParent() && "Instruction should be in a basic block!");
904 assert(I.getParent()->getParent() && "Instruction should be in a function!");
905
906 LLVM_DEBUG(dbgs() << "Checking if instruction is dead: " << I;);
907 if (isDead(I, *MRI)) {
908 LLVM_DEBUG(dbgs() << "Instruction is dead.\n");
909 removeDeadInstruction(I);
910 return true;
911 }
912
913 Register Opcode = I.getOpcode();
914 // If it's not a GMIR instruction, we've selected it already.
915 if (!isPreISelGenericOpcode(Opcode)) {
916 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
917 Register DstReg = I.getOperand(0).getReg();
918 Register SrcReg = I.getOperand(1).getReg();
919 auto *Def = MRI->getVRegDef(SrcReg);
920 if (isTypeFoldingSupported(Def->getOpcode()) &&
921 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
922 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
923 if (Def->getOpcode() == TargetOpcode::G_SELECT) {
924 Register SelectDstReg = Def->getOperand(0).getReg();
925 bool SuccessToSelectSelect [[maybe_unused]] = selectSelect(
926 SelectDstReg, GR.getSPIRVTypeForVReg(SelectDstReg), *Def);
927 assert(SuccessToSelectSelect);
929 Def->eraseFromParent();
930 MRI->replaceRegWith(DstReg, SelectDstReg);
932 I.eraseFromParent();
933 return true;
934 }
935
936 bool Res = selectImpl(I, *CoverageInfo);
937 LLVM_DEBUG({
938 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
939 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
940 I.print(dbgs());
941 }
942 });
943 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
944 if (Res) {
945 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI))
946 DeadMIs.insert(Def);
947 return Res;
948 }
949 }
950 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg));
951 MRI->replaceRegWith(SrcReg, DstReg);
953 I.eraseFromParent();
954 return true;
955 } else if (I.getNumDefs() == 1) {
956 // Make all vregs 64 bits (for SPIR-V IDs).
957 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64));
958 }
960 return true;
961 }
962
963 if (DeadMIs.contains(&I)) {
964 // if the instruction has been already made dead by folding it away
965 // erase it
966 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n");
967 removeDeadInstruction(I);
968 DeadMIs.erase(&I);
969 return true;
970 }
971
972 if (I.getNumOperands() != I.getNumExplicitOperands()) {
973 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
974 return false;
975 }
976
977 // Common code for getting return reg+type, and removing selected instr
978 // from parent occurs here. Instr-specific selection happens in spvSelect().
979 bool HasDefs = I.getNumDefs() > 0;
980 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
981 SPIRVTypeInst ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
982 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
983 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
984 if (spvSelect(ResVReg, ResType, I)) {
985 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs).
986 for (unsigned i = 0; i < I.getNumDefs(); ++i)
987 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64));
989 I.eraseFromParent();
990 return true;
991 }
992 return false;
993}
994
995static bool mayApplyGenericSelection(unsigned Opcode) {
996 switch (Opcode) {
997 case TargetOpcode::G_CONSTANT:
998 case TargetOpcode::G_FCONSTANT:
999 return false;
1000 }
1001 return isTypeFoldingSupported(Opcode);
1002}
1003
1004bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg,
1005 MachineInstr &I) const {
1006 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg);
1007 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg);
1008 if (DstRC != SrcRC && SrcRC)
1009 MRI->setRegClass(DestReg, SrcRC);
1010 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1011 .addDef(DestReg)
1012 .addUse(SrcReg)
1013 .constrainAllUses(TII, TRI, RBI);
1014 return true;
1015}
1016
1017bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
1018 SPIRVTypeInst ResType,
1019 MachineInstr &I) const {
1020 const unsigned Opcode = I.getOpcode();
1021 if (mayApplyGenericSelection(Opcode))
1022 return selectImpl(I, *CoverageInfo);
1023 switch (Opcode) {
1024 case TargetOpcode::G_CONSTANT:
1025 case TargetOpcode::G_FCONSTANT:
1026 return selectConst(ResVReg, ResType, I);
1027 case TargetOpcode::G_GLOBAL_VALUE:
1028 return selectGlobalValue(ResVReg, I);
1029 case TargetOpcode::G_IMPLICIT_DEF:
1030 return selectOpUndef(ResVReg, ResType, I);
1031 case TargetOpcode::G_FREEZE:
1032 return selectFreeze(ResVReg, ResType, I);
1033
1034 case TargetOpcode::G_INTRINSIC:
1035 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1036 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1037 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
1038 return selectIntrinsic(ResVReg, ResType, I);
1039 case TargetOpcode::G_BITREVERSE:
1040 return selectBitreverse(ResVReg, ResType, I);
1041
1042 case TargetOpcode::G_BUILD_VECTOR:
1043 return selectBuildVector(ResVReg, ResType, I);
1044 case TargetOpcode::G_SPLAT_VECTOR:
1045 return selectSplatVector(ResVReg, ResType, I);
1046 case TargetOpcode::G_CONCAT_VECTORS:
1047 return selectConcatVectors(ResVReg, ResType, I);
1048
1049 case TargetOpcode::G_SHUFFLE_VECTOR: {
1050 MachineBasicBlock &BB = *I.getParent();
1051 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
1052 .addDef(ResVReg)
1053 .addUse(GR.getSPIRVTypeID(ResType))
1054 .addUse(I.getOperand(1).getReg())
1055 .addUse(I.getOperand(2).getReg());
1056 for (auto V : I.getOperand(3).getShuffleMask())
1057 MIB.addImm(V);
1058 MIB.constrainAllUses(TII, TRI, RBI);
1059 return true;
1060 }
1061 case TargetOpcode::G_MEMMOVE:
1062 case TargetOpcode::G_MEMCPY:
1063 case TargetOpcode::G_MEMCPY_INLINE:
1064 case TargetOpcode::G_MEMSET:
1065 case TargetOpcode::G_MEMSET_INLINE:
1066 return selectMemOperation(ResVReg, I);
1067
1068 case TargetOpcode::G_ICMP:
1069 return selectICmp(ResVReg, ResType, I);
1070 case TargetOpcode::G_FCMP:
1071 return selectFCmp(ResVReg, ResType, I);
1072
1073 case TargetOpcode::G_FRAME_INDEX:
1074 return selectFrameIndex(ResVReg, ResType, I);
1075
1076 case TargetOpcode::G_LOAD:
1077 return selectLoad(ResVReg, ResType, I);
1078 case TargetOpcode::G_STORE:
1079 return selectStore(I);
1080
1081 case TargetOpcode::G_BR:
1082 return selectBranch(I);
1083 case TargetOpcode::G_BRCOND:
1084 return selectBranchCond(I);
1085
1086 case TargetOpcode::G_PHI:
1087 return selectPhi(ResVReg, I);
1088
1089 case TargetOpcode::G_FPTOSI:
1090 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
1091 case TargetOpcode::G_FPTOUI:
1092 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
1093
1094 case TargetOpcode::G_FPTOSI_SAT:
1095 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
1096 case TargetOpcode::G_FPTOUI_SAT:
1097 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
1098
1099 case TargetOpcode::G_SITOFP:
1100 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
1101 case TargetOpcode::G_UITOFP:
1102 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
1103
1104 case TargetOpcode::G_CTPOP:
1105 return selectPopCount(ResVReg, ResType, I, SPIRV::OpBitCount);
1106 case TargetOpcode::G_SMIN:
1107 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
1108 case TargetOpcode::G_UMIN:
1109 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
1110
1111 case TargetOpcode::G_SMAX:
1112 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
1113 case TargetOpcode::G_UMAX:
1114 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
1115
1116 case TargetOpcode::G_SCMP:
1117 return selectSUCmp(ResVReg, ResType, I, true);
1118 case TargetOpcode::G_UCMP:
1119 return selectSUCmp(ResVReg, ResType, I, false);
1120 case TargetOpcode::G_LROUND:
1121 case TargetOpcode::G_LLROUND: {
1122 Register regForLround =
1123 MRI->createVirtualRegister(MRI->getRegClass(ResVReg), "lround");
1124 MRI->setRegClass(regForLround, &SPIRV::iIDRegClass);
1125 GR.assignSPIRVTypeToVReg(GR.getSPIRVTypeForVReg(I.getOperand(1).getReg()),
1126 regForLround, *(I.getParent()->getParent()));
1127 selectExtInst(regForLround, GR.getSPIRVTypeForVReg(regForLround), I,
1128 CL::round, GL::Round, /* setMIFlags */ false);
1129 MachineBasicBlock &BB = *I.getParent();
1130 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConvertFToS))
1131 .addDef(ResVReg)
1132 .addUse(GR.getSPIRVTypeID(ResType))
1133 .addUse(regForLround);
1134 MIB.constrainAllUses(TII, TRI, RBI);
1135 return true;
1136 }
1137 case TargetOpcode::G_STRICT_FMA:
1138 case TargetOpcode::G_FMA: {
1139 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_fma)) {
1140 MachineBasicBlock &BB = *I.getParent();
1141 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpFmaKHR))
1142 .addDef(ResVReg)
1143 .addUse(GR.getSPIRVTypeID(ResType))
1144 .addUse(I.getOperand(1).getReg())
1145 .addUse(I.getOperand(2).getReg())
1146 .addUse(I.getOperand(3).getReg())
1147 .setMIFlags(I.getFlags());
1148 MIB.constrainAllUses(TII, TRI, RBI);
1149 return true;
1150 }
1151 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
1152 }
1153
1154 case TargetOpcode::G_FLDEXP:
1155 case TargetOpcode::G_STRICT_FLDEXP:
1156 return selectLdexp(ResVReg, ResType, I);
1157
1158 case TargetOpcode::G_FPOW:
1159 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
1160 case TargetOpcode::G_FPOWI:
1161 return selectFpowi(ResVReg, ResType, I);
1162
1163 case TargetOpcode::G_FEXP:
1164 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
1165 case TargetOpcode::G_FEXP2:
1166 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
1167 case TargetOpcode::G_FEXP10:
1168 return selectExp10(ResVReg, ResType, I);
1169
1170 case TargetOpcode::G_FMODF:
1171 return selectModf(ResVReg, ResType, I);
1172 case TargetOpcode::G_FSINCOS:
1173 return selectSincos(ResVReg, ResType, I);
1174
1175 case TargetOpcode::G_FLOG:
1176 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
1177 case TargetOpcode::G_FLOG2:
1178 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
1179 case TargetOpcode::G_FLOG10:
1180 return selectLog10(ResVReg, ResType, I);
1181
1182 case TargetOpcode::G_FABS:
1183 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
1184 case TargetOpcode::G_ABS:
1185 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
1186
1187 case TargetOpcode::G_FMINNUM:
1188 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin);
1189 case TargetOpcode::G_FMAXNUM:
1190 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax);
1191
1192 case TargetOpcode::G_FCOPYSIGN:
1193 return selectCopySign(ResVReg, ResType, I);
1194
1195 case TargetOpcode::G_FCEIL:
1196 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
1197 case TargetOpcode::G_FFLOOR:
1198 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
1199
1200 case TargetOpcode::G_FCOS:
1201 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
1202 case TargetOpcode::G_FSIN:
1203 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
1204 case TargetOpcode::G_FTAN:
1205 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan);
1206 case TargetOpcode::G_FACOS:
1207 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos);
1208 case TargetOpcode::G_FASIN:
1209 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin);
1210 case TargetOpcode::G_FATAN:
1211 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan);
1212 case TargetOpcode::G_FATAN2:
1213 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2);
1214 case TargetOpcode::G_FCOSH:
1215 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh);
1216 case TargetOpcode::G_FSINH:
1217 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh);
1218 case TargetOpcode::G_FTANH:
1219 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh);
1220
1221 case TargetOpcode::G_STRICT_FSQRT:
1222 case TargetOpcode::G_FSQRT:
1223 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
1224
1225 case TargetOpcode::G_CTTZ:
1226 case TargetOpcode::G_CTTZ_ZERO_POISON:
1227 return selectExtInst(ResVReg, ResType, I, CL::ctz);
1228 case TargetOpcode::G_CTLZ:
1229 case TargetOpcode::G_CTLZ_ZERO_POISON:
1230 return selectExtInst(ResVReg, ResType, I, CL::clz);
1231
1232 case TargetOpcode::G_INTRINSIC_ROUND:
1233 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
1234 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1235 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
1236 case TargetOpcode::G_INTRINSIC_TRUNC:
1237 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
1238 case TargetOpcode::G_FRINT:
1239 case TargetOpcode::G_FNEARBYINT:
1240 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
1241
1242 case TargetOpcode::G_SMULH:
1243 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
1244 case TargetOpcode::G_UMULH:
1245 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
1246
1247 case TargetOpcode::G_SADDSAT:
1248 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat);
1249 case TargetOpcode::G_UADDSAT:
1250 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat);
1251 case TargetOpcode::G_SSUBSAT:
1252 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat);
1253 case TargetOpcode::G_USUBSAT:
1254 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
1255
1256 case TargetOpcode::G_FFREXP:
1257 return selectFrexp(ResVReg, ResType, I);
1258
1259 case TargetOpcode::G_UADDO:
1260 return selectOverflowArith(ResVReg, ResType, I,
1261 isVectorType(ResType) ? SPIRV::OpIAddCarryV
1262 : SPIRV::OpIAddCarryS);
1263 case TargetOpcode::G_USUBO:
1264 return selectOverflowArith(ResVReg, ResType, I,
1265 isVectorType(ResType) ? SPIRV::OpISubBorrowV
1266 : SPIRV::OpISubBorrowS);
1267 case TargetOpcode::G_UMULO:
1268 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended);
1269 case TargetOpcode::G_SMULO:
1270 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended);
1271
1272 case TargetOpcode::G_SEXT:
1273 return selectExt(ResVReg, ResType, I, true);
1274 case TargetOpcode::G_ANYEXT:
1275 case TargetOpcode::G_ZEXT:
1276 return selectExt(ResVReg, ResType, I, false);
1277 case TargetOpcode::G_TRUNC:
1278 return selectTrunc(ResVReg, ResType, I);
1279 case TargetOpcode::G_FPTRUNC:
1280 case TargetOpcode::G_FPEXT:
1281 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
1282
1283 case TargetOpcode::G_PTRTOINT:
1284 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
1285 case TargetOpcode::G_INTTOPTR:
1286 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
1287 case TargetOpcode::G_BITCAST:
1288 return selectBitcast(ResVReg, ResType, I);
1289 case TargetOpcode::G_ADDRSPACE_CAST:
1290 return selectAddrSpaceCast(ResVReg, ResType, I);
1291 case TargetOpcode::G_PTRMASK:
1292 return selectPtrMask(ResVReg, ResType, I);
1293 case TargetOpcode::G_PTR_ADD: {
1294 // Currently, we get G_PTR_ADD only applied to global variables, possibly
1295 // through an address space cast.
1296 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
1297 Register GV = I.getOperand(1).getReg();
1299 const bool IsAddrSpaceCast =
1300 (*II).getOpcode() == TargetOpcode::G_ADDRSPACE_CAST;
1301 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1302 (*II).getOpcode() == TargetOpcode::COPY || IsAddrSpaceCast ||
1303 (*II).getOpcode() == SPIRV::OpVariable ||
1304 (*II).getOpcode() == SPIRV::OpUntypedVariableKHR) &&
1305 getImm(I.getOperand(2), MRI));
1306 // It may be the initialization of a global variable.
1307 bool IsGVInit = false;
1309 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()),
1310 UseEnd = MRI->use_instr_end();
1311 UseIt != UseEnd; UseIt = std::next(UseIt)) {
1312 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1313 (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
1314 (*UseIt).getOpcode() == SPIRV::OpVariable ||
1315 (*UseIt).getOpcode() == SPIRV::OpUntypedVariableKHR) {
1316 IsGVInit = true;
1317 break;
1318 }
1319 }
1320 // An address space cast of null can be typed as an integer, but the base of
1321 // an access chain must be a pointer. Other integer-typed casts cannot be
1322 // represented by replacing their base with a pointer constant.
1323 if (SPIRVTypeInst BaseType = GR.getSPIRVTypeForVReg(GV);
1324 IsAddrSpaceCast && (!BaseType || !BaseType.isPointer())) {
1325 if (!getImm((*II).getOperand(1), MRI) ||
1326 foldImm((*II).getOperand(1), MRI) != 0)
1327 return diagnoseUnsupported(
1328 I, "address space cast of a non-null integer used as a G_PTR_ADD "
1329 "base");
1330 SPIRVTypeInst NullPtrTy =
1331 getUcharPtrType(I, GR.getPointerStorageClass(ResType));
1332 MachineBasicBlock &DepMBB = I.getMF()->front();
1333 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
1334 GV = GR.getOrCreateConstNullPtr(MIRBuilder, NullPtrTy);
1335 }
1336 MachineBasicBlock &BB = *I.getParent();
1337 // An untyped result needs OpUntypedInBoundsPtrAccessChainKHR, which spells
1338 // out the Base Type. The offset of G_PTR_ADD is a byte count, so the Base
1339 // Type is i8 and the offset is the Element index as is. The opcode is
1340 // picked by the result type alone because an untyped access chain accepts
1341 // a typed base while a typed access chain rejects an untyped result.
1342 SPIRVTypeInst GVType = GR.getSPIRVTypeForVReg(GV);
1343 const bool UseUntypedPointers =
1344 ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1345 if (UseUntypedPointers) {
1346 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1347 .addDef(ResVReg)
1348 .addUse(GR.getSPIRVTypeID(ResType))
1349 .addImm(static_cast<uint32_t>(
1350 SPIRV::Opcode::UntypedInBoundsPtrAccessChainKHR))
1352 .addUse(GV)
1353 .addUse(I.getOperand(2).getReg())
1354 .constrainAllUses(TII, TRI, RBI);
1355 return true;
1356 }
1357 if (!IsGVInit) {
1358 SPIRVTypeInst GVPointeeType = GR.getPointeeType(GVType);
1359 SPIRVTypeInst ResPointeeType = GR.getPointeeType(ResType);
1360 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
1361 // Build a new virtual register that is associated with the required
1362 // data type.
1364 MRI->setRegClass(NewVReg, MRI->getRegClass(GV));
1365 // Having a correctly typed base we are ready to build the actually
1366 // required GEP. It may not be a constant though, because all Operands
1367 // of OpSpecConstantOp is to originate from other const instructions,
1368 // and only the AccessChain named opcodes accept a global OpVariable
1369 // instruction. We can't use an AccessChain opcode because of the type
1370 // mismatch between result and base types.
1371 if (!GR.isBitcastCompatible(ResType, GVType))
1372 return diagnoseUnsupported(
1373 I, "incompatible result and operand types in a bitcast");
1374 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
1375 MachineInstrBuilder MIB =
1376 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
1377 .addDef(NewVReg)
1378 .addUse(ResTypeReg)
1379 .addUse(GV);
1380 MIB.constrainAllUses(TII, TRI, RBI);
1381 BuildMI(BB, I, I.getDebugLoc(),
1382 TII.get(STI.isLogicalSPIRV() ? SPIRV::OpInBoundsAccessChain
1383 : SPIRV::OpInBoundsPtrAccessChain))
1384 .addDef(ResVReg)
1385 .addUse(ResTypeReg)
1386 .addUse(NewVReg)
1387 .addUse(I.getOperand(2).getReg())
1388 .constrainAllUses(TII, TRI, RBI);
1389 } else {
1390 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1391 .addDef(ResVReg)
1392 .addUse(GR.getSPIRVTypeID(ResType))
1393 .addImm(
1394 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
1395 .addUse(GV)
1396 .addUse(I.getOperand(2).getReg())
1397 .constrainAllUses(TII, TRI, RBI);
1398 }
1399 return true;
1400 }
1401 // The offset of G_PTR_ADD is a byte count, so bitcast a typed base to i8*
1402 // in its own storage class and use that offset as the Element operand.
1403 // selectAddrSpaceCast() later defines an address-space cast as that i8*
1404 // pointer, so no bitcast is emitted for a cast. An i8* base is unchanged.
1405 Register ChainBase = IsAddrSpaceCast ? GV : bitcastConstPtrToI8(I, GV);
1406
1407 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to
1408 // initialize a global variable with a constant expression (e.g., the test
1409 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case
1410 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1411 .addDef(ResVReg)
1412 .addUse(GR.getSPIRVTypeID(ResType))
1413 .addImm(static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain))
1414 .addUse(ChainBase)
1415 .addUse(I.getOperand(2).getReg())
1416 .constrainAllUses(TII, TRI, RBI);
1417 return true;
1418 }
1419
1420 case TargetOpcode::G_ATOMICRMW_OR:
1421 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
1422 case TargetOpcode::G_ATOMICRMW_ADD:
1423 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
1424 case TargetOpcode::G_ATOMICRMW_AND:
1425 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
1426 case TargetOpcode::G_ATOMICRMW_MAX:
1427 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
1428 case TargetOpcode::G_ATOMICRMW_MIN:
1429 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
1430 case TargetOpcode::G_ATOMICRMW_SUB:
1431 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
1432 case TargetOpcode::G_ATOMICRMW_XOR:
1433 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
1434 case TargetOpcode::G_ATOMICRMW_UMAX:
1435 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
1436 case TargetOpcode::G_ATOMICRMW_UMIN:
1437 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
1438 case TargetOpcode::G_ATOMICRMW_XCHG:
1439 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
1440
1441 case TargetOpcode::G_ATOMICRMW_FADD:
1442 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT);
1443 case TargetOpcode::G_ATOMICRMW_FSUB:
1444 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
1445 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT,
1446 isVectorType(ResType) ? SPIRV::OpFNegateV
1447 : SPIRV::OpFNegate);
1448 case TargetOpcode::G_ATOMICRMW_FMIN:
1449 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT);
1450 case TargetOpcode::G_ATOMICRMW_FMAX:
1451 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT);
1452
1453 case TargetOpcode::G_FENCE:
1454 return selectFence(I);
1455
1456 case TargetOpcode::G_STACKSAVE:
1457 return selectStackSave(ResVReg, ResType, I);
1458 case TargetOpcode::G_STACKRESTORE:
1459 return selectStackRestore(I);
1460
1461 case TargetOpcode::G_UNMERGE_VALUES:
1462 return selectUnmergeValues(I);
1463
1464 case TargetOpcode::G_TRAP:
1465 case TargetOpcode::G_UBSANTRAP:
1466 return selectTrap(I);
1467
1468 // Discard gen opcodes for intrinsics which we do not expect to actually
1469 // represent code after lowering or intrinsics which are not implemented but
1470 // should not crash when found in a customer's LLVM IR input.
1471 case TargetOpcode::DBG_LABEL:
1472 return true;
1473 case TargetOpcode::G_DEBUGTRAP:
1474 return selectDebugTrap(ResVReg, ResType, I);
1475 case TargetOpcode::G_PREFETCH:
1476 return selectPrefetch(I);
1477
1478 default:
1479 return false;
1480 }
1481}
1482
1483bool SPIRVInstructionSelector::selectDebugTrap(Register ResVReg,
1484 SPIRVTypeInst ResType,
1485 MachineInstr &I) const {
1486 if (!STI.canUseExtInstSet(SPIRV::InstructionSet::NonSemantic_DebugBreak)) {
1487 MachineBasicBlock &BB = *I.getParent();
1488 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpNop))
1489 .constrainAllUses(TII, TRI, RBI);
1490 return true;
1491 }
1492 MachineIRBuilder MIRBuilder(I);
1493 SPIRVTypeInst VoidTy = GR.getOpTypeVoid(MIRBuilder);
1494 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(VoidTy));
1495 return selectExtInst(ExtReg, VoidTy, I,
1496 {{SPIRV::InstructionSet::NonSemantic_DebugBreak,
1497 SPIRV::NonSemanticDebugBreakExtInst::DebugBreak}},
1498 /*setMIFlags=*/false, /*useMISrc=*/false);
1499}
1500
1501bool SPIRVInstructionSelector::selectPrefetch(MachineInstr &I) const {
1502 // llvm.prefetch is missing info about how many bytes to prefetch that
1503 // OpUntypedPrefetchKHR instruction requires, so just drop the instruction
1504 // in case untyped pointers are enabled since OCL prefetch cannot take
1505 // untyped pointers.
1506 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_untyped_pointers))
1507 return true;
1508
1509 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
1510 MachineIRBuilder MIRBuilder(I);
1511 const unsigned PointerSize = GR.getPointerSize();
1512 const SPIRVTypeInst PointerSizeType =
1513 GR.getOrCreateSPIRVIntegerType(PointerSize, MIRBuilder);
1514 bool ZeroAsNull = !STI.isShader();
1515
1516 // Assume that a single element needs prefetch.
1517 Register ConstIntOne =
1518 GR.getOrCreateConstInt(1, I, PointerSizeType, TII, ZeroAsNull);
1519
1520 Register AddrVal = I.getOperand(0).getReg();
1522 GR.getRegClass(GR.getOpTypeVoid(MIRBuilder)));
1523 return selectExtInst(ExtReg, GR.getOpTypeVoid(MIRBuilder), I, CL::prefetch,
1524 /*setMIFlags=*/false, /*useMSSrc=*/false,
1525 {AddrVal, ConstIntOne});
1526 }
1527 return true;
1528}
1529
1530bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1531 SPIRVTypeInst ResType,
1532 MachineInstr &I,
1533 GL::GLSLExtInst GLInst,
1534 bool setMIFlags, bool useMISrc,
1535 ArrayRef<Register> SrcRegs) const {
1536 if (!STI.canUseExtInstSet(
1537 SPIRV::InstructionSet::InstructionSet::GLSL_std_450))
1538 return diagnoseUnsupported(
1539 I,
1540 "this instruction is only supported with the GLSL extended instruction "
1541 "set.");
1542 return selectExtInst(ResVReg, ResType, I,
1543 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}},
1544 setMIFlags, useMISrc, SrcRegs);
1545}
1546
1547bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1548 SPIRVTypeInst ResType,
1549 MachineInstr &I,
1550 CL::OpenCLExtInst CLInst,
1551 bool setMIFlags, bool useMISrc,
1552 ArrayRef<Register> SrcRegs) const {
1553 return selectExtInst(ResVReg, ResType, I,
1554 {{SPIRV::InstructionSet::OpenCL_std, CLInst}},
1555 setMIFlags, useMISrc, SrcRegs);
1556}
1557
1558bool SPIRVInstructionSelector::selectExtInst(
1559 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
1560 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst, bool setMIFlags,
1561 bool useMISrc, ArrayRef<Register> SrcRegs) const {
1562 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1563 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1564 return selectExtInst(ResVReg, ResType, I, ExtInsts, setMIFlags, useMISrc,
1565 SrcRegs);
1566}
1567
1568bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
1569 SPIRVTypeInst ResType,
1570 MachineInstr &I,
1571 const ExtInstList &Insts,
1572 bool setMIFlags, bool useMISrc,
1573 ArrayRef<Register> SrcRegs) const {
1574
1575 for (const auto &[InstructionSet, Opcode] : Insts) {
1576 if (!STI.canUseExtInstSet(InstructionSet))
1577 continue;
1578 MachineBasicBlock &BB = *I.getParent();
1579 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1580 .addDef(ResVReg)
1581 .addUse(GR.getSPIRVTypeID(ResType))
1582 .addImm(static_cast<uint32_t>(InstructionSet))
1583 .addImm(Opcode);
1584 if (setMIFlags)
1585 MIB.setMIFlags(I.getFlags());
1586 if (useMISrc) {
1587 const unsigned NumOps = I.getNumOperands();
1588 unsigned Index = 1;
1589 if (Index < NumOps &&
1590 I.getOperand(Index).getType() ==
1591 MachineOperand::MachineOperandType::MO_IntrinsicID)
1592 Index = 2;
1593 for (; Index < NumOps; ++Index)
1594 MIB.add(I.getOperand(Index));
1595 } else {
1596 for (Register SReg : SrcRegs) {
1597 MIB.addUse(SReg);
1598 }
1599 }
1600 MIB.constrainAllUses(TII, TRI, RBI);
1601 return true;
1602 }
1603 return false;
1604}
1605
1606bool SPIRVInstructionSelector::selectCopySign(Register ResVReg,
1607 SPIRVTypeInst ResType,
1608 MachineInstr &I) const {
1609 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std))
1610 return selectExtInst(ResVReg, ResType, I, CL::copysign);
1611
1612 // There is no copysign instruction in the GLSL Extended Instruction set, so
1613 // it is implemented with bit manipulation:
1614 // bitcast((bitcast(magnitude) & ~signBit) | (bitcast(sign) & signBit))
1615 Register MagnitudeReg = I.getOperand(1).getReg();
1616 Register SignReg = I.getOperand(2).getReg();
1617
1618 const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1619 const unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
1620 SPIRVTypeInst IntType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
1621 const APInt SignMaskVal = APInt::getSignMask(BitWidth);
1622
1623 Register SignMask, NotSignMask;
1624 unsigned AndOpcode, OrOpcode;
1625 if (ComponentCount > 1) {
1626 IntType = GR.getOrCreateSPIRVVectorType(IntType, ComponentCount, I, TII);
1627 SignMask = GR.getOrCreateConstVector(SignMaskVal, I, IntType, TII);
1628 NotSignMask = GR.getOrCreateConstVector(~SignMaskVal, I, IntType, TII);
1629 AndOpcode = SPIRV::OpBitwiseAndV;
1630 OrOpcode = SPIRV::OpBitwiseOrV;
1631 } else {
1632 SignMask = GR.getOrCreateConstInt(SignMaskVal, I, IntType, TII);
1633 NotSignMask = GR.getOrCreateConstInt(~SignMaskVal, I, IntType, TII);
1634 AndOpcode = SPIRV::OpBitwiseAndS;
1635 OrOpcode = SPIRV::OpBitwiseOrS;
1636 }
1637
1638 auto EmitBitOp = [&](Register &ResReg, ArrayRef<Register> SrcRegs,
1639 unsigned Opcode) {
1640 ResReg = MRI->createVirtualRegister(GR.getRegClass(IntType));
1641 return selectOpWithSrcs(ResReg, IntType, I, SrcRegs, Opcode);
1642 };
1643
1644 Register MagnitudeInt, SignInt, MagnitudeBits, SignBits, CombinedInt;
1645 if (!EmitBitOp(MagnitudeInt, {MagnitudeReg}, SPIRV::OpBitcast) ||
1646 !EmitBitOp(SignInt, {SignReg}, SPIRV::OpBitcast) ||
1647 !EmitBitOp(MagnitudeBits, {MagnitudeInt, NotSignMask}, AndOpcode) ||
1648 !EmitBitOp(SignBits, {SignInt, SignMask}, AndOpcode) ||
1649 !EmitBitOp(CombinedInt, {MagnitudeBits, SignBits}, OrOpcode))
1650 return false;
1651
1652 return selectOpWithSrcs(ResVReg, ResType, I, {CombinedInt}, SPIRV::OpBitcast);
1653}
1654
1655bool SPIRVInstructionSelector::selectFrexp(Register ResVReg,
1656 SPIRVTypeInst ResType,
1657 MachineInstr &I) const {
1658 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp},
1659 {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}};
1660 for (const auto &Ex : ExtInsts) {
1661 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
1662 uint32_t Opcode = Ex.second;
1663 if (!STI.canUseExtInstSet(Set))
1664 continue;
1665
1666 MachineIRBuilder MIRBuilder(I);
1667 SPIRVTypeInst PointeeTy = GR.getSPIRVTypeForVReg(I.getOperand(1).getReg());
1668 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1669 PointeeTy, MIRBuilder, SPIRV::StorageClass::Function);
1670 Register PointerVReg =
1671 createVirtualRegister(PointerType, &GR, MRI, MRI->getMF());
1672
1673 auto It = getOpVariableMBBIt(*I.getMF());
1674 // An untyped pointer result type is only legal on OpUntypedVariableKHR,
1675 // but not on OpVariable.
1676 const bool IsUntyped =
1677 PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1678 auto VarMIB =
1679 BuildMI(*It->getParent(), It, It->getDebugLoc(),
1680 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
1681 : SPIRV::OpVariable))
1682 .addDef(PointerVReg)
1683 .addUse(GR.getSPIRVTypeID(PointerType))
1684 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
1685 if (IsUntyped)
1686 VarMIB.addUse(GR.getSPIRVTypeID(PointeeTy)); // Data Type
1687 VarMIB.constrainAllUses(TII, TRI, RBI);
1688
1689 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1690 .addDef(ResVReg)
1691 .addUse(GR.getSPIRVTypeID(ResType))
1692 .addImm(static_cast<uint32_t>(Ex.first))
1693 .addImm(Opcode)
1694 .add(I.getOperand(2))
1695 .addUse(PointerVReg)
1696 .constrainAllUses(TII, TRI, RBI);
1697
1698 Register ExpResReg = I.getOperand(1).getReg();
1699 if (!MRI->use_nodbg_empty(ExpResReg))
1700 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1701 .addDef(ExpResReg)
1702 .addUse(GR.getSPIRVTypeID(PointeeTy))
1703 .addUse(PointerVReg)
1704 .constrainAllUses(TII, TRI, RBI);
1705 return true;
1706 }
1707 return false;
1708}
1709
1710bool SPIRVInstructionSelector::selectLdexp(Register ResVReg,
1711 SPIRVTypeInst ResType,
1712 MachineInstr &I) const {
1713 Register XReg = I.getOperand(1).getReg();
1714 Register ExpReg = I.getOperand(2).getReg();
1715
1716 // Both OpenCL.std ldexp and GLSL.std.450 Ldexp require the exponent to have
1717 // the same number of components as the result, but G_FLDEXP's exponent is
1718 // scalar even for vector operands. Splat it when the result is a vector.
1719 SPIRVTypeInst ExpType = GR.getSPIRVTypeForVReg(ExpReg);
1720 if (ResType->getOpcode() == SPIRV::OpTypeVector &&
1721 ExpType->getOpcode() != SPIRV::OpTypeVector) {
1722 unsigned NumElts = ResType->getOperand(2).getImm();
1723 ExpReg = buildVectorSplat(ExpReg, NumElts, I);
1724 }
1725
1726 return selectExtInst(ResVReg, ResType, I, CL::ldexp, GL::Ldexp,
1727 /*setMIFlags=*/true, /*useMISrc=*/false, {XReg, ExpReg});
1728}
1729
1730bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
1731 SPIRVTypeInst ResType,
1732 MachineInstr &I) const {
1733 Register CosResVReg = I.getOperand(1).getReg();
1734 unsigned SrcIdx = I.getNumExplicitDefs();
1735 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
1736
1737 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
1738 // OpenCL.std sincos(x, cosval*) -> returns sin(x), writes cos(x) to ptr.
1739 MachineIRBuilder MIRBuilder(I);
1740 const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType(
1741 ResType, MIRBuilder, SPIRV::StorageClass::Function);
1742 Register PointerVReg =
1743 createVirtualRegister(PointerType, &GR, MRI, MRI->getMF());
1744
1745 auto It = getOpVariableMBBIt(*I.getMF());
1746 // An untyped pointer result type is only legal on OpUntypedVariableKHR,
1747 // but not on OpVariable.
1748 const bool IsUntyped =
1749 PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1750 auto VarMIB =
1751 BuildMI(*It->getParent(), It, It->getDebugLoc(),
1752 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
1753 : SPIRV::OpVariable))
1754 .addDef(PointerVReg)
1755 .addUse(GR.getSPIRVTypeID(PointerType))
1756 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
1757 if (IsUntyped)
1758 VarMIB.addUse(GR.getSPIRVTypeID(ResType)); // Data Type
1759 VarMIB.constrainAllUses(TII, TRI, RBI);
1760 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1761 .addDef(ResVReg)
1762 .addUse(ResTypeReg)
1763 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1764 .addImm(CL::sincos)
1765 .add(I.getOperand(SrcIdx))
1766 .addUse(PointerVReg)
1767 .constrainAllUses(TII, TRI, RBI);
1768 if (!MRI->use_nodbg_empty(CosResVReg))
1769 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
1770 .addDef(CosResVReg)
1771 .addUse(ResTypeReg)
1772 .addUse(PointerVReg)
1773 .constrainAllUses(TII, TRI, RBI);
1774 return true;
1775 } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
1776 // GLSL.std.450 has no combined sincos; emit separate Sin and Cos.
1777 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1778 .addDef(ResVReg)
1779 .addUse(ResTypeReg)
1780 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1781 .addImm(GL::Sin)
1782 .add(I.getOperand(SrcIdx))
1783 .constrainAllUses(TII, TRI, RBI);
1784 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1785 .addDef(CosResVReg)
1786 .addUse(ResTypeReg)
1787 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1788 .addImm(GL::Cos)
1789 .add(I.getOperand(SrcIdx))
1790 .constrainAllUses(TII, TRI, RBI);
1791 return true;
1792 }
1793 return false;
1794}
1795
1796bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
1797 SPIRVTypeInst ResType,
1798 MachineInstr &I,
1799 ArrayRef<Register> Srcs,
1800 unsigned Opcode) const {
1801 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1802 .addDef(ResVReg)
1803 .addUse(GR.getSPIRVTypeID(ResType));
1804 for (Register SReg : Srcs) {
1805 MIB.addUse(SReg);
1806 }
1807 MIB.constrainAllUses(TII, TRI, RBI);
1808 return true;
1809}
1810
1811bool SPIRVInstructionSelector::selectPopCount16(Register ResVReg,
1812 SPIRVTypeInst ResType,
1813 MachineInstr &I,
1814 unsigned ExtOpcode,
1815 unsigned Opcode) const {
1816 MachineIRBuilder MIRBuilder(I);
1817
1818 Register OpReg = I.getOperand(1).getReg();
1819 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(OpReg);
1820 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(SrcType);
1821 bool IsScalar = !isVectorType(SrcType);
1822 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
1823
1824 SPIRVTypeInst ExtType =
1825 IsScalar ? I32Type
1826 : GR.getOrCreateSPIRVVectorType(I32Type, ComponentCount,
1827 MIRBuilder, /*IsSigned=*/false);
1828
1829 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ExtType));
1830 // Always use OpUConvert to always use a 0 extend
1831 if (!selectOpWithSrcs(ExtReg, ExtType, I, {OpReg}, SPIRV::OpUConvert))
1832 return false;
1833
1834 Register PopCountReg = MRI->createVirtualRegister(GR.getRegClass(ExtType));
1835 if (!selectPopCount32(PopCountReg, ExtType, I, ExtReg, Opcode))
1836 return false;
1837
1838 return selectOpWithSrcs(ResVReg, ResType, I, {PopCountReg}, ExtOpcode);
1839}
1840
1841bool SPIRVInstructionSelector::selectPopCount32(Register ResVReg,
1842 SPIRVTypeInst ResType,
1843 MachineInstr &I,
1844 Register SrcReg,
1845 unsigned Opcode) const {
1846 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
1847}
1848
1849bool SPIRVInstructionSelector::selectPopCount64(Register ResVReg,
1850 SPIRVTypeInst ResType,
1851 MachineInstr &I,
1852 Register SrcReg,
1853 unsigned Opcode) const {
1854 MachineIRBuilder MIRBuilder(I);
1855
1856 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg);
1857 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(SrcType);
1858 bool IsScalar = !isVectorType(SrcType);
1859 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
1860
1861 // We need to work with a type matching the shape of the input but using 32
1862 // bit int instead of 64.
1863 SPIRVTypeInst WorkingType =
1864 IsScalar ? I32Type
1865 : GR.getOrCreateSPIRVVectorType(I32Type, ComponentCount,
1866 MIRBuilder, /*IsSigned=*/false);
1867
1868 // Truncate and count the low bits.
1869 Register Trunc = MRI->createVirtualRegister(GR.getRegClass(WorkingType));
1870 if (!selectOpWithSrcs(Trunc, WorkingType, I, {SrcReg}, SPIRV::OpUConvert))
1871 return false;
1872
1873 Register LowCount = MRI->createVirtualRegister(GR.getRegClass(WorkingType));
1874 if (!selectOpWithSrcs(LowCount, WorkingType, I, {Trunc}, SPIRV::OpBitCount))
1875 return false;
1876
1877 // Shift the high bits over and count them too.
1878 Register ShiftAmount = IsScalar
1879 ? GR.getOrCreateConstInt(32, I, SrcType, TII)
1880 : GR.getOrCreateConstVector(32, I, SrcType, TII);
1881 unsigned ShiftOp =
1882 IsScalar ? SPIRV::OpShiftRightLogicalS : SPIRV::OpShiftRightLogicalV;
1883 Register Shift = MRI->createVirtualRegister(GR.getRegClass(SrcType));
1884 if (!selectOpWithSrcs(Shift, SrcType, I, {SrcReg, ShiftAmount}, ShiftOp))
1885 return false;
1886
1887 Trunc = MRI->createVirtualRegister(GR.getRegClass(WorkingType));
1888 if (!selectOpWithSrcs(Trunc, WorkingType, I, {Shift}, SPIRV::OpUConvert))
1889 return false;
1890
1891 Register HighCount = MRI->createVirtualRegister(GR.getRegClass(WorkingType));
1892 if (!selectOpWithSrcs(HighCount, WorkingType, I, {Trunc}, SPIRV::OpBitCount))
1893 return false;
1894
1895 // Add them up and zext or sext back to 64 bit values.
1896 Register Sum = MRI->createVirtualRegister(GR.getRegClass(WorkingType));
1897 if (!selectOpWithSrcs(Sum, WorkingType, I, {HighCount, LowCount},
1898 IsScalar ? SPIRV::OpIAddS : SPIRV::OpIAddV))
1899 return false;
1900
1901 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
1902 unsigned ConvOp = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1903 return selectOpWithSrcs(ResVReg, ResType, I, {Sum}, ConvOp);
1904}
1905
1906bool SPIRVInstructionSelector::selectPopCount(Register ResVReg,
1907 SPIRVTypeInst ResType,
1908 MachineInstr &I,
1909 unsigned Opcode) const {
1910 // Vulkan restricts OpBitCount to 32-bit integers or vectors of 32-bit
1911 // integers unless VK_KHR_maintenance9 is enabled. Until VK_KHR_maintenance9
1912 // is core we will not generate OpBitCount with any other types when
1913 // targeting Vulkan.
1914 if (!STI.getTargetTriple().isVulkanOS())
1915 return selectUnOp(ResVReg, ResType, I, Opcode);
1916
1917 Register OpReg = I.getOperand(1).getReg();
1918 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
1919 unsigned ExtOpcode = GR.isScalarOrVectorSigned(ResType) ? SPIRV::OpSConvert
1920 : SPIRV::OpUConvert;
1921 switch (GR.getScalarOrVectorBitWidth(OpType)) {
1922 case 8:
1923 case 16:
1924 return selectPopCount16(ResVReg, ResType, I, ExtOpcode, Opcode);
1925 case 32:
1926 return selectPopCount32(ResVReg, ResType, I, OpReg, Opcode);
1927 case 64:
1928 return selectPopCount64(ResVReg, ResType, I, OpReg, Opcode);
1929 default:
1930 return diagnoseUnsupported(I, "unsupported operand bit width for popcount");
1931 }
1932}
1933
1934bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
1935 SPIRVTypeInst ResType,
1936 MachineInstr &I,
1937 unsigned Opcode) const {
1938 if (STI.isPhysicalSPIRV() && I.getOperand(1).isReg()) {
1939 Register SrcReg = I.getOperand(1).getReg();
1940 bool IsGV = false;
1942 MRI->def_instr_begin(SrcReg);
1943 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) {
1944 unsigned DefOpCode = DefIt->getOpcode();
1945 if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
1946 // We need special handling to look through the type assignment or the
1947 // COPY pseudo-op and see if this is a constant or a global.
1948 if (auto *VRD = getVRegDef(*MRI, DefIt->getOperand(1).getReg()))
1949 DefOpCode = VRD->getOpcode();
1950 }
1951 if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE ||
1952 DefOpCode == TargetOpcode::G_CONSTANT ||
1953 DefOpCode == SPIRV::OpVariable ||
1954 DefOpCode == SPIRV::OpUntypedVariableKHR ||
1955 DefOpCode == SPIRV::OpConstantI) {
1956 IsGV = true;
1957 break;
1958 }
1959 }
1960 if (IsGV) {
1961 uint32_t SpecOpcode = 0;
1962 switch (Opcode) {
1963 case SPIRV::OpConvertPtrToU:
1964 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
1965 break;
1966 case SPIRV::OpConvertUToPtr:
1967 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
1968 break;
1969 }
1970 if (SpecOpcode) {
1971 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1972 TII.get(SPIRV::OpSpecConstantOp))
1973 .addDef(ResVReg)
1974 .addUse(GR.getSPIRVTypeID(ResType))
1975 .addImm(SpecOpcode)
1976 .addUse(SrcReg)
1977 .constrainAllUses(TII, TRI, RBI);
1978 return true;
1979 }
1980 }
1981 }
1982 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()},
1983 Opcode);
1984}
1985
1986bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
1987 SPIRVTypeInst ResType,
1988 MachineInstr &I) const {
1989 Register OpReg = I.getOperand(1).getReg();
1990 SPIRVTypeInst OpType =
1991 OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
1992 if (!GR.isBitcastCompatible(ResType, OpType))
1993 return diagnoseUnsupported(
1994 I, "incompatible result and operand types in a bitcast");
1995 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
1996}
1997
1998static void
2000 MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry &GR,
2001 std::optional<Align> AlignOverride = std::nullopt) {
2002 const SPIRVSubtarget *ST =
2003 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
2004 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
2005 if (MemOp->isVolatile())
2006 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
2007 if (MemOp->isNonTemporal())
2008 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
2009 // Aligned memory operand requires the Kernel capability.
2010 if (!ST->isShader() && MemOp->getAlign().value())
2011 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
2012
2013 [[maybe_unused]] MachineInstr *AliasList = nullptr;
2014 [[maybe_unused]] MachineInstr *NoAliasList = nullptr;
2015 if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
2016 if (auto *MD = MemOp->getAAInfo().Scope) {
2017 AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD);
2018 if (AliasList)
2019 SpvMemOp |=
2020 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
2021 }
2022 if (auto *MD = MemOp->getAAInfo().NoAlias) {
2023 NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD);
2024 if (NoAliasList)
2025 SpvMemOp |=
2026 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
2027 }
2028 }
2029
2030 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
2031 MIB.addImm(SpvMemOp);
2032 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
2033 MIB.addImm(AlignOverride.value_or(MemOp->getAlign()).value());
2034 if (AliasList)
2035 MIB.addUse(AliasList->getOperand(0).getReg());
2036 if (NoAliasList)
2037 MIB.addUse(NoAliasList->getOperand(0).getReg());
2038 }
2039}
2040
2042 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
2044 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
2046 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
2047
2048 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
2049 MIB.addImm(SpvMemOp);
2050}
2051
2052bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
2053 SPIRVTypeInst ResType,
2054 MachineInstr &I) const {
2055 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2056 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2057
2058 auto *PtrDef = getVRegDef(*MRI, Ptr);
2059 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
2060 if (IntPtrDef &&
2061 (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
2062 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
2063
2064 Register HandleReg = IntPtrDef->getOperand(2).getReg();
2065 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg);
2066 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
2067 Register NewHandleReg =
2069 auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg));
2070 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) {
2071 return false;
2072 }
2073
2074 Register IdxReg = IntPtrDef->getOperand(3).getReg();
2075 return generateImageReadOrFetch(ResVReg, ResType, NewHandleReg, IdxReg,
2076 I.getDebugLoc(), I);
2077 }
2078 }
2079
2080 MachineIRBuilder MIRBuilder(I);
2081
2082 if (I.getNumMemOperands()) {
2083 const MachineMemOperand *MemOp = *I.memoperands_begin();
2084 if (MemOp->isAtomic())
2085 return selectAtomicLoad(ResVReg, ResType, I);
2086 }
2087
2088 auto MIB = MIRBuilder.buildInstr(SPIRV::OpLoad)
2089 .addDef(ResVReg)
2090 .addUse(GR.getSPIRVTypeID(ResType))
2091 .addUse(Ptr);
2092 if (!I.getNumMemOperands()) {
2093 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
2094 I.getOpcode() ==
2095 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
2096 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
2097 } else {
2098 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2099 }
2100 MIB.constrainAllUses(TII, TRI, RBI);
2101 return true;
2102}
2103
2104Register SPIRVInstructionSelector::createPtrSizedIntReg(
2105 MachineIRBuilder &MIRBuilder) const {
2106 SPIRVTypeInst IntType =
2107 GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder);
2108 Register Reg =
2110 MRI->setRegClass(Reg, GR.getRegClass(IntType));
2111 GR.assignSPIRVTypeToVReg(IntType, Reg, MIRBuilder.getMF());
2112 return Reg;
2113}
2114
2116SPIRVInstructionSelector::convertPtrToInt(Register PtrVal,
2117 MachineIRBuilder &MIRBuilder) const {
2118 SPIRVTypeInst IntType =
2119 GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder);
2120 Register IntReg = createPtrSizedIntReg(MIRBuilder);
2121 MIRBuilder.buildInstr(SPIRV::OpConvertPtrToU)
2122 .addDef(IntReg)
2123 .addUse(GR.getSPIRVTypeID(IntType)) // Result type
2124 .addUse(PtrVal) // Pointer operand
2125 .constrainAllUses(TII, TRI, RBI);
2126 return IntReg;
2127}
2128
2129Register SPIRVInstructionSelector::castPtrToPtrToInt(
2130 Register Ptr, SPIRV::StorageClass::StorageClass SC,
2131 MachineIRBuilder &MIRBuilder) const {
2132 SPIRVTypeInst IntType =
2133 GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder);
2134 SPIRVTypeInst PtrType =
2135 GR.getOrCreateSPIRVPointerType(IntType, MIRBuilder, SC);
2136 Register CastedPtr =
2138 MRI->setRegClass(CastedPtr, GR.getRegClass(PtrType));
2139 GR.assignSPIRVTypeToVReg(PtrType, CastedPtr, MIRBuilder.getMF());
2140 MIRBuilder.buildInstr(SPIRV::OpBitcast)
2141 .addDef(CastedPtr)
2142 .addUse(GR.getSPIRVTypeID(PtrType))
2143 .addUse(Ptr)
2144 .constrainAllUses(TII, TRI, RBI);
2145 return CastedPtr;
2146}
2147
2148bool SPIRVInstructionSelector::selectAtomicPtrValue(
2149 Register ResVReg, SPIRVTypeInst ResType, MachineIRBuilder &MIRBuilder,
2150 function_ref<Register(SPIRVTypeInst IntType)> EmitAtomic) const {
2151 // Pointer-typed atomics are lowered by bitcasting the Ptr operand to a
2152 // pointer to an integer of the same size as the pointer, so that the actual
2153 // atomic instruction operates on integers as required by the spec. Value
2154 // operands and results are converted with OpConvertPtrToU/OpConvertUToPtr.
2155 unsigned PtrSize = GR.getPointerSize();
2156 SPIRVTypeInst IntType = GR.getOrCreateSPIRVIntegerType(PtrSize, MIRBuilder);
2157
2158 Register IntResult = EmitAtomic(IntType);
2159 if (IntResult.isValid())
2160 MIRBuilder.buildInstr(SPIRV::OpConvertUToPtr)
2161 .addDef(ResVReg)
2162 .addUse(GR.getSPIRVTypeID(ResType))
2163 .addUse(IntResult)
2164 .constrainAllUses(TII, TRI, RBI);
2165 return true;
2166}
2167
2168bool SPIRVInstructionSelector::selectAtomicLoad(Register ResVReg,
2169 SPIRVTypeInst ResType,
2170 MachineInstr &I) const {
2171 LLVMContext &Context = I.getMF()->getFunction().getContext();
2172
2173 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2174 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2175
2176 if (!ResType.isTypeIntOrFloat() && !ResType.isTypePtr())
2177 return diagnoseUnsupported(
2178 I, "Lowering to SPIR-V of atomic load is only "
2179 "allowed for integer, floating point or pointer types");
2180
2181 assert(I.getNumMemOperands());
2182 const MachineMemOperand &MemOp = **I.memoperands_begin();
2183 assert(MemOp.isAtomic());
2184
2185 uint32_t Scope = static_cast<uint32_t>(
2186 getMemScope(STI.getTargetTriple(), Context, MemOp.getSyncScopeID()));
2187 Register ScopeReg = buildI32ConstantInEntryBlock(Scope, I);
2188
2189 AtomicOrdering AO = MemOp.getSuccessOrdering();
2190 uint32_t StorageClass = static_cast<uint32_t>(getMemSemanticsForStorageClass(
2191 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI)));
2192 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
2193 if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS())
2194 MemSem |= static_cast<uint32_t>(SPIRV::MemorySemantics::Volatile);
2195 uint32_t Sem = getMemSemanticsWithStorageClass(STI.getTargetTriple(), MemSem,
2196 StorageClass);
2197 Register MemSemReg = buildI32ConstantInEntryBlock(Sem, I);
2198
2199 MachineIRBuilder MIRBuilder(I);
2200
2201 if (ResType.isTypePtr()) {
2202 if (!STI.isPhysicalSPIRV())
2203 return diagnoseUnsupported(
2204 I, "Lowering to SPIR-V of atomic load is only "
2205 "allowed for pointer types for physical addressing model");
2206 // If data to load is a pointer type we bitcast the Ptr parameter to pointer
2207 // to an integer type of the same size as the pointer size and then generate
2208 // OpAtomicLoad the return value of that OpAtomicLoad is an integer that is
2209 // converted back to a pointer type using OpConvertUToPtr.
2210 SPIRV::StorageClass::StorageClass SC =
2211 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI);
2212 return selectAtomicPtrValue(
2213 ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) {
2214 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2215 Register IntResult = createPtrSizedIntReg(MIRBuilder);
2216 MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
2217 .addDef(IntResult)
2218 .addUse(GR.getSPIRVTypeID(IntType))
2219 .addUse(CastedPtr)
2220 .addUse(ScopeReg)
2221 .addUse(MemSemReg)
2222 .constrainAllUses(TII, TRI, RBI);
2223 return IntResult;
2224 });
2225 }
2226 auto AtomicLoad = MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
2227 .addDef(ResVReg)
2228 .addUse(GR.getSPIRVTypeID(ResType))
2229 .addUse(Ptr)
2230 .addUse(ScopeReg)
2231 .addUse(MemSemReg);
2232 AtomicLoad.constrainAllUses(TII, TRI, RBI);
2233
2234 return true;
2235}
2236
2237bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
2238 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2239 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
2240 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2241
2242 auto *PtrDef = getVRegDef(*MRI, Ptr);
2243 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef);
2244 if (IntPtrDef &&
2245 (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
2246 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
2247
2248 Register HandleReg = IntPtrDef->getOperand(2).getReg();
2249 Register NewHandleReg =
2251 auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg));
2252 SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg);
2253 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) {
2254 return false;
2255 }
2256
2257 Register IdxReg = IntPtrDef->getOperand(3).getReg();
2258 if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
2259 SPIRVTypeInst SampledType =
2260 GR.getSPIRVTypeForVReg(HandleType->getOperand(1).getReg());
2261 SPIRVTypeInst StoreValCompType =
2263 if (StoreValCompType && StoreValCompType != SampledType) {
2264 // A wide element (e.g. int64_t2) is emulated with a narrower packed
2265 // image. This bitcasts the value to match the format.
2266 SPIRVTypeInst PackedType = widenTypeToVec4(SampledType, I);
2267 Register PackedReg =
2268 MRI->createVirtualRegister(GR.getRegClass(PackedType));
2269 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
2270 .addDef(PackedReg)
2271 .addUse(GR.getSPIRVTypeID(PackedType))
2272 .addUse(StoreVal)
2273 .constrainAllUses(TII, TRI, RBI);
2274 StoreVal = PackedReg;
2275 }
2276
2277 auto BMI = BuildMI(*I.getParent(), I, I.getDebugLoc(),
2278 TII.get(SPIRV::OpImageWrite))
2279 .addUse(NewHandleReg)
2280 .addUse(IdxReg)
2281 .addUse(StoreVal);
2282
2283 const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(HandleType);
2284 if (sampledTypeIsSignedInteger(LLVMHandleType))
2285 BMI.addImm(0x1000); // SignExtend
2286
2287 BMI.constrainAllUses(TII, TRI, RBI);
2288 return true;
2289 }
2290 }
2291
2292 SPIRVTypeInst PointeeTy = GR.getPointeeType(GR.getSPIRVTypeForVReg(Ptr));
2293 SPIRVTypeInst StoreTy = GR.getSPIRVTypeForVReg(StoreVal);
2294 if (PointeeTy && PointeeTy->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
2295 StoreTy->getOpcode() != SPIRV::OpTypeVectorIdEXT &&
2296 GR.getScalarOrVectorComponentCount(PointeeTy) == 1) {
2297 MachineInstr *StoreValDef = getVRegDef(*MRI, StoreVal);
2298 Register Reg = StoreValDef->getOperand(0).getReg();
2299 if (auto FC = GFConstant::getConstant(Reg, *MRI))
2300 StoreVal =
2301 GR.getOrCreateConstVector(FC->getScalarValue(), I, PointeeTy, TII);
2302 else if (auto IC = GIConstant::getConstant(Reg, *MRI))
2303 StoreVal =
2304 GR.getOrCreateConstVector(IC->getScalarValue(), I, PointeeTy, TII);
2305 else
2306 llvm_unreachable("Unexpected <1 x T> Store type!");
2307 }
2308
2309 if (I.getNumMemOperands()) {
2310 const MachineMemOperand *MemOp = *I.memoperands_begin();
2311 if (MemOp->isAtomic())
2312 return selectAtomicStore(I);
2313 }
2314
2315 // Stores into a read-only storage class produce invalid SPIR-V. Reject such
2316 // input with a diagnostic rather than silently emitting an OpStore that
2317 // validation rejects.
2318 SPIRV::StorageClass::StorageClass PtrSC = GR.getPointerStorageClass(Ptr);
2319 if (PtrSC == SPIRV::StorageClass::UniformConstant ||
2320 PtrSC == SPIRV::StorageClass::Input ||
2321 PtrSC == SPIRV::StorageClass::PushConstant)
2322 return diagnoseUnsupported(
2323 I, "store into a read-only SPIR-V storage class is not allowed");
2324
2325 MachineIRBuilder MIRBuilder(I);
2326 auto MIB = MIRBuilder.buildInstr(SPIRV::OpStore).addUse(Ptr).addUse(StoreVal);
2327 if (!I.getNumMemOperands()) {
2328 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
2329 I.getOpcode() ==
2330 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
2331 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
2332 } else {
2333 addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
2334 }
2335 MIB.constrainAllUses(TII, TRI, RBI);
2336 return true;
2337}
2338
2339bool SPIRVInstructionSelector::selectAtomicStore(MachineInstr &I) const {
2340 LLVMContext &Context = I.getMF()->getFunction().getContext();
2341
2342 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
2343 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
2344 Register Ptr = I.getOperand(1 + OpOffset).getReg();
2345
2346 SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(Ptr);
2347 SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
2348 // For an untyped pointer the data type is the stored value's type.
2349 if (!PointeeType && PtrType &&
2350 PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
2351 PointeeType = GR.getSPIRVTypeForVReg(StoreVal);
2352 if (!PointeeType)
2353 return diagnoseUnsupported(I,
2354 "Lowering to SPIR-V of atomic store is only "
2355 "allowed for integer or floating point types");
2356
2357 assert(I.getNumMemOperands());
2358 const MachineMemOperand &MemOp = **I.memoperands_begin();
2359 assert(MemOp.isAtomic());
2360
2361 uint32_t Scope = static_cast<uint32_t>(
2362 getMemScope(STI.getTargetTriple(), Context, MemOp.getSyncScopeID()));
2363 Register ScopeReg = buildI32ConstantInEntryBlock(Scope, I);
2364
2365 AtomicOrdering AO = MemOp.getSuccessOrdering();
2366 uint32_t StorageClass = static_cast<uint32_t>(getMemSemanticsForStorageClass(
2367 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI)));
2368 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
2369 if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS())
2370 MemSem |= static_cast<uint32_t>(SPIRV::MemorySemantics::Volatile);
2371 uint32_t Sem = getMemSemanticsWithStorageClass(STI.getTargetTriple(), MemSem,
2372 StorageClass);
2373 Register MemSemReg = buildI32ConstantInEntryBlock(Sem, I);
2374 MachineIRBuilder MIRBuilder(I);
2375
2376 if (PointeeType.isTypePtr()) {
2377 if (!STI.isPhysicalSPIRV())
2378 return diagnoseUnsupported(
2379 I, "Lowering to SPIR-V of atomic store is only "
2380 "allowed for pointer types for physical addressing model");
2381 // If data to store is a pointer type we cast it to an integer type of the
2382 // same size as the pointer size using OpConvertPtrToU, bitcast Ptr
2383 // parameter to pointer to integer type and then generate OpAtomicStore
2384 // with casted values as required by spec.
2385 SPIRV::StorageClass::StorageClass SC =
2386 addressSpaceToStorageClass(MemOp.getAddrSpace(), STI);
2387 return selectAtomicPtrValue(
2388 Register(), SPIRVTypeInst(), MIRBuilder, [&](SPIRVTypeInst IntType) {
2389 Register ValueAsInt = convertPtrToInt(StoreVal, MIRBuilder);
2390 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2391 MIRBuilder.buildInstr(SPIRV::OpAtomicStore)
2392 .addUse(CastedPtr)
2393 .addUse(ScopeReg)
2394 .addUse(MemSemReg)
2395 .addUse(ValueAsInt)
2396 .constrainAllUses(TII, TRI, RBI);
2397 // Stores produce no result, so no OpConvertUToPtr is needed.
2398 return Register();
2399 });
2400 }
2401
2402 if (!PointeeType.isTypeIntOrFloat())
2403 return diagnoseUnsupported(I,
2404 "Lowering to SPIR-V of atomic store is only "
2405 "allowed for integer or floating point types");
2406
2407 auto AtomicStore = MIRBuilder.buildInstr(SPIRV::OpAtomicStore)
2408 .addUse(Ptr)
2409 .addUse(ScopeReg)
2410 .addUse(MemSemReg)
2411 .addUse(StoreVal);
2412 AtomicStore.constrainAllUses(TII, TRI, RBI);
2413
2414 return true;
2415}
2416
2417bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg,
2418 SPIRVTypeInst ResType,
2419 MachineInstr &I) const {
2420 assert(I.getNumExplicitDefs() == 1 && "Expected single def for gather");
2421 // Operand indices:
2422 // 0: result (def)
2423 // 1: intrinsic ID
2424 // 2: vector of pointers
2425 // 3: alignment (i32 immediate)
2426 // 4: mask (vector of i1)
2427 // 5: passthru/fill value
2428 const Register PtrsReg = I.getOperand(2).getReg();
2429 const uint32_t Alignment = I.getOperand(3).getImm();
2430 const Register MaskReg = I.getOperand(4).getReg();
2431 const Register PassthruReg = I.getOperand(5).getReg();
2432 const Register AlignmentReg = buildI32ConstantInEntryBlock(Alignment, I);
2433
2434 MachineBasicBlock &BB = *I.getParent();
2435 auto MIB =
2436 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedGatherINTEL))
2437 .addDef(ResVReg)
2438 .addUse(GR.getSPIRVTypeID(ResType))
2439 .addUse(PtrsReg)
2440 .addUse(AlignmentReg)
2441 .addUse(MaskReg)
2442 .addUse(PassthruReg);
2443 MIB.constrainAllUses(TII, TRI, RBI);
2444 return true;
2445}
2446
2447bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &I) const {
2448 assert(I.getNumExplicitDefs() == 0 && "Expected no defs for scatter");
2449 // Operand indices (no explicit defs):
2450 // 0: intrinsic ID
2451 // 1: value vector
2452 // 2: vector of pointers
2453 // 3: alignment (i32 immediate)
2454 // 4: mask (vector of i1)
2455 const Register ValuesReg = I.getOperand(1).getReg();
2456 const Register PtrsReg = I.getOperand(2).getReg();
2457 const uint32_t Alignment = I.getOperand(3).getImm();
2458 const Register MaskReg = I.getOperand(4).getReg();
2459 const Register AlignmentReg = buildI32ConstantInEntryBlock(Alignment, I);
2460 MachineBasicBlock &BB = *I.getParent();
2461
2462 auto MIB =
2463 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedScatterINTEL))
2464 .addUse(PtrsReg)
2465 .addUse(AlignmentReg)
2466 .addUse(MaskReg)
2467 .addUse(ValuesReg);
2468 MIB.constrainAllUses(TII, TRI, RBI);
2469 return true;
2470}
2471
2472bool SPIRVInstructionSelector::diagnoseUnsupported(const MachineInstr &I,
2473 const Twine &Msg) const {
2474 const Function &F = I.getMF()->getFunction();
2475 F.getContext().diagnose(
2476 DiagnosticInfoUnsupported(F, Msg, I.getDebugLoc(), DS_Error));
2477 return false;
2478}
2479
2480bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
2481 SPIRVTypeInst ResType,
2482 MachineInstr &I) const {
2483 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2484 return diagnoseUnsupported(
2485 I, "llvm.stacksave intrinsic: this instruction requires the following "
2486 "SPIR-V extension: SPV_INTEL_variable_length_array");
2487 MachineBasicBlock &BB = *I.getParent();
2488 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL))
2489 .addDef(ResVReg)
2490 .addUse(GR.getSPIRVTypeID(ResType))
2491 .constrainAllUses(TII, TRI, RBI);
2492 return true;
2493}
2494
2495bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
2496 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2497 return diagnoseUnsupported(
2498 I,
2499 "llvm.stackrestore intrinsic: this instruction requires the following "
2500 "SPIR-V extension: SPV_INTEL_variable_length_array");
2501 if (!I.getOperand(0).isReg())
2502 return false;
2503 MachineBasicBlock &BB = *I.getParent();
2504 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL))
2505 .addUse(I.getOperand(0).getReg())
2506 .constrainAllUses(TII, TRI, RBI);
2507 return true;
2508}
2509
2511SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &I) const {
2512 MachineIRBuilder MIRBuilder(I);
2513 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
2514
2515 // TODO: check if we have such GV, add init, use buildGlobalVariable.
2516 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
2517 Function &CurFunction = GR.CurMF->getFunction();
2518 Type *LLVMArrTy =
2519 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num);
2520 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
2522 Constant::getNullValue(LLVMArrTy));
2523
2524 Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext());
2525 Type *ArrTy = ArrayType::get(ValTy, Num);
2526 SPIRVTypeInst VarTy = GR.getOrCreateSPIRVPointerType(
2527 ArrTy, MIRBuilder, SPIRV::StorageClass::UniformConstant);
2528
2529 SPIRVTypeInst SpvArrTy = GR.getOrCreateSPIRVType(
2530 ArrTy, MIRBuilder, SPIRV::AccessQualifier::None, false);
2531
2532 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
2533 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvArrTy, TII);
2534
2536 // With SPV_KHR_untyped_pointers enabled, getOrCreateSPIRVPointerType returns
2537 // an untyped pointer type. An untyped pointer result type is only legal on
2538 // OpUntypedVariableKHR, not on OpVariable.
2539 // Pick the matching opcode/operands so the synthesized constant global is
2540 // valid SPIR-V.
2541 const bool IsUntyped = VarTy->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
2542 auto MIBVar = BuildMI(*I.getParent(), I, I.getDebugLoc(),
2543 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
2544 : SPIRV::OpVariable))
2545 .addDef(VarReg)
2546 .addUse(GR.getSPIRVTypeID(VarTy))
2547 .addImm(SPIRV::StorageClass::UniformConstant);
2548 if (IsUntyped)
2549 MIBVar.addUse(GR.getSPIRVTypeID(SpvArrTy)); // Data Type
2550 MIBVar.addUse(Const); // Initializer
2551 MIBVar.constrainAllUses(TII, TRI, RBI);
2552
2553 GR.add(GV, MIBVar);
2554 GR.addGlobalObject(GV, GR.CurMF, VarReg);
2555
2556 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
2557 return VarReg;
2558}
2559
2560bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I,
2561 Register SrcReg) const {
2562 MachineBasicBlock &BB = *I.getParent();
2563 Register DstReg = I.getOperand(0).getReg();
2564 const bool IsLogical = STI.isLogicalSPIRV();
2565 if (IsLogical) {
2566 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(DstReg);
2567 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(SrcReg);
2568 if (GR.getPointeeType(DstTy) != GR.getPointeeType(SrcTy))
2569 return diagnoseUnsupported(
2570 I, "OpCopyMemory requires operands to have the same type");
2571 uint64_t CopySize = getIConstVal(I.getOperand(2).getReg(), MRI);
2572 SPIRVTypeInst PointeeTy = GR.getPointeeType(DstTy);
2573 const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(PointeeTy);
2574 if (!LLVMPointeeTy)
2575 return diagnoseUnsupported(
2576 I, "Unable to determine pointee type size for OpCopyMemory");
2577 const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
2578 if (CopySize != DL.getTypeStoreSize(const_cast<Type *>(LLVMPointeeTy)))
2579 return diagnoseUnsupported(
2580 I, "OpCopyMemory requires the size to match the pointee type size");
2581 }
2582
2583 const unsigned Opcode =
2584 IsLogical ? SPIRV::OpCopyMemory : SPIRV::OpCopyMemorySized;
2585 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2586 .addUse(DstReg)
2587 .addUse(SrcReg);
2588 if (!IsLogical)
2589 MIB.addUse(I.getOperand(2).getReg());
2590
2591 if (I.getNumMemOperands()) {
2592 MachineIRBuilder MIRBuilder(I);
2593 const MachineMemOperand *DstMemOp = *I.memoperands_begin();
2594 const Align DstAlign = DstMemOp->getAlign();
2595 Align SrcAlign = DstAlign;
2596 // Copies carry destination and source MMOs; memset carries only the
2597 // destination. Shader memory operands do not include alignment.
2598 if (I.getNumMemOperands() > 1 && !STI.isShader())
2599 SrcAlign = (*std::next(I.memoperands_begin()))->getAlign();
2600
2601 // A single mask applies to both pointers. SPIR-V 1.4 allows separate
2602 // destination and source masks, preserving their individual alignments.
2603 if (DstAlign != SrcAlign && STI.isAtLeastSPIRVVer(VersionTuple(1, 4))) {
2604 // Preserve the existing flags and metadata on both accesses.
2605 addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR);
2606 addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR, SrcAlign);
2607 } else {
2608 addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR,
2609 std::min(DstAlign, SrcAlign));
2610 }
2611 }
2612 MIB.constrainAllUses(TII, TRI, RBI);
2613 return true;
2614}
2615
2616bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
2617 MachineInstr &I) const {
2618 // Zero-sized memcpy/memmove/memset are no-ops.
2619 Register SizeReg = I.getOperand(2).getReg();
2620 if (MachineInstr *SizeDef = getDefInstrMaybeConstant(SizeReg, MRI);
2621 SizeDef && SizeDef->getOpcode() == TargetOpcode::G_CONSTANT &&
2622 getIConstVal(SizeReg, MRI) == 0)
2623 return true;
2624
2625 Register SrcReg = I.getOperand(1).getReg();
2626 if (I.getOpcode() == TargetOpcode::G_MEMSET ||
2627 I.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
2628 Register VarReg = getOrCreateMemSetGlobal(I);
2629 if (!VarReg.isValid())
2630 return false;
2631 Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext());
2632 SPIRVTypeInst SourceTy = GR.getOrCreateSPIRVPointerType(
2633 ValTy, I, SPIRV::StorageClass::UniformConstant);
2635 if (!selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast))
2636 return false;
2637 }
2638 if (!selectCopyMemory(I, SrcReg))
2639 return false;
2640 if (ResVReg.isValid() && ResVReg != I.getOperand(0).getReg())
2641 if (!BuildCOPY(ResVReg, I.getOperand(0).getReg(), I))
2642 return false;
2643 return true;
2644}
2645
2646bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
2647 SPIRVTypeInst ResType,
2648 MachineInstr &I,
2649 unsigned NewOpcode,
2650 unsigned NegateOpcode) const {
2651 assert(I.hasOneMemOperand());
2652 const MachineMemOperand *MemOp = *I.memoperands_begin();
2653 uint32_t Scope = static_cast<uint32_t>(
2654 getMemScope(STI.getTargetTriple(), GR.CurMF->getFunction().getContext(),
2655 MemOp->getSyncScopeID()));
2656 Register ScopeReg = buildI32ConstantInEntryBlock(Scope, I);
2657
2658 Register Ptr = I.getOperand(1).getReg();
2659 uint32_t ScSem = static_cast<uint32_t>(
2661 AtomicOrdering AO = MemOp->getSuccessOrdering();
2662 uint32_t OrderSem = static_cast<uint32_t>(getMemSemantics(AO));
2663 Register MemSemReg = buildI32ConstantInEntryBlock(
2664 getMemSemanticsWithStorageClass(STI.getTargetTriple(), OrderSem, ScSem),
2665 I);
2666
2667 Register ValueReg = I.getOperand(2).getReg();
2668 if (NegateOpcode != 0) {
2669 // Translation with negative value operand is requested
2670 Register TmpReg = createVirtualRegister(ResType, &GR, MRI, MRI->getMF());
2671 if (!selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode))
2672 return false;
2673 ValueReg = TmpReg;
2674 }
2675
2676 if (ResType.isTypePtr()) {
2677 if (NewOpcode != SPIRV::OpAtomicExchange)
2678 return diagnoseUnsupported(
2679 I, "Lowering to SPIR-V of this atomic operation is not "
2680 "allowed for pointer types");
2681 if (!STI.isPhysicalSPIRV())
2682 return diagnoseUnsupported(
2683 I, "Lowering to SPIR-V of atomic exchange is only "
2684 "allowed for pointer types for physical addressing model");
2685 // If the exchanged value is a pointer type we convert the value operand to
2686 // an integer type of the same size as the pointer size using
2687 // OpConvertPtrToU, bitcast the Ptr parameter to pointer to integer type and
2688 // then generate OpAtomicExchange on integers. The integer result is
2689 // converted back to a pointer type using OpConvertUToPtr, similar to atomic
2690 // load and store.
2691 MachineIRBuilder MIRBuilder(I);
2692 SPIRV::StorageClass::StorageClass SC = GR.getPointerStorageClass(Ptr);
2693 return selectAtomicPtrValue(
2694 ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) {
2695 Register ValueAsInt = convertPtrToInt(ValueReg, MIRBuilder);
2696 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2697 Register ExchangeResReg = createPtrSizedIntReg(MIRBuilder);
2698 MIRBuilder.buildInstr(SPIRV::OpAtomicExchange)
2699 .addDef(ExchangeResReg)
2700 .addUse(GR.getSPIRVTypeID(IntType))
2701 .addUse(CastedPtr)
2702 .addUse(ScopeReg)
2703 .addUse(MemSemReg)
2704 .addUse(ValueAsInt)
2705 .constrainAllUses(TII, TRI, RBI);
2706 return ExchangeResReg;
2707 });
2708 }
2709
2710 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
2711 .addDef(ResVReg)
2712 .addUse(GR.getSPIRVTypeID(ResType))
2713 .addUse(Ptr)
2714 .addUse(ScopeReg)
2715 .addUse(MemSemReg)
2716 .addUse(ValueReg)
2717 .constrainAllUses(TII, TRI, RBI);
2718 return true;
2719}
2720
2721bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
2722 unsigned ArgI = I.getNumOperands() - 1;
2723 Register SrcReg =
2724 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0);
2725 SPIRVTypeInst SrcType =
2726 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr;
2727 if (!SrcType || !isVectorType(SrcType))
2729 "cannot select G_UNMERGE_VALUES with a non-vector argument");
2730
2731 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(SrcType);
2732 MachineBasicBlock &BB = *I.getParent();
2733 unsigned CurrentIndex = 0;
2734 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2735 Register ResVReg = I.getOperand(i).getReg();
2736 SPIRVTypeInst ResType = GR.getSPIRVTypeForVReg(ResVReg);
2737 if (!ResType) {
2738 LLT ResLLT = MRI->getType(ResVReg);
2739 assert(ResLLT.isValid());
2740 if (ResLLT.isVector()) {
2741 ResType = GR.getOrCreateSPIRVVectorType(
2742 ScalarType, ResLLT.getNumElements(), I, TII);
2743 } else {
2744 ResType = ScalarType;
2745 }
2746 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
2747 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
2748 }
2749
2750 if (isVectorType(ResType)) {
2751 Register UndefReg = GR.getOrCreateUndef(I, SrcType, TII);
2752 auto MIB =
2753 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
2754 .addDef(ResVReg)
2755 .addUse(GR.getSPIRVTypeID(ResType))
2756 .addUse(SrcReg)
2757 .addUse(UndefReg);
2758 unsigned NumElements = GR.getScalarOrVectorComponentCount(ResType);
2759 for (unsigned j = 0; j < NumElements; ++j) {
2760 MIB.addImm(CurrentIndex + j);
2761 }
2762 CurrentIndex += NumElements;
2763 MIB.constrainAllUses(TII, TRI, RBI);
2764 } else {
2765 auto MIB =
2766 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2767 .addDef(ResVReg)
2768 .addUse(GR.getSPIRVTypeID(ResType))
2769 .addUse(SrcReg)
2770 .addImm(CurrentIndex);
2771 CurrentIndex++;
2772 MIB.constrainAllUses(TII, TRI, RBI);
2773 }
2774 }
2775 return true;
2776}
2777
2778bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
2779 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
2780 // In the kernel environment image ordering is requested explicitly via
2781 // CLK_IMAGE_MEM_FENCE, so a fence covers only pointer-addressable memory.
2782 uint32_t ScSem = STI.isShader()
2783 ? SPIRV::MemorySemantics::UniformMemory |
2784 SPIRV::MemorySemantics::WorkgroupMemory |
2785 SPIRV::MemorySemantics::ImageMemory
2786 : SPIRV::MemorySemantics::WorkgroupMemory |
2787 SPIRV::MemorySemantics::CrossWorkgroupMemory;
2788 uint32_t MemSem = getMemSemanticsWithStorageClass(
2789 STI.getTargetTriple(), static_cast<uint32_t>(getMemSemantics(AO)), ScSem);
2790 Register MemSemReg = buildI32ConstantInEntryBlock(MemSem, I);
2791 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
2792 uint32_t Scope = static_cast<uint32_t>(getMemScope(
2793 STI.getTargetTriple(), GR.CurMF->getFunction().getContext(), Ord));
2794 Register ScopeReg = buildI32ConstantInEntryBlock(Scope, I);
2795 MachineBasicBlock &BB = *I.getParent();
2796 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
2797 .addUse(ScopeReg)
2798 .addUse(MemSemReg)
2799 .constrainAllUses(TII, TRI, RBI);
2800 return true;
2801}
2802
2803bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
2804 SPIRVTypeInst ResType,
2805 MachineInstr &I,
2806 unsigned Opcode) const {
2807 Type *ResTy = nullptr;
2808 StringRef ResName;
2809 if (!GR.findValueAttrs(&I, ResTy, ResName))
2810 return diagnoseUnsupported(
2811 I,
2812 "Not enough info to select the arithmetic with overflow instruction");
2813 if (!ResTy || !ResTy->isStructTy())
2814 return diagnoseUnsupported(I,
2815 "Expect struct type result for the arithmetic "
2816 "with overflow instruction");
2817 // "Result Type must be from OpTypeStruct. The struct must have two members,
2818 // and the two members must be the same type."
2819 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
2820 ResTy = StructType::get(ResElemTy, ResElemTy);
2821 // Build SPIR-V types and constant(s) if needed.
2822 MachineIRBuilder MIRBuilder(I);
2823 SPIRVTypeInst StructType = GR.getOrCreateSPIRVType(
2824 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
2825 assert(I.getNumDefs() > 1 && "Not enought operands");
2826 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
2827 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
2828 if (N > 1 || (isVectorType(ResType) &&
2829 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
2830 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
2831 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
2832 Register ZeroReg = buildZerosVal(ResType, I);
2833 // A new virtual register to store the result struct.
2835 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
2836 // Build the result name if needed.
2837 if (ResName.size() > 0)
2838 buildOpName(StructVReg, ResName, MIRBuilder);
2839 // Build the arithmetic with overflow instruction.
2840 MachineBasicBlock &BB = *I.getParent();
2841 auto MIB =
2842 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode))
2843 .addDef(StructVReg)
2844 .addUse(GR.getSPIRVTypeID(StructType));
2845 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
2846 MIB.addUse(I.getOperand(i).getReg());
2847 MIB.constrainAllUses(TII, TRI, RBI);
2848 // Build instructions to extract fields of the instruction's result.
2849 // A new virtual register to store the higher part of the result struct.
2851 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass);
2852 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
2853 auto MIB =
2854 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2855 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg())
2856 .addUse(GR.getSPIRVTypeID(ResType))
2857 .addUse(StructVReg)
2858 .addImm(i);
2859 MIB.constrainAllUses(TII, TRI, RBI);
2860 }
2861 // Build boolean value from the higher part.
2862 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
2863 .addDef(I.getOperand(1).getReg())
2864 .addUse(BoolTypeReg)
2865 .addUse(HigherVReg)
2866 .addUse(ZeroReg)
2867 .constrainAllUses(TII, TRI, RBI);
2868 return true;
2869}
2870
2871bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
2872 SPIRVTypeInst ResType,
2873 MachineInstr &I) const {
2875 "selectAtomicCmpXchg only handles the spv_cmpxchg intrinsic");
2876 Register Ptr = I.getOperand(2).getReg();
2877 Register ScopeReg = I.getOperand(5).getReg();
2878 Register MemSemEqReg = I.getOperand(6).getReg();
2879 Register MemSemNeqReg = I.getOperand(7).getReg();
2880 Register Cmp = I.getOperand(3).getReg();
2881 Register Val = I.getOperand(4).getReg();
2882 SPIRVTypeInst SpvValTy = GR.getSPIRVTypeForVReg(Val);
2883 Register ACmpRes = createVirtualRegister(SpvValTy, &GR, MRI, *I.getMF());
2884 const DebugLoc &DL = I.getDebugLoc();
2885 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
2886 .addDef(ACmpRes)
2887 .addUse(GR.getSPIRVTypeID(SpvValTy))
2888 .addUse(Ptr)
2889 .addUse(ScopeReg)
2890 .addUse(MemSemEqReg)
2891 .addUse(MemSemNeqReg)
2892 .addUse(Val)
2893 .addUse(Cmp)
2894 .constrainAllUses(TII, TRI, RBI);
2895 SPIRVTypeInst BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
2896 Register CmpSuccReg = createVirtualRegister(BoolTy, &GR, MRI, *I.getMF());
2897 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
2898 .addDef(CmpSuccReg)
2899 .addUse(GR.getSPIRVTypeID(BoolTy))
2900 .addUse(ACmpRes)
2901 .addUse(Cmp)
2902 .constrainAllUses(TII, TRI, RBI);
2903 Register TmpReg = createVirtualRegister(ResType, &GR, MRI, *I.getMF());
2904 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
2905 .addDef(TmpReg)
2906 .addUse(GR.getSPIRVTypeID(ResType))
2907 .addUse(ACmpRes)
2908 .addUse(GR.getOrCreateUndef(I, ResType, TII))
2909 .addImm(0)
2910 .constrainAllUses(TII, TRI, RBI);
2911 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
2912 .addDef(ResVReg)
2913 .addUse(GR.getSPIRVTypeID(ResType))
2914 .addUse(CmpSuccReg)
2915 .addUse(TmpReg)
2916 .addImm(1)
2917 .constrainAllUses(TII, TRI, RBI);
2918 return true;
2919}
2920
2921static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
2922 switch (SC) {
2923 case SPIRV::StorageClass::DeviceOnlyINTEL:
2924 case SPIRV::StorageClass::HostOnlyINTEL:
2925 return true;
2926 default:
2927 return false;
2928 }
2929}
2930
2931// Returns true ResVReg is referred only from global vars, constant expressions
2932// and OpName's.
2934 bool IsGRef = false;
2935 bool IsAllowedRefs =
2936 llvm::all_of(MRI->use_instructions(ResVReg), [&IsGRef](auto const &It) {
2937 unsigned Opcode = It.getOpcode();
2938 if (Opcode == SPIRV::OpConstantComposite ||
2939 Opcode == SPIRV::OpSpecConstantComposite ||
2940 Opcode == SPIRV::OpSpecConstantOp || Opcode == SPIRV::OpVariable ||
2941 Opcode == SPIRV::OpUntypedVariableKHR ||
2942 isSpvIntrinsic(It, Intrinsic::spv_init_global))
2943 return IsGRef = true;
2944 return Opcode == SPIRV::OpName;
2945 });
2946 return IsAllowedRefs && IsGRef;
2947}
2948
2949SPIRVTypeInst SPIRVInstructionSelector::getUcharPtrType(
2950 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
2952 Type::getInt8Ty(I.getMF()->getFunction().getContext()), I, SC);
2953}
2954
2955Register SPIRVInstructionSelector::getUcharPtrTypeReg(
2956 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const {
2957 return GR.getSPIRVTypeID(getUcharPtrType(I, SC));
2958}
2959
2960MachineInstrBuilder
2961SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest,
2962 Register Src, Register DestType,
2963 uint32_t Opcode) const {
2964 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
2965 TII.get(SPIRV::OpSpecConstantOp))
2966 .addDef(Dest)
2967 .addUse(DestType)
2968 .addImm(Opcode)
2969 .addUse(Src);
2970}
2971
2972// Bitcast a constant pointer to i8* in its own storage class. Src is returned
2973// unchanged when it is already that pointer, or is not a pointer.
2974Register SPIRVInstructionSelector::bitcastConstPtrToI8(MachineInstr &I,
2975 Register Src) const {
2976 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(Src);
2977 if (!SrcTy || !SrcTy.isPointer())
2978 return Src;
2979 SPIRVTypeInst I8Ty = getUcharPtrType(I, GR.getPointerStorageClass(SrcTy));
2980 if (SrcTy == I8Ty)
2981 return Src;
2982 Register Tmp = createVirtualRegister(I8Ty, &GR, MRI, *I.getMF());
2983 MachineInstrBuilder Cast =
2984 buildSpecConstantOp(I, Tmp, Src, GR.getSPIRVTypeID(I8Ty),
2985 static_cast<uint32_t>(SPIRV::Opcode::Bitcast));
2986 GR.add(Cast.getInstr(), Cast);
2987 Cast.constrainAllUses(TII, TRI, RBI);
2988 return Tmp;
2989}
2990
2991MachineInstrBuilder
2992SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr,
2993 SPIRVTypeInst SrcPtrTy) const {
2994 SPIRVTypeInst GenericPtrTy =
2995 GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I);
2996 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass);
2998 SPIRV::StorageClass::Generic),
2999 GR.getPointerSize()));
3000 MachineFunction *MF = I.getParent()->getParent();
3001 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF);
3002 MachineInstrBuilder MIB = buildSpecConstantOp(
3003 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy),
3004 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric));
3005 GR.add(MIB.getInstr(), MIB);
3006 return MIB;
3007}
3008
3009// In SPIR-V address space casting can only happen to and from the Generic
3010// storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
3011// pointers to and from Generic pointers. As such, we can convert e.g. from
3012// Workgroup to Function by going via a Generic pointer as an intermediary. All
3013// other combinations can only be done by a bitcast, and are probably not safe.
3014bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
3015 SPIRVTypeInst ResType,
3016 MachineInstr &I) const {
3017 MachineBasicBlock &BB = *I.getParent();
3018 const DebugLoc &DL = I.getDebugLoc();
3019
3020 Register SrcPtr = I.getOperand(1).getReg();
3021 SPIRVTypeInst SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
3022
3023 // don't generate a cast for a null that may be represented by OpTypeInt
3024 if (!SrcPtrTy || !SrcPtrTy.isPointer() || !ResType || !ResType.isPointer())
3025 return BuildCOPY(ResVReg, SrcPtr, I);
3026
3027 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy);
3028 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType);
3029
3030 if (isASCastInGVar(MRI, ResVReg)) {
3031 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions
3032 // are expressed by OpSpecConstantOp with an Opcode.
3033 // TODO: maybe insert a check whether the Kernel capability was declared and
3034 // so PtrCastToGeneric/GenericCastToPtr are available.
3035 unsigned SpecOpcode = [&]() -> unsigned {
3036 if (SrcSC == SPIRV::StorageClass::CodeSectionINTEL)
3037 return static_cast<uint32_t>(SPIRV::Opcode::Bitcast);
3038 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
3039 return static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric);
3040 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
3041 return static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr);
3042 return 0u;
3043 }();
3044 // TODO: OpConstantComposite expects i8*, so we are forced to forget a
3045 // correct value of ResType and use general i8* instead. Maybe this should
3046 // be addressed in the emit-intrinsic step to infer a correct
3047 // OpConstantComposite type.
3048 // Generic casts require matching pointee types, so cast the source to i8*
3049 // too, but only if a cast is emitted below.
3050 bool ViaGeneric =
3052 if (SrcSC != SPIRV::StorageClass::CodeSectionINTEL &&
3053 (SpecOpcode || ViaGeneric)) {
3054 SrcPtr = bitcastConstPtrToI8(I, SrcPtr);
3055 SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
3056 }
3057 if (SpecOpcode) {
3058 buildSpecConstantOp(I, ResVReg, SrcPtr, getUcharPtrTypeReg(I, DstSC),
3059 SpecOpcode)
3060 .constrainAllUses(TII, TRI, RBI);
3061 } else if (ViaGeneric) {
3062 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy);
3063 MIB.constrainAllUses(TII, TRI, RBI);
3064 buildSpecConstantOp(
3065 I, ResVReg, MIB->getOperand(0).getReg(), getUcharPtrTypeReg(I, DstSC),
3066 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr))
3067 .constrainAllUses(TII, TRI, RBI);
3068 }
3069 return true;
3070 }
3071
3072 // don't generate a cast between identical storage classes
3073 if (SrcSC == DstSC)
3074 return BuildCOPY(ResVReg, SrcPtr, I);
3075
3076 if ((SrcSC == SPIRV::StorageClass::Function &&
3077 DstSC == SPIRV::StorageClass::Private) ||
3078 (DstSC == SPIRV::StorageClass::Function &&
3079 SrcSC == SPIRV::StorageClass::Private))
3080 return BuildCOPY(ResVReg, SrcPtr, I);
3081
3082 // Casting from an eligible pointer to Generic.
3083 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
3084 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
3085 // Casting from Generic to an eligible pointer.
3086 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
3087 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
3088 // Casting between 2 eligible pointers using Generic as an intermediary.
3089 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
3090 SPIRVTypeInst GenericPtrTy =
3091 GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I);
3092 Register Tmp = createVirtualRegister(GenericPtrTy, &GR, MRI, MRI->getMF());
3093 BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
3094 .addDef(Tmp)
3095 .addUse(GR.getSPIRVTypeID(GenericPtrTy))
3096 .addUse(SrcPtr)
3097 .constrainAllUses(TII, TRI, RBI);
3098 BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
3099 .addDef(ResVReg)
3100 .addUse(GR.getSPIRVTypeID(ResType))
3101 .addUse(Tmp)
3102 .constrainAllUses(TII, TRI, RBI);
3103 return true;
3104 }
3105
3106 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
3107 // be applied
3108 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
3109 return selectUnOp(ResVReg, ResType, I,
3110 SPIRV::OpPtrCastToCrossWorkgroupINTEL);
3111 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC))
3112 return selectUnOp(ResVReg, ResType, I,
3113 SPIRV::OpCrossWorkgroupCastToPtrINTEL);
3114 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic)
3115 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
3116 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC))
3117 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
3118
3119 // Bitcast for pointers requires that the address spaces must match
3120 return false;
3121}
3122
3123// G_PTRMASK - Apply a bitmask to a pointer value.
3124// Result = Ptr & Mask
3125// We need to convert the pointer to an integer, perform the AND operation,
3126// and convert back to a pointer.
3127bool SPIRVInstructionSelector::selectPtrMask(Register ResVReg,
3128 SPIRVTypeInst ResType,
3129 MachineInstr &I) const {
3130 if (STI.isLogicalSPIRV())
3131 return diagnoseUnsupported(
3132 I, "G_PTRMASK is not supported with logical SPIR-V");
3133 MachineBasicBlock &BB = *I.getParent();
3134 MachineFunction &MF = *BB.getParent();
3135 const DebugLoc &DL = I.getDebugLoc();
3136
3137 Register PtrReg = I.getOperand(1).getReg();
3138 Register MaskReg = I.getOperand(2).getReg();
3139
3140 SPIRVTypeInst MaskType = GR.getSPIRVTypeForVReg(MaskReg);
3141
3142 // Convert pointer to integer.
3143 Register PtrAsInt = MRI->createVirtualRegister(GR.getRegClass(MaskType));
3144 GR.assignSPIRVTypeToVReg(MaskType, PtrAsInt, MF);
3145
3146 BuildMI(BB, I, DL, TII.get(SPIRV::OpConvertPtrToU))
3147 .addDef(PtrAsInt)
3148 .addUse(GR.getSPIRVTypeID(MaskType))
3149 .addUse(PtrReg)
3150 .constrainAllUses(TII, TRI, RBI);
3151
3152 // Perform bitwise AND.
3153 Register MaskedInt = MRI->createVirtualRegister(GR.getRegClass(MaskType));
3154 GR.assignSPIRVTypeToVReg(MaskType, MaskedInt, MF);
3155
3156 unsigned AndOpcode = GR.getScalarOrVectorComponentCount(MaskType) > 1
3157 ? SPIRV::OpBitwiseAndV
3158 : SPIRV::OpBitwiseAndS;
3159
3160 BuildMI(BB, I, DL, TII.get(AndOpcode))
3161 .addDef(MaskedInt)
3162 .addUse(GR.getSPIRVTypeID(MaskType))
3163 .addUse(PtrAsInt)
3164 .addUse(MaskReg)
3165 .constrainAllUses(TII, TRI, RBI);
3166
3167 // Convert integer back to pointer.
3168 BuildMI(BB, I, DL, TII.get(SPIRV::OpConvertUToPtr))
3169 .addDef(ResVReg)
3170 .addUse(GR.getSPIRVTypeID(ResType))
3171 .addUse(MaskedInt)
3172 .constrainAllUses(TII, TRI, RBI);
3173
3174 return true;
3175}
3176
3177static unsigned getFCmpOpcode(unsigned PredNum) {
3178 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
3179 switch (Pred) {
3180 case CmpInst::FCMP_OEQ:
3181 return SPIRV::OpFOrdEqual;
3182 case CmpInst::FCMP_OGE:
3183 return SPIRV::OpFOrdGreaterThanEqual;
3184 case CmpInst::FCMP_OGT:
3185 return SPIRV::OpFOrdGreaterThan;
3186 case CmpInst::FCMP_OLE:
3187 return SPIRV::OpFOrdLessThanEqual;
3188 case CmpInst::FCMP_OLT:
3189 return SPIRV::OpFOrdLessThan;
3190 case CmpInst::FCMP_ONE:
3191 return SPIRV::OpFOrdNotEqual;
3192 case CmpInst::FCMP_ORD:
3193 return SPIRV::OpOrdered;
3194 case CmpInst::FCMP_UEQ:
3195 return SPIRV::OpFUnordEqual;
3196 case CmpInst::FCMP_UGE:
3197 return SPIRV::OpFUnordGreaterThanEqual;
3198 case CmpInst::FCMP_UGT:
3199 return SPIRV::OpFUnordGreaterThan;
3200 case CmpInst::FCMP_ULE:
3201 return SPIRV::OpFUnordLessThanEqual;
3202 case CmpInst::FCMP_ULT:
3203 return SPIRV::OpFUnordLessThan;
3204 case CmpInst::FCMP_UNE:
3205 return SPIRV::OpFUnordNotEqual;
3206 case CmpInst::FCMP_UNO:
3207 return SPIRV::OpUnordered;
3208 default:
3209 llvm_unreachable("Unknown predicate type for FCmp");
3210 }
3211}
3212
3213static unsigned getICmpOpcode(unsigned PredNum) {
3214 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
3215 switch (Pred) {
3216 case CmpInst::ICMP_EQ:
3217 return SPIRV::OpIEqual;
3218 case CmpInst::ICMP_NE:
3219 return SPIRV::OpINotEqual;
3220 case CmpInst::ICMP_SGE:
3221 return SPIRV::OpSGreaterThanEqual;
3222 case CmpInst::ICMP_SGT:
3223 return SPIRV::OpSGreaterThan;
3224 case CmpInst::ICMP_SLE:
3225 return SPIRV::OpSLessThanEqual;
3226 case CmpInst::ICMP_SLT:
3227 return SPIRV::OpSLessThan;
3228 case CmpInst::ICMP_UGE:
3229 return SPIRV::OpUGreaterThanEqual;
3230 case CmpInst::ICMP_UGT:
3231 return SPIRV::OpUGreaterThan;
3232 case CmpInst::ICMP_ULE:
3233 return SPIRV::OpULessThanEqual;
3234 case CmpInst::ICMP_ULT:
3235 return SPIRV::OpULessThan;
3236 default:
3237 llvm_unreachable("Unknown predicate type for ICmp");
3238 }
3239}
3240
3241static unsigned getPtrCmpOpcode(unsigned Pred) {
3242 switch (static_cast<CmpInst::Predicate>(Pred)) {
3243 case CmpInst::ICMP_EQ:
3244 return SPIRV::OpPtrEqual;
3245 case CmpInst::ICMP_NE:
3246 return SPIRV::OpPtrNotEqual;
3247 default:
3248 llvm_unreachable("Unknown predicate type for pointer comparison");
3249 }
3250}
3251
3252// Return the logical operation, or abort if none exists.
3253static unsigned getBoolCmpOpcode(unsigned PredNum) {
3254 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
3255 switch (Pred) {
3256 case CmpInst::ICMP_EQ:
3257 return SPIRV::OpLogicalEqual;
3258 case CmpInst::ICMP_NE:
3259 return SPIRV::OpLogicalNotEqual;
3260 default:
3261 llvm_unreachable("Unknown predicate type for Bool comparison");
3262 }
3263}
3264
3265static APFloat getZeroFP(const Type *LLVMFloatTy) {
3266 if (!LLVMFloatTy)
3268 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
3269 case Type::HalfTyID:
3271 case Type::BFloatTyID:
3273 default:
3274 case Type::FloatTyID:
3276 case Type::DoubleTyID:
3278 }
3279}
3280
3281static APFloat getOneFP(const Type *LLVMFloatTy) {
3282 if (!LLVMFloatTy)
3284 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
3285 case Type::HalfTyID:
3287 case Type::BFloatTyID:
3289 default:
3290 case Type::FloatTyID:
3292 case Type::DoubleTyID:
3294 }
3295}
3296
3297bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
3298 SPIRVTypeInst ResType,
3299 MachineInstr &I,
3300 unsigned OpAnyOrAll) const {
3301 assert(I.getNumOperands() == 3);
3302 assert(I.getOperand(2).isReg());
3303 MachineBasicBlock &BB = *I.getParent();
3304 Register InputRegister = I.getOperand(2).getReg();
3305 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3306
3307 assert(InputType && "VReg has no type assigned");
3308
3309 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool);
3310 if (IsBoolTy && !isVectorType(InputType)) {
3311 assert(ResVReg == I.getOperand(0).getReg());
3312 return BuildCOPY(ResVReg, InputRegister, I);
3313 }
3314
3315 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3316 unsigned SpirvNotEqualId =
3317 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
3318 SPIRVTypeInst SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
3319 SPIRVTypeInst SpvBoolTy = SpvBoolScalarTy;
3320 Register NotEqualReg = ResVReg;
3321
3322 if (isVectorType(InputType)) {
3323 NotEqualReg =
3324 IsBoolTy ? InputRegister
3325 : createVirtualRegister(SpvBoolTy, &GR, MRI, MRI->getMF());
3326 const unsigned NumElts = GR.getScalarOrVectorComponentCount(InputType);
3327 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII);
3328 }
3329
3330 if (!IsBoolTy) {
3331 Register ConstZeroReg =
3332 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I);
3333
3334 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId))
3335 .addDef(NotEqualReg)
3336 .addUse(GR.getSPIRVTypeID(SpvBoolTy))
3337 .addUse(InputRegister)
3338 .addUse(ConstZeroReg)
3339 .constrainAllUses(TII, TRI, RBI);
3340 }
3341
3342 if (isVectorType(InputType))
3343 BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
3344 .addDef(ResVReg)
3345 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy))
3346 .addUse(NotEqualReg)
3347 .constrainAllUses(TII, TRI, RBI);
3348 return true;
3349}
3350
3351bool SPIRVInstructionSelector::selectAll(Register ResVReg,
3352 SPIRVTypeInst ResType,
3353 MachineInstr &I) const {
3354 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
3355}
3356
3357bool SPIRVInstructionSelector::selectAny(Register ResVReg,
3358 SPIRVTypeInst ResType,
3359 MachineInstr &I) const {
3360 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
3361}
3362
3363// Select the OpDot instruction for the given float dot
3364bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
3365 SPIRVTypeInst ResType,
3366 MachineInstr &I) const {
3367 assert(I.getNumOperands() == 4);
3368 assert(I.getOperand(2).isReg());
3369 assert(I.getOperand(3).isReg());
3370
3371 [[maybe_unused]] SPIRVTypeInst VecType =
3372 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg());
3373
3374 assert(((VecType->getOpcode() == SPIRV::OpTypeVector &&
3375 GR.getScalarOrVectorComponentCount(VecType) > 1) ||
3376 VecType->getOpcode() == SPIRV::OpTypeVectorIdEXT) &&
3377 "dot product requires either a vector of at least 2 components or"
3378 " the SPV_EXT_long vector extension.");
3379
3380 [[maybe_unused]] SPIRVTypeInst EltType =
3382
3383 assert(EltType->getOpcode() == SPIRV::OpTypeFloat);
3384
3385 MachineBasicBlock &BB = *I.getParent();
3386 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot))
3387 .addDef(ResVReg)
3388 .addUse(GR.getSPIRVTypeID(ResType))
3389 .addUse(I.getOperand(2).getReg())
3390 .addUse(I.getOperand(3).getReg())
3391 .constrainAllUses(TII, TRI, RBI);
3392 return true;
3393}
3394
3395bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg,
3396 SPIRVTypeInst ResType,
3397 MachineInstr &I,
3398 bool Signed) const {
3399 assert(I.getNumOperands() == 4);
3400 assert(I.getOperand(2).isReg());
3401 assert(I.getOperand(3).isReg());
3402 MachineBasicBlock &BB = *I.getParent();
3403
3404 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
3405 BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
3406 .addDef(ResVReg)
3407 .addUse(GR.getSPIRVTypeID(ResType))
3408 .addUse(I.getOperand(2).getReg())
3409 .addUse(I.getOperand(3).getReg())
3410 .constrainAllUses(TII, TRI, RBI);
3411 return true;
3412}
3413
3414// Since pre-1.6 SPIRV has no integer dot implementation,
3415// expand by piecewise multiplying and adding the results
3416bool SPIRVInstructionSelector::selectIntegerDotExpansion(
3417 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3418 assert(I.getNumOperands() == 4);
3419 assert(I.getOperand(2).isReg());
3420 assert(I.getOperand(3).isReg());
3421 MachineBasicBlock &BB = *I.getParent();
3422
3423 // Multiply the vectors, then sum the results
3424 Register Vec0 = I.getOperand(2).getReg();
3425 Register Vec1 = I.getOperand(3).getReg();
3426 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType));
3427 SPIRVTypeInst VecType = GR.getSPIRVTypeForVReg(Vec0);
3428
3429 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV))
3430 .addDef(TmpVec)
3431 .addUse(GR.getSPIRVTypeID(VecType))
3432 .addUse(Vec0)
3433 .addUse(Vec1)
3434 .constrainAllUses(TII, TRI, RBI);
3435
3436 assert(((VecType->getOpcode() == SPIRV::OpTypeVector &&
3437 GR.getScalarOrVectorComponentCount(VecType) > 1) ||
3438 VecType->getOpcode() == SPIRV::OpTypeVectorIdEXT) &&
3439 "dot product requires either a vector of at least 2 components "
3440 "or the SPV_EXT_long_vector extension.");
3441
3442 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType));
3443 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
3444 .addDef(Res)
3445 .addUse(GR.getSPIRVTypeID(ResType))
3446 .addUse(TmpVec)
3447 .addImm(0)
3448 .constrainAllUses(TII, TRI, RBI);
3449
3450 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) {
3451 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType));
3452
3453 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
3454 .addDef(Elt)
3455 .addUse(GR.getSPIRVTypeID(ResType))
3456 .addUse(TmpVec)
3457 .addImm(i)
3458 .constrainAllUses(TII, TRI, RBI);
3459
3460 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1
3461 ? MRI->createVirtualRegister(GR.getRegClass(ResType))
3462 : ResVReg;
3463
3464 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3465 .addDef(Sum)
3466 .addUse(GR.getSPIRVTypeID(ResType))
3467 .addUse(Res)
3468 .addUse(Elt)
3469 .constrainAllUses(TII, TRI, RBI);
3470 Res = Sum;
3471 }
3472
3473 return true;
3474}
3475
3476bool SPIRVInstructionSelector::selectOpIsInf(Register ResVReg,
3477 SPIRVTypeInst ResType,
3478 MachineInstr &I) const {
3479 MachineBasicBlock &BB = *I.getParent();
3480 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsInf))
3481 .addDef(ResVReg)
3482 .addUse(GR.getSPIRVTypeID(ResType))
3483 .addUse(I.getOperand(2).getReg())
3484 .constrainAllUses(TII, TRI, RBI);
3485 return true;
3486}
3487
3488bool SPIRVInstructionSelector::selectOpIsNan(Register ResVReg,
3489 SPIRVTypeInst ResType,
3490 MachineInstr &I) const {
3491 MachineBasicBlock &BB = *I.getParent();
3492 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNan))
3493 .addDef(ResVReg)
3494 .addUse(GR.getSPIRVTypeID(ResType))
3495 .addUse(I.getOperand(2).getReg())
3496 .constrainAllUses(TII, TRI, RBI);
3497 return true;
3498}
3499
3500bool SPIRVInstructionSelector::selectOpIsFinite(Register ResVReg,
3501 SPIRVTypeInst ResType,
3502 MachineInstr &I) const {
3503 // OpIsFinite requires Kernel capability; emulate with OpIsInf & OpIsNan
3504 MachineBasicBlock &BB = *I.getParent();
3505 Register Src = I.getOperand(2).getReg();
3506 Register TypeID = GR.getSPIRVTypeID(ResType);
3507 const DebugLoc &DL = I.getDebugLoc();
3508
3509 if (!STI.isShader()) {
3510 BuildMI(BB, I, DL, TII.get(SPIRV::OpIsFinite))
3511 .addDef(ResVReg)
3512 .addUse(TypeID)
3513 .addUse(Src)
3514 .constrainAllUses(TII, TRI, RBI);
3515 return true;
3516 }
3517
3518 Register IsInfReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3519 BuildMI(BB, I, DL, TII.get(SPIRV::OpIsInf))
3520 .addDef(IsInfReg)
3521 .addUse(TypeID)
3522 .addUse(Src)
3523 .constrainAllUses(TII, TRI, RBI);
3524
3525 Register IsNanReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3526 BuildMI(BB, I, DL, TII.get(SPIRV::OpIsNan))
3527 .addDef(IsNanReg)
3528 .addUse(TypeID)
3529 .addUse(Src)
3530 .constrainAllUses(TII, TRI, RBI);
3531
3532 Register OrReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3533 BuildMI(BB, I, DL, TII.get(SPIRV::OpLogicalOr))
3534 .addDef(OrReg)
3535 .addUse(TypeID)
3536 .addUse(IsInfReg)
3537 .addUse(IsNanReg)
3538 .constrainAllUses(TII, TRI, RBI);
3539
3540 BuildMI(BB, I, DL, TII.get(SPIRV::OpLogicalNot))
3541 .addDef(ResVReg)
3542 .addUse(TypeID)
3543 .addUse(OrReg)
3544 .constrainAllUses(TII, TRI, RBI);
3545 return true;
3546}
3547
3548bool SPIRVInstructionSelector::selectOpIsNormal(Register ResVReg,
3549 SPIRVTypeInst ResType,
3550 MachineInstr &I) const {
3551 MachineBasicBlock &BB = *I.getParent();
3552 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNormal))
3553 .addDef(ResVReg)
3554 .addUse(GR.getSPIRVTypeID(ResType))
3555 .addUse(I.getOperand(2).getReg())
3556 .constrainAllUses(TII, TRI, RBI);
3557 return true;
3558}
3559
3560template <bool Signed>
3561bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg,
3562 SPIRVTypeInst ResType,
3563 MachineInstr &I) const {
3564 assert(I.getNumOperands() == 5);
3565 assert(I.getOperand(2).isReg());
3566 assert(I.getOperand(3).isReg());
3567 assert(I.getOperand(4).isReg());
3568 MachineBasicBlock &BB = *I.getParent();
3569
3570 Register Acc = I.getOperand(2).getReg();
3571 Register X = I.getOperand(3).getReg();
3572 Register Y = I.getOperand(4).getReg();
3573
3574 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
3575 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType));
3576 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp))
3577 .addDef(Dot)
3578 .addUse(GR.getSPIRVTypeID(ResType))
3579 .addUse(X)
3580 .addUse(Y);
3581 MIB.addImm(SPIRV::BuiltIn::PackedVectorFormat4x8Bit);
3582 MIB.constrainAllUses(TII, TRI, RBI);
3583
3584 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3585 .addDef(ResVReg)
3586 .addUse(GR.getSPIRVTypeID(ResType))
3587 .addUse(Dot)
3588 .addUse(Acc)
3589 .constrainAllUses(TII, TRI, RBI);
3590 return true;
3591}
3592
3593// Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation,
3594// extract the elements of the packed inputs, multiply them and add the result
3595// to the accumulator.
3596template <bool Signed>
3597bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
3598 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3599 assert(I.getNumOperands() == 5);
3600 assert(I.getOperand(2).isReg());
3601 assert(I.getOperand(3).isReg());
3602 assert(I.getOperand(4).isReg());
3603 MachineBasicBlock &BB = *I.getParent();
3604
3605 Register Acc = I.getOperand(2).getReg();
3606 Register X = I.getOperand(3).getReg();
3607 Register Y = I.getOperand(4).getReg();
3608
3609 SPIRVTypeInst EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII);
3610 auto ExtractOp =
3611 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
3612
3613 bool ZeroAsNull = !STI.isShader();
3614 // Extract the i8 element, multiply and add it to the accumulator
3615 for (unsigned i = 0; i < 4; i++) {
3616 // A[i]
3617 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3618 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3619 .addDef(AElt)
3620 .addUse(GR.getSPIRVTypeID(ResType))
3621 .addUse(X)
3622 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull))
3623 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3624 .constrainAllUses(TII, TRI, RBI);
3625
3626 // B[i]
3627 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3628 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3629 .addDef(BElt)
3630 .addUse(GR.getSPIRVTypeID(ResType))
3631 .addUse(Y)
3632 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull))
3633 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3634 .constrainAllUses(TII, TRI, RBI);
3635
3636 // A[i] * B[i]
3637 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3638 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS))
3639 .addDef(Mul)
3640 .addUse(GR.getSPIRVTypeID(ResType))
3641 .addUse(AElt)
3642 .addUse(BElt)
3643 .constrainAllUses(TII, TRI, RBI);
3644
3645 // Discard 24 highest-bits so that stored i32 register is i8 equivalent
3646 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3647 BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp))
3648 .addDef(MaskMul)
3649 .addUse(GR.getSPIRVTypeID(ResType))
3650 .addUse(Mul)
3651 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII, ZeroAsNull))
3652 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull))
3653 .constrainAllUses(TII, TRI, RBI);
3654
3655 // Acc = Acc + A[i] * B[i]
3656 Register Sum =
3657 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg;
3658 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
3659 .addDef(Sum)
3660 .addUse(GR.getSPIRVTypeID(ResType))
3661 .addUse(Acc)
3662 .addUse(MaskMul)
3663 .constrainAllUses(TII, TRI, RBI);
3664
3665 Acc = Sum;
3666 }
3667
3668 return true;
3669}
3670
3671/// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV
3672/// does not have a saturate builtin.
3673bool SPIRVInstructionSelector::selectSaturate(Register ResVReg,
3674 SPIRVTypeInst ResType,
3675 MachineInstr &I) const {
3676 assert(I.getNumOperands() == 3);
3677 assert(I.getOperand(2).isReg());
3678 MachineBasicBlock &BB = *I.getParent();
3679 Register VZero = buildZerosValF(ResType, I);
3680 Register VOne = buildOnesValF(ResType, I);
3681
3682 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
3683 .addDef(ResVReg)
3684 .addUse(GR.getSPIRVTypeID(ResType))
3685 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3686 .addImm(GL::FClamp)
3687 .addUse(I.getOperand(2).getReg())
3688 .addUse(VZero)
3689 .addUse(VOne)
3690 .constrainAllUses(TII, TRI, RBI);
3691 return true;
3692}
3693
3694bool SPIRVInstructionSelector::selectSign(Register ResVReg,
3695 SPIRVTypeInst ResType,
3696 MachineInstr &I) const {
3697 assert(I.getNumOperands() == 3);
3698 assert(I.getOperand(2).isReg());
3699 MachineBasicBlock &BB = *I.getParent();
3700 Register InputRegister = I.getOperand(2).getReg();
3701 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3702 auto &DL = I.getDebugLoc();
3703
3704 if (!InputType)
3705 return diagnoseUnsupported(I, "Input Type could not be determined.");
3706
3707 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3708
3709 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType);
3710 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType);
3711
3712 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
3713
3714 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
3715 Register SignReg = NeedsConversion
3716 ? MRI->createVirtualRegister(&SPIRV::IDRegClass)
3717 : ResVReg;
3718
3719 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst))
3720 .addDef(SignReg)
3721 .addUse(GR.getSPIRVTypeID(InputType))
3722 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
3723 .addImm(SignOpcode)
3724 .addUse(InputRegister)
3725 .constrainAllUses(TII, TRI, RBI);
3726
3727 if (NeedsConversion) {
3728 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
3729 BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode))
3730 .addDef(ResVReg)
3731 .addUse(GR.getSPIRVTypeID(ResType))
3732 .addUse(SignReg)
3733 .constrainAllUses(TII, TRI, RBI);
3734 }
3735
3736 return true;
3737}
3738
3739bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg,
3740 SPIRVTypeInst ResType,
3741 MachineInstr &I,
3742 unsigned Opcode) const {
3743 MachineBasicBlock &BB = *I.getParent();
3744 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3745
3746 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
3747 .addDef(ResVReg)
3748 .addUse(GR.getSPIRVTypeID(ResType))
3749 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I,
3750 IntTy, TII, !STI.isShader()));
3751
3752 for (unsigned J = 2; J < I.getNumOperands(); J++) {
3753 BMI.addUse(I.getOperand(J).getReg());
3754 }
3755
3756 BMI.constrainAllUses(TII, TRI, RBI);
3757 return true;
3758}
3759
3760bool SPIRVInstructionSelector::selectBarrierInst(MachineInstr &I,
3761 unsigned Scope,
3762 unsigned MemSem,
3763 bool WithGroupSync) const {
3764 auto BarrierType =
3765 WithGroupSync ? SPIRV::OpControlBarrier : SPIRV::OpMemoryBarrier;
3766
3767 MemSem |= SPIRV::MemorySemantics::AcquireRelease;
3768
3769 assert(((Scope != SPIRV::Scope::Workgroup) ||
3770 ((MemSem & SPIRV::MemorySemantics::WorkgroupMemory) > 0)) &&
3771 "Workgroup Scope must set WorkGroupMemory semantic "
3772 "in Barrier instruction");
3773
3774 assert(((Scope != SPIRV::Scope::Device) ||
3775 ((MemSem & SPIRV::MemorySemantics::UniformMemory) > 0 &&
3776 (MemSem & SPIRV::MemorySemantics::ImageMemory) > 0)) &&
3777 "Device Scope must set UniformMemory and ImageMemory semantic "
3778 "in Barrier instruction");
3779
3780 MachineBasicBlock &BB = *I.getParent();
3781 auto MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType));
3782
3783 // OpControlBarrier needs to also set Execution Scope
3784 if (WithGroupSync) {
3785 Register ExecReg = buildI32ConstantInEntryBlock(SPIRV::Scope::Workgroup, I);
3786 MI.addUse(ExecReg);
3787 }
3788
3789 Register ScopeReg = buildI32ConstantInEntryBlock(Scope, I);
3790 Register MemSemReg = buildI32ConstantInEntryBlock(MemSem, I);
3791
3792 MI.addUse(ScopeReg).addUse(MemSemReg).constrainAllUses(TII, TRI, RBI);
3793 return true;
3794}
3795
3796bool SPIRVInstructionSelector::selectWaveActiveCountBits(
3797 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
3798
3799 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3800 SPIRVTypeInst BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII);
3801 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType));
3802 if (!selectWaveOpInst(BallotReg, BallotType, I,
3803 SPIRV::OpGroupNonUniformBallot))
3804 return false;
3805
3806 MachineBasicBlock &BB = *I.getParent();
3807 BuildMI(BB, I, I.getDebugLoc(),
3808 TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3809 .addDef(ResVReg)
3810 .addUse(GR.getSPIRVTypeID(ResType))
3811 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
3812 !STI.isShader()))
3813 .addImm(SPIRV::GroupOperation::Reduce)
3814 .addUse(BallotReg)
3815 .constrainAllUses(TII, TRI, RBI);
3816
3817 return true;
3818}
3819
3820bool SPIRVInstructionSelector::selectWaveActiveAllEqual(Register ResVReg,
3821 SPIRVTypeInst ResType,
3822 MachineInstr &I) const {
3823 MachineBasicBlock &BB = *I.getParent();
3824 const DebugLoc &DL = I.getDebugLoc();
3825
3826 // Input to the intrinsic
3827 Register InputReg = I.getOperand(2).getReg();
3828 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputReg);
3829
3830 // Determine if input is vector
3831 unsigned NumElems = GR.getScalarOrVectorComponentCount(InputType);
3832 bool IsVector = NumElems > 1 ||
3833 (InputType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
3834 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector));
3835
3836 // Determine element types
3837 SPIRVTypeInst ElemInputType = GR.getScalarOrVectorComponentType(InputType);
3838 SPIRVTypeInst ElemBoolType = GR.getScalarOrVectorComponentType(ResType);
3839
3840 // Subgroup scope constant
3841 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3842 Register ScopeConst = GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy,
3843 TII, !STI.isShader());
3844
3845 // Scalar case
3846 if (!IsVector) {
3847 return selectWaveOpInst(ResVReg, ElemBoolType, I,
3848 SPIRV::OpGroupNonUniformAllEqual);
3849 }
3850
3851 // Vector case
3852 SmallVector<Register, 4> ElementResults;
3853 ElementResults.reserve(NumElems);
3854
3855 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
3856 // Extract element
3857 Register ElemInput = InputReg;
3858 Register Extracted =
3859 MRI->createVirtualRegister(GR.getRegClass(ElemInputType));
3860
3861 BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeExtract))
3862 .addDef(Extracted)
3863 .addUse(GR.getSPIRVTypeID(ElemInputType))
3864 .addUse(InputReg)
3865 .addImm(Idx)
3866 .constrainAllUses(TII, TRI, RBI);
3867
3868 ElemInput = Extracted;
3869
3870 // Emit per-element AllEqual
3871 Register ElemResult =
3872 MRI->createVirtualRegister(GR.getRegClass(ElemBoolType));
3873
3874 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformAllEqual))
3875 .addDef(ElemResult)
3876 .addUse(GR.getSPIRVTypeID(ElemBoolType))
3877 .addUse(ScopeConst)
3878 .addUse(ElemInput)
3879 .constrainAllUses(TII, TRI, RBI);
3880
3881 ElementResults.push_back(ElemResult);
3882 }
3883
3884 // Reconstruct vector<bool>
3885 auto MIB = BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeConstruct))
3886 .addDef(ResVReg)
3887 .addUse(GR.getSPIRVTypeID(ResType));
3888 for (Register R : ElementResults)
3889 MIB.addUse(R);
3890
3891 MIB.constrainAllUses(TII, TRI, RBI);
3892
3893 return true;
3894}
3895
3896bool SPIRVInstructionSelector::selectWavePrefixBitCount(Register ResVReg,
3897 SPIRVTypeInst ResType,
3898 MachineInstr &I) const {
3899
3900 assert(I.getNumOperands() == 3);
3901
3902 auto Op = I.getOperand(2);
3903 assert(Op.isReg());
3904
3905 MachineBasicBlock &BB = *I.getParent();
3906 DebugLoc DL = I.getDebugLoc();
3907
3908 Register InputRegister = Op.getReg();
3909 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
3910
3911 if (!InputType)
3912 return diagnoseUnsupported(I, "Input Type could not be determined.");
3913
3914 if (InputType->getOpcode() != SPIRV::OpTypeBool)
3915 return diagnoseUnsupported(I, "WavePrefixBitCount requires boolean input");
3916
3917 // Types
3918 SPIRVTypeInst Int32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
3919
3920 // Ballot result type: vector<uint32>
3921 // Match DXC: %v4uint for Subgroup size
3922 SPIRVTypeInst BallotTy = GR.getOrCreateSPIRVVectorType(Int32Ty, 4, I, TII);
3923
3924 // Create a vreg for the ballot result
3925 Register BallotVReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
3926
3927 // 1. OpGroupNonUniformBallot
3928 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallot))
3929 .addDef(BallotVReg)
3930 .addUse(GR.getSPIRVTypeID(BallotTy))
3931 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII))
3932 .addUse(InputRegister)
3933 .constrainAllUses(TII, TRI, RBI);
3934
3935 // 2. OpGroupNonUniformBallotBitCount
3936 BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3937 .addDef(ResVReg)
3938 .addUse(GR.getSPIRVTypeID(ResType))
3939 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII))
3940 .addImm(SPIRV::GroupOperation::ExclusiveScan)
3941 .addUse(BallotVReg)
3942 .constrainAllUses(TII, TRI, RBI);
3943
3944 return true;
3945}
3946
3947bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg,
3948 SPIRVTypeInst ResType,
3949 MachineInstr &I,
3950 bool IsUnsigned) const {
3951 return selectWaveReduce(
3952 ResVReg, ResType, I, IsUnsigned,
3953 [&](Register InputRegister, bool IsUnsigned) {
3954 const bool IsFloatTy =
3955 GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3956 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax
3957 : SPIRV::OpGroupNonUniformSMax;
3958 return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp;
3959 });
3960}
3961
3962bool SPIRVInstructionSelector::selectWaveReduceMin(Register ResVReg,
3963 SPIRVTypeInst ResType,
3964 MachineInstr &I,
3965 bool IsUnsigned) const {
3966 return selectWaveReduce(
3967 ResVReg, ResType, I, IsUnsigned,
3968 [&](Register InputRegister, bool IsUnsigned) {
3969 const bool IsFloatTy =
3970 GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
3971 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin
3972 : SPIRV::OpGroupNonUniformSMin;
3973 return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp;
3974 });
3975}
3976
3977bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg,
3978 SPIRVTypeInst ResType,
3979 MachineInstr &I) const {
3980 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3981 [&](Register InputRegister, bool IsUnsigned) {
3982 bool IsFloatTy = GR.isScalarOrVectorOfType(
3983 InputRegister, SPIRV::OpTypeFloat);
3984 return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
3985 : SPIRV::OpGroupNonUniformIAdd;
3986 });
3987}
3988
3989bool SPIRVInstructionSelector::selectWaveReduceProduct(Register ResVReg,
3990 SPIRVTypeInst ResType,
3991 MachineInstr &I) const {
3992 return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false,
3993 [&](Register InputRegister, bool IsUnsigned) {
3994 bool IsFloatTy = GR.isScalarOrVectorOfType(
3995 InputRegister, SPIRV::OpTypeFloat);
3996 return IsFloatTy ? SPIRV::OpGroupNonUniformFMul
3997 : SPIRV::OpGroupNonUniformIMul;
3998 });
3999}
4000
4001template <typename PickOpcodeFn>
4002bool SPIRVInstructionSelector::selectWaveReduce(
4003 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
4004 PickOpcodeFn &&PickOpcode) const {
4005 assert(I.getNumOperands() == 3);
4006 assert(I.getOperand(2).isReg());
4007 MachineBasicBlock &BB = *I.getParent();
4008 Register InputRegister = I.getOperand(2).getReg();
4009 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
4010
4011 if (!InputType)
4012 return diagnoseUnsupported(I, "Input Type could not be determined.");
4013
4014 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4015 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
4016 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4017 .addDef(ResVReg)
4018 .addUse(GR.getSPIRVTypeID(ResType))
4019 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
4020 !STI.isShader()))
4021 .addImm(SPIRV::GroupOperation::Reduce)
4022 .addUse(I.getOperand(2).getReg())
4023 .constrainAllUses(TII, TRI, RBI);
4024 return true;
4025}
4026
4027bool SPIRVInstructionSelector::selectWaveReduceOp(Register ResVReg,
4028 SPIRVTypeInst ResType,
4029 MachineInstr &I,
4030 unsigned Opcode) const {
4031 return selectWaveReduce(
4032 ResVReg, ResType, I, false,
4033 [&](Register InputRegister, bool IsUnsigned) { return Opcode; });
4034}
4035
4036bool SPIRVInstructionSelector::selectWaveExclusiveScanSum(
4037 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4038 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
4039 [&](Register InputRegister, bool IsUnsigned) {
4040 bool IsFloatTy = GR.isScalarOrVectorOfType(
4041 InputRegister, SPIRV::OpTypeFloat);
4042 return IsFloatTy
4043 ? SPIRV::OpGroupNonUniformFAdd
4044 : SPIRV::OpGroupNonUniformIAdd;
4045 });
4046}
4047
4048bool SPIRVInstructionSelector::selectWaveExclusiveScanProduct(
4049 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
4050 return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false,
4051 [&](Register InputRegister, bool IsUnsigned) {
4052 bool IsFloatTy = GR.isScalarOrVectorOfType(
4053 InputRegister, SPIRV::OpTypeFloat);
4054 return IsFloatTy
4055 ? SPIRV::OpGroupNonUniformFMul
4056 : SPIRV::OpGroupNonUniformIMul;
4057 });
4058}
4059
4060template <typename PickOpcodeFn>
4061bool SPIRVInstructionSelector::selectWaveExclusiveScan(
4062 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned,
4063 PickOpcodeFn &&PickOpcode) const {
4064 assert(I.getNumOperands() == 3);
4065 assert(I.getOperand(2).isReg());
4066 MachineBasicBlock &BB = *I.getParent();
4067 Register InputRegister = I.getOperand(2).getReg();
4068 SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister);
4069
4070 if (!InputType)
4071 return diagnoseUnsupported(I, "Input Type could not be determined.");
4072
4073 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4074 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
4075 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4076 .addDef(ResVReg)
4077 .addUse(GR.getSPIRVTypeID(ResType))
4078 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
4079 !STI.isShader()))
4080 .addImm(SPIRV::GroupOperation::ExclusiveScan)
4081 .addUse(I.getOperand(2).getReg())
4082 .constrainAllUses(TII, TRI, RBI);
4083 return true;
4084}
4085
4086bool SPIRVInstructionSelector::selectWaveMultiPrefixOp(
4087 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, unsigned Opcode,
4088 StringRef BuiltinName) const {
4089 if (!STI.canUseExtension(
4090 SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned) &&
4091 !STI.canUseExtension(
4092 SPIRV::Extension::SPV_NV_shader_subgroup_partitioned))
4093 return diagnoseUnsupported(
4094 I, Twine(BuiltinName) +
4095 " requires SPV_EXT_shader_subgroup_partitioned or "
4096 "SPV_NV_shader_subgroup_partitioned");
4097
4098 assert(I.getNumOperands() == 7);
4099 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4100 SPIRVTypeInst MaskTy = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII);
4102 GR.assignSPIRVTypeToVReg(MaskTy, Mask, MRI->getMF());
4103 auto MaskMIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4104 TII.get(SPIRV::OpCompositeConstruct))
4105 .addDef(Mask)
4106 .addUse(GR.getSPIRVTypeID(MaskTy));
4107 for (unsigned Op = 3; Op != 7; ++Op)
4108 MaskMIB.addUse(I.getOperand(Op).getReg());
4109 MaskMIB.constrainAllUses(TII, TRI, RBI);
4110
4111 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
4112 .addDef(ResVReg)
4113 .addUse(GR.getSPIRVTypeID(ResType))
4114 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
4115 !STI.isShader()))
4116 .addImm(SPIRV::GroupOperation::PartitionedExclusiveScanNV)
4117 .addUse(I.getOperand(2).getReg())
4118 .addUse(Mask)
4119 .constrainAllUses(TII, TRI, RBI);
4120 return true;
4121}
4122
4123bool SPIRVInstructionSelector::selectQuadSwap(Register ResVReg,
4124 SPIRVTypeInst ResType,
4125 MachineInstr &I,
4126 unsigned Direction) const {
4127 assert(I.getNumOperands() == 3);
4128 assert(I.getOperand(2).isReg());
4129 MachineBasicBlock &BB = *I.getParent();
4130 Register InputRegister = I.getOperand(2).getReg();
4131
4132 SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4133 bool ZeroAsNull = !STI.isShader();
4134 Register DirectionReg =
4135 GR.getOrCreateConstInt(Direction, I, IntTy, TII, ZeroAsNull);
4136 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGroupNonUniformQuadSwap))
4137 .addDef(ResVReg)
4138 .addUse(GR.getSPIRVTypeID(ResType))
4139 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII,
4140 ZeroAsNull))
4141 .addUse(InputRegister)
4142 .addUse(DirectionReg)
4143 .constrainAllUses(TII, TRI, RBI);
4144 return true;
4145}
4146
4147bool SPIRVInstructionSelector::selectQuadAll(Register ResVReg,
4148 SPIRVTypeInst ResType,
4149 MachineInstr &I) const {
4150 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_quad_control))
4151 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(2).getReg()},
4152 SPIRV::OpGroupNonUniformQuadAllKHR);
4153
4154 Register Result = I.getOperand(2).getReg();
4155 for (unsigned Direction = 0; Direction != 3; ++Direction) {
4156 Register QuadValue = createVirtualRegister(ResType, &GR, MRI, MRI->getMF());
4157 Register Next =
4158 Direction == 2 ? ResVReg
4159 : createVirtualRegister(ResType, &GR, MRI, MRI->getMF());
4160 if (!selectQuadSwap(QuadValue, ResType, I, Direction) ||
4161 !selectOpWithSrcs(Next, ResType, I, {Result, QuadValue},
4162 SPIRV::OpLogicalAnd))
4163 return false;
4164 Result = Next;
4165 }
4166 return true;
4167}
4168
4169bool SPIRVInstructionSelector::selectBitreverseViaI32(Register ResVReg,
4170 SPIRVTypeInst ResType,
4171 MachineInstr &I,
4172 Register Op) const {
4173 SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(32, I, TII);
4174 const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4175 Register ShiftConst =
4176 GR.getOrCreateConstInt(32 - BitWidth, I, Int32Type, TII);
4177 unsigned ShiftOp = SPIRV::OpShiftRightLogicalS;
4178
4179 const unsigned N = GR.getScalarOrVectorComponentCount(ResType);
4180 const unsigned ExtendOpcode = GR.isScalarOrVectorSigned(ResType)
4181 ? SPIRV::OpSConvert
4182 : SPIRV::OpUConvert;
4183
4184 if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4185 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) {
4186 Int32Type = GR.getOrCreateSPIRVVectorType(Int32Type, N, I, TII);
4187 ShiftOp = SPIRV::OpShiftRightLogicalV;
4188
4189 // Vector shifts require a composite constant
4190 const Register CompositeReg =
4191 MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4192 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4193 TII.get(SPIRV::OpConstantComposite))
4194 .addDef(CompositeReg)
4195 .addUse(GR.getSPIRVTypeID(Int32Type));
4196 for (unsigned It = 0; It < N; ++It)
4197 MIB.addUse(ShiftConst);
4198 MIB.constrainAllUses(TII, TRI, RBI);
4199
4200 ShiftConst = CompositeReg;
4201 }
4202
4203 // Converts the input to i32 (or vector of i32)
4204 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4205 if (!selectOpWithSrcs(ExtReg, Int32Type, I, {Op}, ExtendOpcode))
4206 return false;
4207
4208 // Perform bitreverse on the i32 value
4209 Register BitrevReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4210 if (!selectBitreverseNative(BitrevReg, Int32Type, I, ExtReg))
4211 return false;
4212
4213 // Shift the bit-reversed value to get the final result.
4214 Register ShiftReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type));
4215 if (!selectOpWithSrcs(ShiftReg, Int32Type, I, {BitrevReg, ShiftConst},
4216 ShiftOp))
4217 return false;
4218
4219 // Finally, convert the result back.
4220 return selectOpWithSrcs(ResVReg, ResType, I, {ShiftReg}, ExtendOpcode);
4221}
4222
4223bool SPIRVInstructionSelector::handle64BitOverflow(
4224 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
4225 unsigned int Opcode,
4226 std::function<bool(Register, SPIRVTypeInst, MachineInstr &, Register,
4227 unsigned)>
4228 CallbackFunction) const {
4229
4230 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType);
4231 assert(BaseType->getOpcode() == SPIRV::OpTypeInt &&
4232 "handle64BitOverflow should only be used for integer types");
4233 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
4234 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops");
4235
4236 MachineIRBuilder MIRBuilder(I);
4237 SPIRVTypeInst I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder);
4238 SPIRVTypeInst I64x2Type =
4239 GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder, false);
4240 SPIRVTypeInst Vec2ResType =
4241 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder, false);
4242
4243 std::vector<Register> PartialRegs;
4244
4245 unsigned CurrentComponent = 0;
4246 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
4247 Register PopCountResult =
4248 MRI->createVirtualRegister(GR.getRegClass(I64x2Type));
4249
4250 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4251 TII.get(SPIRV::OpVectorShuffle))
4252 .addDef(PopCountResult)
4253 .addUse(GR.getSPIRVTypeID(I64x2Type))
4254 .addUse(SrcReg)
4255 .addUse(SrcReg)
4256 .addImm(CurrentComponent)
4257 .addImm(CurrentComponent + 1);
4258
4259 MIB.constrainAllUses(TII, TRI, RBI);
4260
4261 Register SubVecReg =
4262 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType));
4263
4264 if (!CallbackFunction(SubVecReg, Vec2ResType, I, PopCountResult, Opcode))
4265 return false;
4266
4267 PartialRegs.push_back(SubVecReg);
4268 }
4269 // On odd component counts we need to handle one more component
4270 if (CurrentComponent != ComponentCount) {
4271 bool ZeroAsNull = !STI.isShader();
4272 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type));
4273 Register ConstIntLastIdx = GR.getOrCreateConstInt(
4274 ComponentCount - 1, I, BaseType, TII, ZeroAsNull);
4275
4276 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx},
4277 SPIRV::OpVectorExtractDynamic))
4278 return false;
4279
4280 Register FinalElemResReg =
4282
4283 if (!CallbackFunction(FinalElemResReg, BaseType, I, FinalElemReg, Opcode))
4284 return false;
4285
4286 PartialRegs.push_back(FinalElemResReg);
4287 }
4288 // Join all the resulting registers back into the return type in order
4289 // (ie i32x2, i32x2, i32x1 -> i32x5)
4290 return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs,
4291 SPIRV::OpCompositeConstruct);
4292}
4293
4294bool SPIRVInstructionSelector::selectBitreverse64(Register ResVReg,
4295 SPIRVTypeInst ResType,
4296 MachineInstr &I,
4297 Register SrcReg) const {
4298 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
4299 if (ComponentCount > 2)
4300 return handle64BitOverflow(
4301 ResVReg, ResType, I, SrcReg, SPIRV::OpBitReverse,
4302 [this](Register R, SPIRVTypeInst T, MachineInstr &I, Register S,
4303 unsigned O) { return this->selectBitreverse64(R, T, I, S); });
4304
4305 MachineIRBuilder MIRBuilder(I);
4306
4307 SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
4308 SPIRVTypeInst VecI32Type = GR.getOrCreateSPIRVVectorType(
4309 I32Type, 2 * ComponentCount, MIRBuilder, /*IsSigned=*/false);
4310
4311 // Converts 64 bit into and array of 32 bit, containing 2 elements.
4312 Register Vec32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
4313 if (!selectOpWithSrcs(Vec32, VecI32Type, I, {SrcReg}, SPIRV::OpBitcast))
4314 return false;
4315
4316 // Apply bitreverse on each 32 bit lane
4317 Register Reverse32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
4318 if (!selectBitreverseNative(Reverse32, VecI32Type, I, Vec32))
4319 return false;
4320
4321 // Reversing a 64-bit value = reverse each 32-bit half AND swap them,
4322 // so the old High word becomes lane 0 (low) and old Low becomes lane 1
4323 // (high).
4324 Register SwappedVec = MRI->createVirtualRegister(GR.getRegClass(VecI32Type));
4325 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4326 TII.get(SPIRV::OpVectorShuffle))
4327 .addDef(SwappedVec)
4328 .addUse(GR.getSPIRVTypeID(VecI32Type))
4329 .addUse(Reverse32)
4330 .addUse(Reverse32);
4331 for (unsigned J = 0; J < ComponentCount; ++J) {
4332 MIB.addImm(2 * J + 1);
4333 MIB.addImm(2 * J);
4334 }
4335 MIB.constrainAllUses(TII, TRI, RBI);
4336
4337 // Groups 32 bit vector back to 64 bit scalar.
4338 return selectOpWithSrcs(ResVReg, ResType, I, {SwappedVec}, SPIRV::OpBitcast);
4339}
4340
4341bool SPIRVInstructionSelector::selectBitreverseNative(Register ResVReg,
4342 SPIRVTypeInst ResType,
4343 MachineInstr &I,
4344 Register Op) const {
4345 MachineBasicBlock &BB = *I.getParent();
4346 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
4347 .addDef(ResVReg)
4348 .addUse(GR.getSPIRVTypeID(ResType))
4349 .addUse(Op)
4350 .constrainAllUses(TII, TRI, RBI);
4351 return true;
4352}
4353
4354bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
4355 SPIRVTypeInst ResType,
4356 MachineInstr &I) const {
4357 Register OpReg = I.getOperand(1).getReg();
4358
4359 // TODO: Fix shader behavior in case of VK_KHR_maintenance9 extension is set
4360 if (STI.isShader()) {
4361 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
4362 switch (GR.getScalarOrVectorBitWidth(OpType)) {
4363 case 8:
4364 case 16:
4365 case 24:
4366 return selectBitreverseViaI32(ResVReg, ResType, I, OpReg);
4367 case 32:
4368 return selectBitreverseNative(ResVReg, ResType, I, OpReg);
4369 case 64:
4370 return selectBitreverse64(ResVReg, ResType, I, OpReg);
4371 }
4372 return SPIRVInstructionSelector::diagnoseUnsupported(
4373 I, "G_BITREVERSE only support 16,32,64 bits.");
4374 }
4375
4376 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions))
4377 return selectBitreverseNative(ResVReg, ResType, I, OpReg);
4378
4379 // Expansion bitreverse using bit manipulation operations
4380 // Algo: https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
4381 const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4382 // TODO: add support for any bit width and bitwidth more than 64.
4383 if (BitWidth > 64 || !isPowerOf2_32(BitWidth))
4384 return false;
4385
4386 const unsigned N = GR.getScalarOrVectorComponentCount(ResType);
4387
4388 unsigned AndOp = SPIRV::OpBitwiseAndS;
4389 unsigned OrOp = SPIRV::OpBitwiseOrS;
4390 unsigned ShlOp = SPIRV::OpShiftLeftLogicalS;
4391 unsigned ShrOp = SPIRV::OpShiftRightLogicalS;
4392 if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4393 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) {
4394 AndOp = SPIRV::OpBitwiseAndV;
4395 OrOp = SPIRV::OpBitwiseOrV;
4396 ShlOp = SPIRV::OpShiftLeftLogicalV;
4397 ShrOp = SPIRV::OpShiftRightLogicalV;
4398 }
4399
4400 // Helper, one swap per step: ((input>>shift)&mask)|((input&mask)<<shift),
4401 // RPN: input shift >> mask & input mask & shift << |
4402 auto SwapBits = [&](const Register Input, const uint64_t Mask,
4403 const unsigned Shift) -> Register {
4404 auto CreateConst = [&](const uint64_t Value) -> Register {
4405 if (N == 1 &&
4406 (ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT ||
4407 !STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
4408 return GR.getOrCreateConstInt(
4409 Value, I, GR.retrieveScalarOrVectorIntType(ResType), TII);
4410 return GR.getOrCreateConstVector(Value, I, ResType, TII);
4411 };
4412
4413 Register MaskReg = CreateConst(Mask);
4414 Register ShiftReg = CreateConst(Shift);
4416 Register T2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
4417 Register T3 = MRI->createVirtualRegister(GR.getRegClass(ResType));
4418 Register T4 = MRI->createVirtualRegister(GR.getRegClass(ResType));
4420
4421 if (!selectOpWithSrcs(T1, ResType, I, {Input, ShiftReg}, ShrOp) ||
4422 !selectOpWithSrcs(T2, ResType, I, {T1, MaskReg}, AndOp) ||
4423 !selectOpWithSrcs(T3, ResType, I, {Input, MaskReg}, AndOp) ||
4424 !selectOpWithSrcs(T4, ResType, I, {T3, ShiftReg}, ShlOp) ||
4425 !selectOpWithSrcs(Result, ResType, I, {T2, T4}, OrOp))
4426 return Register();
4427
4428 return Result;
4429 };
4430
4431 unsigned Shift = BitWidth;
4432 Register Result = OpReg;
4433 uint64_t Mask = ~0ull;
4434 while ((Shift >>= 1) > 0) {
4435 Mask ^= (Mask << Shift);
4436 Result = SwapBits(Result, Mask, Shift);
4437 if (!Result.isValid())
4438 return false;
4439 }
4440
4441 return BuildCOPY(ResVReg, Result, I);
4442}
4443
4444bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
4445 SPIRVTypeInst ResType,
4446 MachineInstr &I) const {
4447 assert(I.getOperand(0).isReg() && I.getOperand(1).isReg() &&
4448 "G_FREEZE must define and use a register");
4449 Register OpReg = I.getOperand(1).getReg();
4450
4451 // With SPV_KHR_poison_freeze, lower `freeze` to OpFreezeKHR.
4452 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_poison_freeze)) {
4453 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFreezeKHR))
4454 .addDef(ResVReg)
4455 .addUse(GR.getSPIRVTypeID(ResType))
4456 .addUse(OpReg)
4457 .constrainAllUses(TII, TRI, RBI);
4458 return true;
4459 }
4460
4461 // There is no way to implement `freeze` correctly without support on SPIR-V
4462 // standard side, but we may at least address a simple (static) case when
4463 // undef/poison value presence is obvious. The main benefit of even
4464 // incomplete `freeze` support is preventing of translation from crashing due
4465 // to lack of support on legalization and instruction selection steps.
4466 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) {
4467 if (Def->getOpcode() == TargetOpcode::COPY)
4468 Def = MRI->getVRegDef(Def->getOperand(1).getReg());
4469 Register Reg;
4470 switch (Def->getOpcode()) {
4471 case SPIRV::ASSIGN_TYPE:
4472 if (MachineInstr *AssignToDef =
4473 MRI->getVRegDef(Def->getOperand(1).getReg())) {
4474 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
4475 Reg = Def->getOperand(2).getReg();
4476 }
4477 break;
4478 case SPIRV::OpUndef:
4479 Reg = Def->getOperand(1).getReg();
4480 break;
4481 }
4482 unsigned DestOpCode;
4483 if (Reg.isValid()) {
4484 DestOpCode = SPIRV::OpConstantNull;
4485 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. freeze of a "
4486 "static undef/poison lowered to OpConstantNull\n");
4487 } else {
4488 DestOpCode = TargetOpcode::COPY;
4489 Reg = OpReg;
4490 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. freeze "
4491 "skipped, lowered as a copy of the operand\n");
4492 }
4493 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode))
4494 .addDef(I.getOperand(0).getReg())
4495 .addUse(Reg)
4496 .constrainAllUses(TII, TRI, RBI);
4497 return true;
4498 }
4499 return false;
4500}
4501
4502bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg,
4503 SPIRVTypeInst ResType,
4504 MachineInstr &I) const {
4505 unsigned N = 0;
4506 if (isVectorType(ResType))
4507 N = GR.getScalarOrVectorComponentCount(ResType);
4508 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
4509 N = getArrayComponentCount(MRI, ResType);
4510 else
4511 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result");
4512 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N)
4513 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent");
4514
4515 // check if we may construct a constant vector
4516 bool IsConst = true;
4517 for (unsigned i = I.getNumExplicitDefs();
4518 i < I.getNumExplicitOperands() && IsConst; ++i)
4519 if (!isConstReg(MRI, I.getOperand(i).getReg()))
4520 IsConst = false;
4521
4522 if (!IsConst && (N < 2 && ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT))
4523 return diagnoseUnsupported(
4524 I, "There must be at least two constituent operands in a vector");
4525
4526 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
4527
4528 bool IsNullVector = IsConst && !STI.isShader();
4529 for (unsigned i = I.getNumExplicitDefs();
4530 i < I.getNumExplicitOperands() && IsNullVector; ++i) {
4531 MachineInstr *Def = getDef(I.getOperand(i), MRI);
4532 IsNullVector = Def && isNullOrNullSplat(*Def, *MRI);
4533 }
4534
4535 if (IsNullVector) {
4536 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
4537 .addDef(ResVReg)
4538 .addUse(GR.getSPIRVTypeID(ResType))
4539 .constrainAllUses(TII, TRI, RBI);
4540 return true;
4541 }
4542
4543 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4544 TII.get(IsConst ? SPIRV::OpConstantComposite
4545 : SPIRV::OpCompositeConstruct))
4546 .addDef(ResVReg)
4547 .addUse(GR.getSPIRVTypeID(ResType));
4548 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
4549 MIB.addUse(I.getOperand(i).getReg());
4550 MIB.constrainAllUses(TII, TRI, RBI);
4551 return true;
4552}
4553
4554bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
4555 SPIRVTypeInst ResType,
4556 MachineInstr &I) const {
4557 unsigned N = 0;
4558 if (isVectorType(ResType))
4559 N = GR.getScalarOrVectorComponentCount(ResType);
4560 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
4561 N = getArrayComponentCount(MRI, ResType);
4562 else
4563 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result");
4564
4565 unsigned OpIdx = I.getNumExplicitDefs();
4566 if (!I.getOperand(OpIdx).isReg())
4567 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR");
4568
4569 // check if we may construct a constant vector
4570 Register OpReg = I.getOperand(OpIdx).getReg();
4571 bool IsConst = isConstReg(MRI, OpReg);
4572
4573 if (!IsConst && (N < 2 && ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT))
4574 return diagnoseUnsupported(
4575 I, "There must be at least two constituent operands in a vector");
4576
4577 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
4578 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4579 TII.get(IsConst ? SPIRV::OpConstantComposite
4580 : SPIRV::OpCompositeConstruct))
4581 .addDef(ResVReg)
4582 .addUse(GR.getSPIRVTypeID(ResType));
4583 for (unsigned i = 0; i < N; ++i)
4584 MIB.addUse(OpReg);
4585 MIB.constrainAllUses(TII, TRI, RBI);
4586 return true;
4587}
4588
4589bool SPIRVInstructionSelector::selectConcatVectors(Register ResVReg,
4590 SPIRVTypeInst ResType,
4591 MachineInstr &I) const {
4592 // Implement G_CONCAT_VECTORS using OpCompositeConstruct, which allows vector
4593 // constituents that share the result's component type to be
4594 // concatenated in operand order.
4595 if (!isVectorType(ResType))
4597 "Cannot select G_CONCAT_VECTORS with a non-vector result");
4598
4599 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4600 TII.get(SPIRV::OpCompositeConstruct))
4601 .addDef(ResVReg)
4602 .addUse(GR.getSPIRVTypeID(ResType));
4603 for (unsigned OpIdx = I.getNumExplicitDefs();
4604 OpIdx < I.getNumExplicitOperands(); ++OpIdx)
4605 MIB.addUse(I.getOperand(OpIdx).getReg());
4606 MIB.constrainAllUses(TII, TRI, RBI);
4607 return true;
4608}
4609
4610bool SPIRVInstructionSelector::selectDiscard(Register ResVReg,
4611 SPIRVTypeInst ResType,
4612 MachineInstr &I) const {
4613
4614 unsigned Opcode;
4615 MachineBasicBlock &BB = *I.getParent();
4616
4617 if (STI.canUseExtension(
4618 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
4619 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) {
4620 Opcode = SPIRV::OpDemoteToHelperInvocation;
4621 } else {
4622 Opcode = SPIRV::OpKill;
4623 // OpKill must be the last operation of any basic block.
4624 for (MachineInstr &ToErase : make_early_inc_range(
4625 make_range(std::next(I.getIterator()), BB.instr_end()))) {
4626 GR.invalidateMachineInstr(&ToErase);
4627 ToErase.eraseFromParent();
4628 }
4629 }
4630
4631 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
4632 .constrainAllUses(TII, TRI, RBI);
4633 return true;
4634}
4635
4636bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
4637 SPIRVTypeInst ResType, unsigned CmpOpc,
4638 MachineInstr &I) const {
4639 Register Cmp0 = I.getOperand(2).getReg();
4640 Register Cmp1 = I.getOperand(3).getReg();
4641 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
4642 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
4643 "CMP operands should have the same type");
4644 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
4645 .addDef(ResVReg)
4646 .addUse(GR.getSPIRVTypeID(ResType))
4647 .addUse(Cmp0)
4648 .addUse(Cmp1)
4649 .setMIFlags(I.getFlags())
4650 .constrainAllUses(TII, TRI, RBI);
4651 return true;
4652}
4653
4654bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
4655 SPIRVTypeInst ResType,
4656 MachineInstr &I) const {
4657 auto Pred = I.getOperand(1).getPredicate();
4658 unsigned CmpOpc;
4659
4660 Register CmpOperand = I.getOperand(2).getReg();
4661 SPIRVTypeInst CmpOperandType = GR.getSPIRVTypeForVReg(CmpOperand);
4662 bool IsPtrCmp = CmpOperandType && CmpOperandType.isPointer();
4663 if (IsPtrCmp) {
4664 CmpOpc = getPtrCmpOpcode(Pred);
4665 // OpPtrEqual/OpPtrNotEqual require both operands to share an identical
4666 // pointer type. If they are not OpBitcast is inserted.
4667 Register Op1 = I.getOperand(3).getReg();
4668 SPIRVTypeInst Ty0 = GR.getSPIRVTypeForVReg(CmpOperand);
4669 if (Ty0 != GR.getSPIRVTypeForVReg(Op1)) {
4670 Register NewOp1 = createVirtualRegister(Ty0, &GR, MRI, MRI->getMF());
4671 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast))
4672 .addDef(NewOp1)
4673 .addUse(GR.getSPIRVTypeID(Ty0))
4674 .addUse(Op1)
4675 .constrainAllUses(TII, TRI, RBI);
4676 I.getOperand(3).setReg(NewOp1);
4677 }
4678 } else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
4679 CmpOpc = getBoolCmpOpcode(Pred);
4680 else
4681 CmpOpc = getICmpOpcode(Pred);
4682 return selectCmp(ResVReg, ResType, CmpOpc, I);
4683}
4684
4685// Always inserts the constant definition in the entry block so it dominates all
4686// uses regardless of block ordering.
4687Register SPIRVInstructionSelector::buildI32ConstantInEntryBlock(
4688 uint32_t Val, MachineInstr &I, SPIRVTypeInst ResType) const {
4689 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
4690 SPIRVTypeInst SpvI32Ty =
4691 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
4692 auto *ConstInt = ConstantInt::get(LLVMTy, Val);
4693 Register NewReg = GR.find(ConstInt, GR.CurMF);
4694 if (!NewReg.isValid()) {
4696 auto InsertIt = getOpVariableMBBIt(*I.getMF());
4697 MachineBasicBlock &EntryBB = *InsertIt->getParent();
4698 MachineInstr *MI = nullptr;
4699 Register TypeReg = GR.getSPIRVTypeID(SpvI32Ty);
4700 DebugLoc DbgLoc = I.getDebugLoc();
4701 if (Val == 0) {
4702 MI = BuildMI(EntryBB, InsertIt, DbgLoc, TII.get(SPIRV::OpConstantNull))
4703 .addDef(NewReg)
4704 .addUse(TypeReg);
4705 } else {
4706 uint64_t ImmVal = APInt(32, Val).getZExtValue();
4707 MI = BuildMI(EntryBB, InsertIt, DbgLoc, TII.get(SPIRV::OpConstantI))
4708 .addDef(NewReg)
4709 .addUse(TypeReg)
4710 .addImm(ImmVal);
4711 }
4713 GR.add(ConstInt, MI);
4714 }
4715 return NewReg;
4716}
4717
4718bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
4719 SPIRVTypeInst ResType,
4720 MachineInstr &I) const {
4721 auto Pred = I.getOperand(1).getPredicate();
4722 // OpOrdered and OpUnordered require the Kernel capability. For shaders,
4723 // express them as uno(a, b) = isnan(a) || isnan(b), ord(a, b) = !uno(a, b).
4724 if (STI.isShader() &&
4726 MachineBasicBlock &BB = *I.getParent();
4727 Register ResTypeID = GR.getSPIRVTypeID(ResType);
4728 Register IsNan[2];
4729 for (unsigned Idx = 0; Idx < 2; ++Idx) {
4730 IsNan[Idx] = createVirtualRegister(ResType, &GR, MRI, *I.getMF());
4731 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNan))
4732 .addDef(IsNan[Idx])
4733 .addUse(ResTypeID)
4734 .addUse(I.getOperand(2 + Idx).getReg())
4735 .constrainAllUses(TII, TRI, RBI);
4736 }
4737 Register UnoReg =
4739 ? ResVReg
4740 : createVirtualRegister(ResType, &GR, MRI, *I.getMF());
4741 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLogicalOr))
4742 .addDef(UnoReg)
4743 .addUse(ResTypeID)
4744 .addUse(IsNan[0])
4745 .addUse(IsNan[1])
4746 .constrainAllUses(TII, TRI, RBI);
4747 if (Pred == CmpInst::FCMP_ORD)
4748 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLogicalNot))
4749 .addDef(ResVReg)
4750 .addUse(ResTypeID)
4751 .addUse(UnoReg)
4752 .constrainAllUses(TII, TRI, RBI);
4753 return true;
4754 }
4755 unsigned CmpOp = getFCmpOpcode(Pred);
4756 return selectCmp(ResVReg, ResType, CmpOp, I);
4757}
4758
4759bool SPIRVInstructionSelector::selectExp10(Register ResVReg,
4760 SPIRVTypeInst ResType,
4761 MachineInstr &I) const {
4762 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
4763 return selectExtInst(ResVReg, ResType, I, CL::exp10);
4764 }
4765
4766 if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
4767 /// There is no exp10 in GLSL. Use exp10(x) = exp2(x * log2(10)) instead
4768 /// log2(10) ~= 3.3219280948874l
4769
4770 if (!isVectorType(ResType) && !ResType.isAnyTypeFloat())
4771 return false;
4772
4773 MachineIRBuilder MIRBuilder(I);
4774
4775 SPIRVTypeInst SpirvScalarType = GR.getScalarOrVectorComponentType(ResType);
4776
4777 // Match the literal precision to the scalar type so the OpConstant
4778 // literal does not contain non-zero high-order bits that would fail
4779 // SPIR-V validation when the type is narrower than 32 bits (e.g. half).
4780 APFloat ConstVal(3.3219280948873623);
4781 bool LosesInfo;
4782 ConstVal.convert(
4783 getZeroFP(GR.getTypeForSPIRVType(SpirvScalarType)).getSemantics(),
4784 APFloat::rmNearestTiesToEven, &LosesInfo);
4785 Register ConstReg =
4786 GR.buildConstantFP(ConstVal, MIRBuilder, SpirvScalarType);
4787 Register ArgReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
4788 auto Opcode =
4789 isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS;
4790
4791 if (!selectOpWithSrcs(ArgReg, ResType, I,
4792 {I.getOperand(1).getReg(), ConstReg}, Opcode))
4793 return false;
4794 if (!selectExtInst(ResVReg, ResType, I,
4795 {{SPIRV::InstructionSet::GLSL_std_450, GL::Exp2}}, false,
4796 false, {ArgReg}))
4797 return false;
4798
4799 return true;
4800 }
4801
4802 return false;
4803}
4804
4805Register SPIRVInstructionSelector::buildZerosVal(SPIRVTypeInst ResType,
4806 MachineInstr &I) const {
4807 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
4808 bool ZeroAsNull = !STI.isShader();
4809 if (isVectorType(ResType))
4810 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
4811 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
4812}
4813
4814bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero(
4815 Register Reg) const {
4816 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(Reg);
4817 if (!Type)
4818 return false;
4819 SPIRVTypeInst CompType = GR.getScalarOrVectorComponentType(Type);
4820 if (!CompType || CompType->getOpcode() != SPIRV::OpTypeInt)
4821 return false;
4822
4823 auto IsZero = [this](Register Reg) {
4824 MachineInstr *Def = getDefInstrMaybeConstant(Reg, MRI);
4825 if (!Def)
4826 return false;
4827
4828 if (Def->getOpcode() == SPIRV::OpConstantNull)
4829 return true;
4830
4831 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||
4832 Def->getOpcode() == SPIRV::OpConstantI)
4833 return getIConstVal(Reg, MRI) == 0;
4834
4835 return false;
4836 };
4837
4838 if (IsZero(Reg))
4839 return true;
4840
4841 MachineInstr *Def = MRI->getVRegDef(Reg);
4842 if (!Def)
4843 return false;
4844
4845 if (Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
4846 (Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
4847 cast<GIntrinsic>(Def)->getIntrinsicID() ==
4848 Intrinsic::spv_const_composite)) {
4849 unsigned StartOp = Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2;
4850 for (unsigned i = StartOp; i < Def->getNumOperands(); ++i) {
4851 if (!IsZero(Def->getOperand(i).getReg()))
4852 return false;
4853 }
4854 return true;
4855 }
4856
4857 return false;
4858}
4859
4860Register SPIRVInstructionSelector::buildZerosValF(SPIRVTypeInst ResType,
4861 MachineInstr &I) const {
4862 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
4863 bool ZeroAsNull = !STI.isShader();
4864 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType));
4865 if (isVectorType(ResType))
4866 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
4867 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull);
4868}
4869
4870Register SPIRVInstructionSelector::buildOnesValF(SPIRVTypeInst ResType,
4871 MachineInstr &I) const {
4872 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
4873 bool ZeroAsNull = !STI.isShader();
4874 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType));
4875 if (isVectorType(ResType))
4876 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull);
4877 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull);
4878}
4879
4880Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
4881 SPIRVTypeInst ResType,
4882 MachineInstr &I) const {
4883 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4884 APInt One =
4885 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
4886 if (isVectorType(ResType))
4887 return GR.getOrCreateConstVector(One, I, ResType, TII);
4888 return GR.getOrCreateConstInt(One, I, ResType, TII);
4889}
4890
4891Register SPIRVInstructionSelector::buildVectorSplat(Register ScalarReg,
4892 unsigned NumElts,
4893 MachineInstr &I) const {
4894 SPIRVTypeInst VecType = GR.getOrCreateSPIRVVectorType(
4895 GR.getSPIRVTypeForVReg(ScalarReg), NumElts, I, TII);
4896 Register SplatReg = MRI->createVirtualRegister(GR.getRegClass(VecType));
4897 GR.assignSPIRVTypeToVReg(VecType, SplatReg, MRI->getMF());
4898 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
4899 TII.get(SPIRV::OpCompositeConstruct))
4900 .addDef(SplatReg)
4901 .addUse(GR.getSPIRVTypeID(VecType));
4902 for (unsigned J = 0; J < NumElts; ++J)
4903 MIB.addUse(ScalarReg);
4904 MIB.constrainAllUses(TII, TRI, RBI);
4905 return SplatReg;
4906}
4907
4908bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
4909 SPIRVTypeInst ResType,
4910 MachineInstr &I) const {
4911 Register SelectFirstArg = I.getOperand(2).getReg();
4912 Register SelectSecondArg = I.getOperand(3).getReg();
4913 assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) &&
4914 ResType == GR.getSPIRVTypeForVReg(SelectSecondArg));
4915
4916 bool IsFloatTy =
4917 GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypeFloat);
4918 bool IsPtrTy = GR.getScalarOrVectorComponentType(ResType).isPointer();
4919
4920 Register CondReg = I.getOperand(1).getReg();
4921 bool IsScalarBool = GR.isScalarOfType(CondReg, SPIRV::OpTypeBool);
4922 unsigned Opcode;
4923 if (isVectorType(GR.getSPIRVTypeForVReg(SelectFirstArg))) {
4924 // Before SPIR-V 1.4, the condition of an OpSelect with a vector result
4925 // must be a vector of Booleans with the same number of components, while
4926 // LLVM IR also allows a scalar i1 condition. Splat it in that case.
4927 if (IsScalarBool && !STI.isAtLeastSPIRVVer(VersionTuple(1, 4))) {
4928 unsigned NumElts = GR.getScalarOrVectorComponentCount(ResType);
4929 CondReg = buildVectorSplat(CondReg, NumElts, I);
4930 IsScalarBool = false;
4931 }
4932 if (IsFloatTy) {
4933 Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
4934 } else if (IsPtrTy) {
4935 Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
4936 } else {
4937 Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
4938 }
4939 } else {
4940 assert(IsScalarBool && "OpSelect with a scalar result requires a scalar "
4941 "boolean condition");
4942 if (IsFloatTy) {
4943 Opcode = SPIRV::OpSelectSFSCond;
4944 } else if (IsPtrTy) {
4945 Opcode = SPIRV::OpSelectSPSCond;
4946 } else {
4947 Opcode = SPIRV::OpSelectSISCond;
4948 }
4949 }
4950 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
4951 .addDef(ResVReg)
4952 .addUse(GR.getSPIRVTypeID(ResType))
4953 .addUse(CondReg)
4954 .addUse(SelectFirstArg)
4955 .addUse(SelectSecondArg)
4956 .constrainAllUses(TII, TRI, RBI);
4957 return true;
4958}
4959
4960// This function is used to extend a bool or a vector of bools into an integer
4961// or vector of integers.
4962bool SPIRVInstructionSelector::selectBoolToInt(Register ResVReg,
4963 SPIRVTypeInst ResType,
4964 Register BooleanVReg,
4965 MachineInstr &InsertAt,
4966 bool IsSigned) const {
4967 // To extend a bool, we need to use OpSelect between constants.
4968 Register ZeroReg = buildZerosVal(ResType, InsertAt);
4969 Register OneReg = buildOnesVal(IsSigned, ResType, InsertAt);
4970 bool IsScalarBool = GR.isScalarOfType(BooleanVReg, SPIRV::OpTypeBool);
4971 unsigned Opcode =
4972 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
4973 BuildMI(*InsertAt.getParent(), InsertAt, InsertAt.getDebugLoc(),
4974 TII.get(Opcode))
4975 .addDef(ResVReg)
4976 .addUse(GR.getSPIRVTypeID(ResType))
4977 .addUse(BooleanVReg)
4978 .addUse(OneReg)
4979 .addUse(ZeroReg)
4980 .constrainAllUses(TII, TRI, RBI);
4981 return true;
4982}
4983
4984bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
4985 SPIRVTypeInst ResType,
4986 MachineInstr &I, bool IsSigned,
4987 unsigned Opcode) const {
4988 Register SrcReg = I.getOperand(1).getReg();
4989 // We can convert bool value directly to float type without OpConvert*ToF,
4990 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
4991 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
4992 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
4993 SPIRVTypeInst TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
4994 if (isVectorType(ResType)) {
4995 const unsigned NumElts = GR.getScalarOrVectorComponentCount(ResType);
4996 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
4997 }
4998 SrcReg = createVirtualRegister(TmpType, &GR, MRI, MRI->getMF());
4999 selectBoolToInt(SrcReg, TmpType, I.getOperand(1).getReg(), I, IsSigned);
5000 }
5001 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
5002}
5003
5004bool SPIRVInstructionSelector::selectExt(Register ResVReg,
5005 SPIRVTypeInst ResType, MachineInstr &I,
5006 bool IsSigned) const {
5007 Register SrcReg = I.getOperand(1).getReg();
5008 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool))
5009 return selectBoolToInt(ResVReg, ResType, I.getOperand(1).getReg(), I,
5010 IsSigned);
5011
5012 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg);
5013 if (ResType == SrcType)
5014 return BuildCOPY(ResVReg, SrcReg, I);
5015
5016 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
5017 return selectUnOp(ResVReg, ResType, I, Opcode);
5018}
5019
5020bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg,
5021 SPIRVTypeInst ResType,
5022 MachineInstr &I,
5023 bool IsSigned) const {
5024 MachineIRBuilder MIRBuilder(I);
5025 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
5026 MachineBasicBlock &BB = *I.getParent();
5027 // Ensure we have bool.
5028 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
5029 unsigned N = GR.getScalarOrVectorComponentCount(ResType);
5030 if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
5031 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
5032 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
5033 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
5034 // Build less-than-equal and less-than.
5035 Register IsLessEqReg =
5036 createVirtualRegister(BoolType, &GR, MRI, MIRBuilder.getMF());
5037 BuildMI(BB, I, I.getDebugLoc(),
5038 TII.get(IsSigned ? SPIRV::OpSLessThanEqual : SPIRV::OpULessThanEqual))
5039 .addDef(IsLessEqReg)
5040 .addUse(BoolTypeReg)
5041 .addUse(I.getOperand(1).getReg())
5042 .addUse(I.getOperand(2).getReg())
5043 .constrainAllUses(TII, TRI, RBI);
5044 Register IsLessReg =
5045 createVirtualRegister(BoolType, &GR, MRI, MIRBuilder.getMF());
5046 BuildMI(BB, I, I.getDebugLoc(),
5047 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
5048 .addDef(IsLessReg)
5049 .addUse(BoolTypeReg)
5050 .addUse(I.getOperand(1).getReg())
5051 .addUse(I.getOperand(2).getReg())
5052 .constrainAllUses(TII, TRI, RBI);
5053 // Build selects.
5054 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
5055 Register NegOneOrZeroReg =
5057 MRI->setType(NegOneOrZeroReg, LLT::scalar(64));
5058 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF());
5059 unsigned SelectOpcode =
5060 (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT &&
5061 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)))
5062 ? SPIRV::OpSelectVIVCond
5063 : SPIRV::OpSelectSISCond;
5064 BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
5065 .addDef(NegOneOrZeroReg)
5066 .addUse(ResTypeReg)
5067 .addUse(IsLessReg)
5068 .addUse(buildOnesVal(true, ResType, I)) // -1
5069 .addUse(buildZerosVal(ResType, I))
5070 .constrainAllUses(TII, TRI, RBI);
5071 BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode))
5072 .addDef(ResVReg)
5073 .addUse(ResTypeReg)
5074 .addUse(IsLessEqReg)
5075 .addUse(NegOneOrZeroReg) // -1 or 0
5076 .addUse(buildOnesVal(false, ResType, I))
5077 .constrainAllUses(TII, TRI, RBI);
5078 return true;
5079}
5080
5081bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
5082 Register ResVReg,
5083 MachineInstr &I,
5084 SPIRVTypeInst IntTy,
5085 SPIRVTypeInst BoolTy) const {
5086 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
5087 Register BitIntReg = createVirtualRegister(IntTy, &GR, MRI, MRI->getMF());
5088 unsigned Opcode =
5089 isVectorType(IntTy) ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
5090 Register Zero = buildZerosVal(IntTy, I);
5091 Register One = buildOnesVal(false, IntTy, I);
5092 MachineBasicBlock &BB = *I.getParent();
5093 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
5094 .addDef(BitIntReg)
5095 .addUse(GR.getSPIRVTypeID(IntTy))
5096 .addUse(IntReg)
5097 .addUse(One)
5098 .constrainAllUses(TII, TRI, RBI);
5099 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
5100 .addDef(ResVReg)
5101 .addUse(GR.getSPIRVTypeID(BoolTy))
5102 .addUse(BitIntReg)
5103 .addUse(Zero)
5104 .constrainAllUses(TII, TRI, RBI);
5105 return true;
5106}
5107
5108bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
5109 SPIRVTypeInst ResType,
5110 MachineInstr &I) const {
5111 Register IntReg = I.getOperand(1).getReg();
5112 const SPIRVTypeInst ArgType = GR.getSPIRVTypeForVReg(IntReg);
5113 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool))
5114 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
5115 if (ArgType == ResType)
5116 return BuildCOPY(ResVReg, IntReg, I);
5117 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
5118 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
5119 return selectUnOp(ResVReg, ResType, I, Opcode);
5120}
5121
5122bool SPIRVInstructionSelector::selectConst(Register ResVReg,
5123 SPIRVTypeInst ResType,
5124 MachineInstr &I) const {
5125 unsigned Opcode = I.getOpcode();
5126 unsigned TpOpcode = ResType->getOpcode();
5127 Register Reg;
5128 if (ResType.isPointer() || TpOpcode == SPIRV::OpTypeEvent) {
5129 assert(Opcode == TargetOpcode::G_CONSTANT &&
5130 I.getOperand(1).getCImm()->isZero());
5131 MachineBasicBlock &DepMBB = I.getMF()->front();
5132 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
5133 Reg = GR.getOrCreateConstNullPtr(MIRBuilder, ResType);
5134 } else if (TpOpcode == SPIRV::OpTypeVectorIdEXT) {
5135 // We ended up here coming from a splat on a <1 x T> type, which
5136 // IRTranslator translated into a scalar, so we have to restore the
5137 // vectorness.
5138 assert(GR.getScalarOrVectorComponentCount(ResType) == 1 &&
5139 "Expected <1 x T> Vector!");
5140 if (Opcode == TargetOpcode::G_FCONSTANT)
5141 Reg = GR.getOrCreateConstVector(I.getOperand(1).getFPImm()->getValue(), I,
5142 ResType, TII);
5143 else // We handle vector of pointer here as well.
5144 Reg = GR.getOrCreateConstVector(I.getOperand(1).getCImm()->getValue(), I,
5145 ResType, TII);
5146 } else if (Opcode == TargetOpcode::G_FCONSTANT) {
5147 Reg = GR.getOrCreateConstFP(I.getOperand(1).getFPImm()->getValue(), I,
5148 ResType, TII, !STI.isShader());
5149
5150 } else {
5151 Reg = GR.getOrCreateConstInt(I.getOperand(1).getCImm()->getValue(), I,
5152 ResType, TII, !STI.isShader());
5153 }
5154 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I);
5155}
5156
5157bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
5158 SPIRVTypeInst ResType,
5159 MachineInstr &I) const {
5160 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
5161 .addDef(ResVReg)
5162 .addUse(GR.getSPIRVTypeID(ResType))
5163 .constrainAllUses(TII, TRI, RBI);
5164 return true;
5165}
5166
5167bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
5168 SPIRVTypeInst ResType,
5169 MachineInstr &I) const {
5170 MachineBasicBlock &BB = *I.getParent();
5171 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
5172 .addDef(ResVReg)
5173 .addUse(GR.getSPIRVTypeID(ResType))
5174 // object to insert
5175 .addUse(I.getOperand(3).getReg())
5176 // composite to insert into
5177 .addUse(I.getOperand(2).getReg());
5178 for (unsigned i = 4; i < I.getNumOperands(); i++)
5179 MIB.addImm(foldImm(I.getOperand(i), MRI));
5180 MIB.constrainAllUses(TII, TRI, RBI);
5181 return true;
5182}
5183
5184bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
5185 SPIRVTypeInst ResType,
5186 MachineInstr &I) const {
5187 Type *MaybeResTy = nullptr;
5188 StringRef ResName;
5189 if (GR.findValueAttrs(&I, MaybeResTy, ResName) &&
5190 MaybeResTy != GR.getTypeForSPIRVType(ResType)) {
5191 assert((!MaybeResTy || MaybeResTy->isAggregateType()) &&
5192 "Expected aggregate type for extractv instruction");
5193 ResType = GR.getOrCreateSPIRVType(MaybeResTy, I,
5194 SPIRV::AccessQualifier::ReadWrite, false);
5195 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *I.getMF());
5196 }
5197 MachineBasicBlock &BB = *I.getParent();
5198 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
5199 .addDef(ResVReg)
5200 .addUse(GR.getSPIRVTypeID(ResType))
5201 .addUse(I.getOperand(2).getReg());
5202 for (unsigned i = 3; i < I.getNumOperands(); i++)
5203 MIB.addImm(foldImm(I.getOperand(i), MRI));
5204 MIB.constrainAllUses(TII, TRI, RBI);
5205 return true;
5206}
5207
5208bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
5209 SPIRVTypeInst ResType,
5210 MachineInstr &I) const {
5211 if (getImm(I.getOperand(4), MRI))
5212 return selectInsertVal(ResVReg, ResType, I);
5213 MachineBasicBlock &BB = *I.getParent();
5214 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
5215 .addDef(ResVReg)
5216 .addUse(GR.getSPIRVTypeID(ResType))
5217 .addUse(I.getOperand(2).getReg())
5218 .addUse(I.getOperand(3).getReg())
5219 .addUse(I.getOperand(4).getReg())
5220 .constrainAllUses(TII, TRI, RBI);
5221 return true;
5222}
5223
5224bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
5225 SPIRVTypeInst ResType,
5226 MachineInstr &I) const {
5227 if (getImm(I.getOperand(3), MRI))
5228 return selectExtractVal(ResVReg, ResType, I);
5229 MachineBasicBlock &BB = *I.getParent();
5230 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
5231 .addDef(ResVReg)
5232 .addUse(GR.getSPIRVTypeID(ResType))
5233 .addUse(I.getOperand(2).getReg())
5234 .addUse(I.getOperand(3).getReg())
5235 .constrainAllUses(TII, TRI, RBI);
5236 return true;
5237}
5238
5239bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
5240 SPIRVTypeInst ResType,
5241 MachineInstr &I) const {
5242 const bool IsGEPInBounds = I.getOperand(2).getImm();
5243 // Pointers to opaque types stay typed even with the extension on, so emit the
5244 // untyped variant only when the result is actually an untyped pointer.
5245 const bool UseUntypedPointers =
5246 ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
5247
5248 // Determine the opcode based on pointer type and bounds checking.
5249 // When using untyped pointers, use OpUntyped*AccessChainKHR variants.
5250 unsigned Opcode;
5251 if (UseUntypedPointers) {
5252 if (STI.isLogicalSPIRV()) {
5253 Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsAccessChainKHR
5254 : SPIRV::OpUntypedAccessChainKHR;
5255 } else {
5256 Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsPtrAccessChainKHR
5257 : SPIRV::OpUntypedPtrAccessChainKHR;
5258 }
5259 } else {
5260 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator
5261 // only relies on PtrAccessChain, so we'll try not to deviate. For Vulkan
5262 // however, we have to use Op[InBounds]AccessChain.
5263 // FIXME: fix llvm-spirv.
5264 if (STI.isLogicalSPIRV()) {
5265 Opcode =
5266 IsGEPInBounds ? SPIRV::OpInBoundsAccessChain : SPIRV::OpAccessChain;
5267 } else {
5268 Opcode = IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
5269 : SPIRV::OpPtrAccessChain;
5270 }
5271 }
5272
5273 Register BaseReg = I.getOperand(3).getReg();
5274 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
5275 .addDef(ResVReg)
5276 .addUse(GR.getSPIRVTypeID(ResType));
5277
5278 // For untyped access chains, we need to add the base type operand.
5279 if (UseUntypedPointers) {
5280 // Get the element type from the base pointer register.
5281 // For untyped pointers, this was stored when processing
5282 // spv_assign_ptr_type.
5283 SPIRVTypeInst BaseType = GR.getUntypedPtrElementType(BaseReg);
5284 if (!BaseType) {
5285 // Otherwise try the pointee type for mixed typed-pointer usage.
5286 SPIRVTypeInst BasePtrType = GR.getSPIRVTypeForVReg(BaseReg);
5287 BaseType = BasePtrType ? GR.getPointeeType(BasePtrType) : nullptr;
5288 }
5289 if (!BaseType) {
5290 // The base may be a not-yet-selected global. Read its value type from
5291 // the defining G_GLOBAL_VALUE, following copies.
5292 Register DefReg = BaseReg;
5293 MachineInstr *Def = MRI->getVRegDef(DefReg);
5294 while (Def && Def->getOpcode() == TargetOpcode::COPY &&
5295 Def->getOperand(1).isReg())
5296 Def = MRI->getVRegDef(Def->getOperand(1).getReg());
5297 if (Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE)
5298 if (const auto *GVar =
5299 dyn_cast<GlobalVariable>(Def->getOperand(1).getGlobal()))
5300 BaseType = GR.getOrCreateSPIRVType(GVar->getValueType(), I,
5301 SPIRV::AccessQualifier::ReadWrite,
5302 /*EmitIR=*/false);
5303 }
5304 if (!BaseType)
5305 return diagnoseUnsupported(
5306 I, "could not deduce the base type of an untyped access chain");
5307 Res.addUse(GR.getSPIRVTypeID(BaseType));
5308 }
5309
5310 // Object to get a pointer to.
5311 Res.addUse(BaseReg);
5312
5313 const bool IsAccessChainOpcode =
5314 (Opcode == SPIRV::OpAccessChain ||
5315 Opcode == SPIRV::OpInBoundsAccessChain ||
5316 Opcode == SPIRV::OpUntypedAccessChainKHR ||
5317 Opcode == SPIRV::OpUntypedInBoundsAccessChainKHR);
5318
5319 assert((!IsAccessChainOpcode || (getImm(I.getOperand(4), MRI) &&
5320 foldImm(I.getOperand(4), MRI) == 0)) &&
5321 "Cannot translate GEP to OpAccessChain.");
5322
5323 // Adding indices.
5324 const unsigned StartingIndex = IsAccessChainOpcode ? 5 : 4;
5325 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
5326 Res.addUse(I.getOperand(i).getReg());
5327 Res.constrainAllUses(TII, TRI, RBI);
5328
5329 // IRTranslator doesn't like <1 x T> vectors, and treats them as scalars. This
5330 // creates an information loss issue, and also broken code where a
5331 // OpCompositeExtract is applied to the scalar return of an OpPtrAccessChain,
5332 // when it was meant to be applied to the OpCompositeInsert created <1 x T>.
5333 // TODO: should we also re-create the index <1 x T>?
5334 if (MRI->hasOneUse(ResVReg)) {
5335 MachineInstr &Extract = *MRI->use_instr_begin(ResVReg);
5336 if (Extract.getOpcode() == SPIRV::OpCompositeExtract) {
5337 SPIRVTypeInst V = GR.getOrCreateSPIRVVectorType(ResType, 1, Extract, TII);
5338 // We cannot use GlobalRegistry::getOrCreateUndef directly here because it
5339 // tries to use UndefVal::get, which does not work for TypedPointerType,
5340 // which we can get here if we're dealing with <1 x T*>.
5341 Register Tmp = createVirtualRegister(V, &GR, MRI, *I.getMF());
5342 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
5343 .addDef(Tmp)
5344 .addUse(GR.getSPIRVTypeID(V))
5345 .constrainAllUses(TII, TRI, RBI);
5346 Register InsertReg = createVirtualRegister(V, &GR, MRI, *I.getMF());
5347 BuildMI(*I.getParent(), Extract, Extract.getDebugLoc(),
5348 TII.get(SPIRV::OpCompositeInsert))
5349 .addDef(InsertReg)
5350 .addUse(GR.getSPIRVTypeID(V))
5351 .addUse(ResVReg)
5352 .addUse(Tmp)
5353 .addImm(0)
5354 .constrainAllUses(TII, TRI, RBI);
5355 Extract.substituteRegister(ResVReg, InsertReg, 0, TRI);
5356 }
5357 }
5358 return true;
5359}
5360
5361// Maybe wrap a value into OpSpecConstantOp
5362bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
5363 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
5364 unsigned Lim = I.getNumExplicitOperands();
5365 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
5366 Register OpReg = I.getOperand(i).getReg();
5367 MachineInstr *OpDefine = MRI->getVRegDef(OpReg);
5368 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
5369 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) ||
5370 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
5371 OpDefine->getOpcode() == TargetOpcode::G_INTTOPTR ||
5372 GR.isAggregateType(OpType)) {
5373 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
5374 // by selectAddrSpaceCast(), and G_INTTOPTR is processed by selectUnOp()
5375 CompositeArgs.push_back(OpReg);
5376 continue;
5377 }
5378 MachineFunction *MF = I.getMF();
5379 Register WrapReg = GR.find(OpDefine, MF);
5380 if (WrapReg.isValid()) {
5381 CompositeArgs.push_back(WrapReg);
5382 continue;
5383 }
5384 SPIRVTypeInst WrapType = OpType;
5385 if (OpType->getOpcode() == SPIRV::OpTypePointer &&
5386 GR.getPointerStorageClass(OpType) ==
5387 SPIRV::StorageClass::CodeSectionINTEL) {
5388 WrapType = GR.changePointerStorageClass(OpType,
5389 SPIRV::StorageClass::Function, I);
5390 }
5391 WrapReg = MRI->createVirtualRegister(GR.getRegClass(WrapType));
5392 CompositeArgs.push_back(WrapReg);
5393 // Decorate the wrapper register and generate a new instruction
5394 MRI->setType(WrapReg, LLT::pointer(0, 64));
5395 GR.assignSPIRVTypeToVReg(WrapType, WrapReg, *MF);
5396 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
5397 TII.get(SPIRV::OpSpecConstantOp))
5398 .addDef(WrapReg)
5399 .addUse(GR.getSPIRVTypeID(WrapType))
5400 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
5401 .addUse(OpReg);
5402 GR.add(OpDefine, MIB);
5403 MIB.constrainAllUses(TII, TRI, RBI);
5404 }
5405 return true;
5406}
5407
5408bool SPIRVInstructionSelector::selectDerivativeInst(
5409 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
5410 const unsigned DPdOpCode) const {
5411 // TODO: This should check specifically for Fragment Execution Model, but STI
5412 // doesn't provide that information yet. See #167562
5413 if (!errorIfInstrOutsideShader(I))
5414 return false;
5415
5416 // If the arg/result types are half then we need to wrap the instr in
5417 // conversions to float
5418 // This case occurs because a half arg/result is legal in HLSL but not spirv.
5419 Register SrcReg = I.getOperand(2).getReg();
5420 SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg);
5421 unsigned BitWidth = std::min(GR.getScalarOrVectorBitWidth(SrcType),
5422 GR.getScalarOrVectorBitWidth(ResType));
5423 if (BitWidth == 32)
5424 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode))
5425 .addDef(ResVReg)
5426 .addUse(GR.getSPIRVTypeID(ResType))
5427 .addUse(I.getOperand(2).getReg());
5428
5429 MachineIRBuilder MIRBuilder(I);
5430 unsigned componentCount = GR.getScalarOrVectorComponentCount(SrcType);
5431 SPIRVTypeInst F32ConvertTy = GR.getOrCreateSPIRVFloatType(32, I, TII);
5432 if (componentCount != 1)
5433 F32ConvertTy = GR.getOrCreateSPIRVVectorType(F32ConvertTy, componentCount,
5434 MIRBuilder, false);
5435
5436 const TargetRegisterClass *RegClass = GR.getRegClass(SrcType);
5437 Register ConvertToVReg = MRI->createVirtualRegister(RegClass);
5438 Register DpdOpVReg = MRI->createVirtualRegister(RegClass);
5439
5440 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert))
5441 .addDef(ConvertToVReg)
5442 .addUse(GR.getSPIRVTypeID(F32ConvertTy))
5443 .addUse(SrcReg)
5444 .constrainAllUses(TII, TRI, RBI);
5445 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode))
5446 .addDef(DpdOpVReg)
5447 .addUse(GR.getSPIRVTypeID(F32ConvertTy))
5448 .addUse(ConvertToVReg)
5449 .constrainAllUses(TII, TRI, RBI);
5450 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert))
5451 .addDef(ResVReg)
5452 .addUse(GR.getSPIRVTypeID(ResType))
5453 .addUse(DpdOpVReg)
5454 .constrainAllUses(TII, TRI, RBI);
5455 return true;
5456}
5457
5458bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
5459 SPIRVTypeInst ResType,
5460 MachineInstr &I) const {
5461 MachineBasicBlock &BB = *I.getParent();
5462 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID();
5463 switch (IID) {
5464 case Intrinsic::spv_load:
5465 return selectLoad(ResVReg, ResType, I);
5466 case Intrinsic::spv_atomic_load:
5467 return selectAtomicLoad(ResVReg, ResType, I);
5468 case Intrinsic::spv_store:
5469 return selectStore(I);
5470 case Intrinsic::spv_atomic_store:
5471 return selectAtomicStore(I);
5472 case Intrinsic::spv_extractv:
5473 return selectExtractVal(ResVReg, ResType, I);
5474 case Intrinsic::spv_insertv:
5475 return selectInsertVal(ResVReg, ResType, I);
5476 case Intrinsic::spv_extractelt:
5477 return selectExtractElt(ResVReg, ResType, I);
5478 case Intrinsic::spv_insertelt:
5479 return selectInsertElt(ResVReg, ResType, I);
5480 case Intrinsic::spv_gep:
5481 return selectGEP(ResVReg, ResType, I);
5482 case Intrinsic::spv_bitcast: {
5483 Register OpReg = I.getOperand(2).getReg();
5484 SPIRVTypeInst OpType =
5485 OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
5486 if (!GR.isBitcastCompatible(ResType, OpType))
5487 report_fatal_error("incompatible result and operand types in a bitcast");
5488 return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
5489 }
5490 case Intrinsic::spv_unref_global:
5491 case Intrinsic::spv_init_global: {
5492 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
5493 MachineInstr *Init = I.getNumExplicitOperands() > 2
5494 ? MRI->getVRegDef(I.getOperand(2).getReg())
5495 : nullptr;
5496 assert(MI);
5497 Register GVarVReg = MI->getOperand(0).getReg();
5498 if (!selectGlobalValue(GVarVReg, *MI, Init))
5499 return false;
5500 // We violate SSA form by inserting OpVariable and still having a gMIR
5501 // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing
5502 // the duplicated definition.
5503 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
5505 MI->eraseFromParent();
5506 }
5507 return true;
5508 }
5509 case Intrinsic::spv_undef: {
5510 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
5511 .addDef(ResVReg)
5512 .addUse(GR.getSPIRVTypeID(ResType));
5513 MIB.constrainAllUses(TII, TRI, RBI);
5514 return true;
5515 }
5516 case Intrinsic::spv_poison:
5517 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpPoisonKHR))
5518 .addDef(ResVReg)
5519 .addUse(GR.getSPIRVTypeID(ResType))
5520 .constrainAllUses(TII, TRI, RBI);
5521 return true;
5522 case Intrinsic::spv_freeze:
5523 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpFreezeKHR))
5524 .addDef(ResVReg)
5525 .addUse(GR.getSPIRVTypeID(ResType))
5526 .addUse(I.getOperand(2).getReg())
5527 .constrainAllUses(TII, TRI, RBI);
5528 return true;
5529 case Intrinsic::spv_named_boolean_spec_constant: {
5530 auto Opcode = I.getOperand(3).getImm() ? SPIRV::OpSpecConstantTrue
5531 : SPIRV::OpSpecConstantFalse;
5532
5533 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
5534 .addDef(I.getOperand(0).getReg())
5535 .addUse(GR.getSPIRVTypeID(ResType));
5536 MIB.constrainAllUses(TII, TRI, RBI);
5537 unsigned SpecId = I.getOperand(2).getImm();
5538 buildOpDecorate(I.getOperand(0).getReg(), *++MIB->getIterator(), TII,
5539 SPIRV::Decoration::SpecId, {SpecId});
5540
5541 return true;
5542 }
5543 case Intrinsic::spv_const_composite: {
5544 // If no values are attached, the composite is null constant.
5545 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
5546 SmallVector<Register> CompositeArgs;
5547 MRI->setRegClass(ResVReg, GR.getRegClass(ResType));
5548
5549 // skip type MD node we already used when generated assign.type for this
5550 if (!IsNull) {
5551 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
5552 return false;
5553 std::function<bool(Register)> HasSpecConstOperand =
5554 [&](Register Reg) -> bool {
5555 MachineInstr *Def = MRI->getVRegDef(Reg);
5556 if (!Def)
5557 return false;
5558 if (!isConstReg(MRI, Def))
5559 return true;
5560 // Recurse into not-yet-selected spv_const_composite intrinsics
5561 // to detect transitive spec constant operands.
5562 if (isSpvIntrinsic(*Def, Intrinsic::spv_const_composite)) {
5563 for (unsigned J = Def->getNumExplicitDefs() + 1;
5564 J < Def->getNumExplicitOperands(); ++J) {
5565 if (Def->getOperand(J).isReg() &&
5566 HasSpecConstOperand(Def->getOperand(J).getReg()))
5567 return true;
5568 }
5569 }
5570 return false;
5571 };
5572 bool HasSpecConst = llvm::any_of(CompositeArgs, HasSpecConstOperand);
5573 unsigned CompositeOpc = HasSpecConst ? SPIRV::OpSpecConstantComposite
5574 : SPIRV::OpConstantComposite;
5575 unsigned ContinuedOpc = HasSpecConst
5576 ? SPIRV::OpSpecConstantCompositeContinuedINTEL
5577 : SPIRV::OpConstantCompositeContinuedINTEL;
5578 MachineIRBuilder MIR(I);
5579 SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions(
5580 MIR, CompositeOpc, 3, ContinuedOpc, CompositeArgs, ResVReg,
5581 GR.getSPIRVTypeID(ResType));
5582 for (auto *Instr : Instructions) {
5583 Instr->setDebugLoc(I.getDebugLoc());
5585 }
5586 return true;
5587 } else {
5588 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
5589 .addDef(ResVReg)
5590 .addUse(GR.getSPIRVTypeID(ResType));
5591 MIB.constrainAllUses(TII, TRI, RBI);
5592 return true;
5593 }
5594 }
5595 case Intrinsic::spv_assign_name: {
5596 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
5597 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
5598 for (unsigned i = I.getNumExplicitDefs() + 2;
5599 i < I.getNumExplicitOperands(); ++i) {
5600 MIB.addImm(I.getOperand(i).getImm());
5601 }
5602 MIB.constrainAllUses(TII, TRI, RBI);
5603 return true;
5604 }
5605 case Intrinsic::spv_switch: {
5606 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
5607 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
5608 if (I.getOperand(i).isReg())
5609 MIB.addReg(I.getOperand(i).getReg());
5610 else if (I.getOperand(i).isCImm())
5611 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
5612 else if (I.getOperand(i).isMBB())
5613 MIB.addMBB(I.getOperand(i).getMBB());
5614 else
5615 llvm_unreachable("Unexpected OpSwitch operand");
5616 }
5617 MIB.constrainAllUses(TII, TRI, RBI);
5618 return true;
5619 }
5620 case Intrinsic::spv_loop_merge: {
5621 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge));
5622 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
5623 if (I.getOperand(i).isMBB())
5624 MIB.addMBB(I.getOperand(i).getMBB());
5625 else
5626 MIB.addImm(foldImm(I.getOperand(i), MRI));
5627 }
5628 MIB.constrainAllUses(TII, TRI, RBI);
5629 return true;
5630 }
5631 case Intrinsic::spv_loop_control_intel: {
5632 auto MIB =
5633 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopControlINTEL));
5634 for (unsigned J = 1; J < I.getNumExplicitOperands(); ++J)
5635 MIB.addImm(foldImm(I.getOperand(J), MRI));
5636 MIB.constrainAllUses(TII, TRI, RBI);
5637 return true;
5638 }
5639 case Intrinsic::spv_selection_merge: {
5640 auto MIB =
5641 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge));
5642 assert(I.getOperand(1).isMBB() &&
5643 "operand 1 to spv_selection_merge must be a basic block");
5644 MIB.addMBB(I.getOperand(1).getMBB());
5645 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm()));
5646 MIB.constrainAllUses(TII, TRI, RBI);
5647 return true;
5648 }
5649 case Intrinsic::spv_cmpxchg:
5650 return selectAtomicCmpXchg(ResVReg, ResType, I);
5651 case Intrinsic::spv_unreachable:
5652 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable))
5653 .constrainAllUses(TII, TRI, RBI);
5654 return true;
5655 case Intrinsic::spv_abort:
5656 return selectAbort(I);
5657 case Intrinsic::spv_alloca:
5658 return selectFrameIndex(ResVReg, ResType, I);
5659 case Intrinsic::spv_alloca_array:
5660 return selectAllocaArray(ResVReg, ResType, I);
5661 case Intrinsic::spv_assume:
5662 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
5663 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
5664 .addUse(I.getOperand(1).getReg())
5665 .constrainAllUses(TII, TRI, RBI);
5666 return true;
5667 }
5668 break;
5669 case Intrinsic::spv_expect:
5670 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
5671 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
5672 .addDef(ResVReg)
5673 .addUse(GR.getSPIRVTypeID(ResType))
5674 .addUse(I.getOperand(2).getReg())
5675 .addUse(I.getOperand(3).getReg())
5676 .constrainAllUses(TII, TRI, RBI);
5677 return true;
5678 }
5679 break;
5680 case Intrinsic::arithmetic_fence:
5681 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) {
5682 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT))
5683 .addDef(ResVReg)
5684 .addUse(GR.getSPIRVTypeID(ResType))
5685 .addUse(I.getOperand(2).getReg())
5686 .constrainAllUses(TII, TRI, RBI);
5687 return true;
5688 } else
5689 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I);
5690 break;
5691 case Intrinsic::spv_thread_id:
5692 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id
5693 // intrinsic in LLVM IR for SPIR-V backend.
5694 //
5695 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a
5696 // `GlobalInvocationId` builtin variable
5697 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
5698 ResType, I);
5699 case Intrinsic::spv_thread_id_in_group:
5700 // The HLSL SV_GroupThreadId semantic is lowered to
5701 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend.
5702 //
5703 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly
5704 // translated to a `LocalInvocationId` builtin variable
5705 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg,
5706 ResType, I);
5707 case Intrinsic::spv_group_id:
5708 // The HLSL SV_GroupId semantic is lowered to
5709 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend.
5710 //
5711 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId`
5712 // builtin variable
5713 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
5714 I);
5715 case Intrinsic::spv_flattened_thread_id_in_group:
5716 // The HLSL SV_GroupIndex semantic is lowered to
5717 // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V
5718 // backend.
5719 //
5720 // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to
5721 // a `LocalInvocationIndex` builtin variable
5722 return loadBuiltinInputID(SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
5723 ResType, I);
5724 case Intrinsic::spv_workgroup_size:
5725 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupSize, ResVReg,
5726 ResType, I);
5727 case Intrinsic::spv_global_size:
5728 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalSize, ResVReg, ResType,
5729 I);
5730 case Intrinsic::spv_global_offset:
5731 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalOffset, ResVReg,
5732 ResType, I);
5733 case Intrinsic::spv_num_workgroups:
5734 return loadVec3BuiltinInputID(SPIRV::BuiltIn::NumWorkgroups, ResVReg,
5735 ResType, I);
5736 case Intrinsic::spv_subgroup_size:
5737 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType,
5738 I);
5739 case Intrinsic::spv_num_subgroups:
5740 return loadBuiltinInputID(SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType,
5741 I);
5742 case Intrinsic::spv_subgroup_id:
5743 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupId, ResVReg, ResType, I);
5744 case Intrinsic::spv_subgroup_local_invocation_id:
5745 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupLocalInvocationId,
5746 ResVReg, ResType, I);
5747 case Intrinsic::spv_subgroup_max_size:
5748 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType,
5749 I);
5750 case Intrinsic::spv_fdot:
5751 return selectFloatDot(ResVReg, ResType, I);
5752 case Intrinsic::spv_udot:
5753 case Intrinsic::spv_sdot:
5754 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5755 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
5756 return selectIntegerDot(ResVReg, ResType, I,
5757 /*Signed=*/IID == Intrinsic::spv_sdot);
5758 return selectIntegerDotExpansion(ResVReg, ResType, I);
5759 case Intrinsic::spv_dot4add_i8packed:
5760 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5761 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
5762 return selectDot4AddPacked<true>(ResVReg, ResType, I);
5763 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I);
5764 case Intrinsic::spv_dot4add_u8packed:
5765 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5766 STI.isAtLeastSPIRVVer(VersionTuple(1, 6)))
5767 return selectDot4AddPacked<false>(ResVReg, ResType, I);
5768 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I);
5769 case Intrinsic::spv_all:
5770 return selectAll(ResVReg, ResType, I);
5771 case Intrinsic::spv_any:
5772 return selectAny(ResVReg, ResType, I);
5773 case Intrinsic::spv_distance:
5774 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance);
5775 case Intrinsic::spv_lerp:
5776 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix);
5777 case Intrinsic::spv_length:
5778 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length);
5779 case Intrinsic::spv_degrees:
5780 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees);
5781 case Intrinsic::spv_faceforward:
5782 return selectExtInst(ResVReg, ResType, I, GL::FaceForward);
5783 case Intrinsic::spv_frac:
5784 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract);
5785 case Intrinsic::spv_isinf:
5786 return selectOpIsInf(ResVReg, ResType, I);
5787 case Intrinsic::spv_isnan:
5788 return selectOpIsNan(ResVReg, ResType, I);
5789 case Intrinsic::spv_isfinite:
5790 return selectOpIsFinite(ResVReg, ResType, I);
5791 case Intrinsic::spv_isnormal:
5792 return selectOpIsNormal(ResVReg, ResType, I);
5793 case Intrinsic::spv_normalize:
5794 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize);
5795 case Intrinsic::spv_refract:
5796 return selectExtInst(ResVReg, ResType, I, GL::Refract);
5797 case Intrinsic::spv_reflect:
5798 return selectExtInst(ResVReg, ResType, I, GL::Reflect);
5799 case Intrinsic::spv_rsqrt:
5800 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt);
5801 case Intrinsic::spv_sign:
5802 return selectSign(ResVReg, ResType, I);
5803 case Intrinsic::spv_smoothstep:
5804 return selectExtInst(ResVReg, ResType, I, CL::smoothstep, GL::SmoothStep);
5805 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb
5806 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false);
5807 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb
5808 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true);
5809 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb
5810 return selectFirstBitLow(ResVReg, ResType, I);
5811 case Intrinsic::spv_all_memory_barrier:
5812 return selectBarrierInst(I, SPIRV::Scope::Device,
5813 SPIRV::MemorySemantics::UniformMemory |
5814 SPIRV::MemorySemantics::ImageMemory |
5815 SPIRV::MemorySemantics::WorkgroupMemory,
5816 /*WithGroupSync*/ false);
5817 case Intrinsic::spv_all_memory_barrier_with_group_sync:
5818 return selectBarrierInst(I, SPIRV::Scope::Device,
5819 SPIRV::MemorySemantics::UniformMemory |
5820 SPIRV::MemorySemantics::ImageMemory |
5821 SPIRV::MemorySemantics::WorkgroupMemory,
5822 /*WithGroupSync*/ true);
5823 case Intrinsic::spv_device_memory_barrier:
5824 return selectBarrierInst(I, SPIRV::Scope::Device,
5825 SPIRV::MemorySemantics::UniformMemory |
5826 SPIRV::MemorySemantics::ImageMemory,
5827 /*WithGroupSync*/ false);
5828 case Intrinsic::spv_device_memory_barrier_with_group_sync:
5829 return selectBarrierInst(I, SPIRV::Scope::Device,
5830 SPIRV::MemorySemantics::UniformMemory |
5831 SPIRV::MemorySemantics::ImageMemory,
5832 /*WithGroupSync*/ true);
5833 case Intrinsic::spv_group_memory_barrier:
5834 return selectBarrierInst(I, SPIRV::Scope::Workgroup,
5835 SPIRV::MemorySemantics::WorkgroupMemory,
5836 /*WithGroupSync*/ false);
5837 case Intrinsic::spv_group_memory_barrier_with_group_sync:
5838 return selectBarrierInst(I, SPIRV::Scope::Workgroup,
5839 SPIRV::MemorySemantics::WorkgroupMemory,
5840 /*WithGroupSync*/ true);
5841 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
5842 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 1).getReg();
5843 SPIRV::StorageClass::StorageClass ResSC =
5844 GR.getPointerStorageClass(ResType);
5845 if (!isGenericCastablePtr(ResSC))
5846 return diagnoseUnsupported(I, "The target storage class is not castable "
5847 "from the Generic storage class");
5848 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGenericCastToPtrExplicit))
5849 .addDef(ResVReg)
5850 .addUse(GR.getSPIRVTypeID(ResType))
5851 .addUse(PtrReg)
5852 .addImm(ResSC)
5853 .constrainAllUses(TII, TRI, RBI);
5854 return true;
5855 }
5856 case Intrinsic::spv_lifetime_start:
5857 case Intrinsic::spv_lifetime_end: {
5858 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
5859 : SPIRV::OpLifetimeStop;
5860 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm();
5861 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg();
5862 if (Size == -1)
5863 Size = 0;
5864 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op))
5865 .addUse(PtrReg)
5866 .addImm(Size)
5867 .constrainAllUses(TII, TRI, RBI);
5868 return true;
5869 }
5870 case Intrinsic::spv_saturate:
5871 return selectSaturate(ResVReg, ResType, I);
5872 case Intrinsic::spv_nclamp:
5873 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp);
5874 case Intrinsic::spv_uclamp:
5875 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp);
5876 case Intrinsic::spv_sclamp:
5877 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp);
5878 case Intrinsic::spv_subgroup_prefix_bit_count:
5879 return selectWavePrefixBitCount(ResVReg, ResType, I);
5880 case Intrinsic::spv_wave_active_countbits:
5881 return selectWaveActiveCountBits(ResVReg, ResType, I);
5882 case Intrinsic::spv_wave_all_equal:
5883 return selectWaveActiveAllEqual(ResVReg, ResType, I);
5884 case Intrinsic::spv_wave_all:
5885 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll);
5886 case Intrinsic::spv_wave_any:
5887 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
5888 case Intrinsic::spv_subgroup_ballot:
5889 return selectWaveOpInst(ResVReg, ResType, I,
5890 SPIRV::OpGroupNonUniformBallot);
5891 case Intrinsic::spv_wave_is_first_lane:
5892 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect);
5893 case Intrinsic::spv_wave_reduce_or:
5894 return selectWaveReduceOp(ResVReg, ResType, I,
5895 SPIRV::OpGroupNonUniformBitwiseOr);
5896 case Intrinsic::spv_wave_reduce_xor:
5897 return selectWaveReduceOp(ResVReg, ResType, I,
5898 SPIRV::OpGroupNonUniformBitwiseXor);
5899 case Intrinsic::spv_wave_reduce_and:
5900 return selectWaveReduceOp(ResVReg, ResType, I,
5901 SPIRV::OpGroupNonUniformBitwiseAnd);
5902 case Intrinsic::spv_wave_reduce_umax:
5903 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true);
5904 case Intrinsic::spv_wave_reduce_max:
5905 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false);
5906 case Intrinsic::spv_wave_reduce_umin:
5907 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ true);
5908 case Intrinsic::spv_wave_reduce_min:
5909 return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ false);
5910 case Intrinsic::spv_wave_reduce_sum:
5911 return selectWaveReduceSum(ResVReg, ResType, I);
5912 case Intrinsic::spv_wave_product:
5913 return selectWaveReduceProduct(ResVReg, ResType, I);
5914 case Intrinsic::spv_wave_readlane:
5915 return selectWaveOpInst(ResVReg, ResType, I,
5916 SPIRV::OpGroupNonUniformShuffle);
5917 case Intrinsic::spv_wave_readlane_first:
5918 return selectWaveOpInst(ResVReg, ResType, I,
5919 SPIRV::OpGroupNonUniformBroadcastFirst);
5920 case Intrinsic::spv_wave_prefix_sum:
5921 return selectWaveExclusiveScanSum(ResVReg, ResType, I);
5922 case Intrinsic::spv_subgroup_multi_prefix_bit_and:
5923 return selectWaveMultiPrefixOp(ResVReg, ResType, I,
5924 SPIRV::OpGroupNonUniformBitwiseAnd,
5925 "WaveMultiPrefixBitAnd");
5926 case Intrinsic::spv_subgroup_multi_prefix_bit_or:
5927 return selectWaveMultiPrefixOp(ResVReg, ResType, I,
5928 SPIRV::OpGroupNonUniformBitwiseOr,
5929 "WaveMultiPrefixBitOr");
5930 case Intrinsic::spv_subgroup_multi_prefix_bit_xor:
5931 return selectWaveMultiPrefixOp(ResVReg, ResType, I,
5932 SPIRV::OpGroupNonUniformBitwiseXor,
5933 "WaveMultiPrefixBitXor");
5934 case Intrinsic::spv_subgroup_multi_prefix_product: {
5935 bool IsFloatTy =
5936 GR.isScalarOrVectorOfType(I.getOperand(2).getReg(), SPIRV::OpTypeFloat);
5937 return selectWaveMultiPrefixOp(ResVReg, ResType, I,
5938 IsFloatTy ? SPIRV::OpGroupNonUniformFMul
5939 : SPIRV::OpGroupNonUniformIMul,
5940 "WaveMultiPrefixProduct");
5941 }
5942 case Intrinsic::spv_subgroup_multi_prefix_sum: {
5943 bool IsFloatTy =
5944 GR.isScalarOrVectorOfType(I.getOperand(2).getReg(), SPIRV::OpTypeFloat);
5945 return selectWaveMultiPrefixOp(ResVReg, ResType, I,
5946 IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
5947 : SPIRV::OpGroupNonUniformIAdd,
5948 "WaveMultiPrefixSum");
5949 }
5950 case Intrinsic::spv_wave_prefix_product:
5951 return selectWaveExclusiveScanProduct(ResVReg, ResType, I);
5952 case Intrinsic::spv_quad_all:
5953 return selectQuadAll(ResVReg, ResType, I);
5954 case Intrinsic::spv_quad_read_across_x: {
5955 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 0);
5956 }
5957 case Intrinsic::spv_quad_read_across_y: {
5958 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 1);
5959 }
5960 case Intrinsic::spv_quad_read_across_diagonal: {
5961 return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 2);
5962 }
5963 case Intrinsic::spv_radians:
5964 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians);
5965 // Discard intrinsics which we do not expect to actually represent code after
5966 // lowering or intrinsics which are not implemented but should not crash when
5967 // found in a customer's LLVM IR input.
5968 case Intrinsic::instrprof_increment:
5969 case Intrinsic::instrprof_increment_step:
5970 case Intrinsic::instrprof_value_profile:
5971 break;
5972 // Discard internal intrinsics.
5973 case Intrinsic::spv_value_md:
5974 break;
5975 case Intrinsic::spv_resource_handlefrombinding: {
5976 return selectHandleFromBinding(ResVReg, ResType, I);
5977 }
5978 case Intrinsic::spv_resource_counterhandlefrombinding:
5979 return selectCounterHandleFromBinding(ResVReg, ResType, I);
5980 case Intrinsic::spv_resource_updatecounter:
5981 return selectUpdateCounter(ResVReg, ResType, I);
5982 case Intrinsic::spv_resource_store_typedbuffer: {
5983 return selectImageWriteIntrinsic(I);
5984 }
5985 case Intrinsic::spv_resource_load_typedbuffer: {
5986 return selectReadImageIntrinsic(ResVReg, ResType, I);
5987 }
5988 case Intrinsic::spv_resource_load_level: {
5989 return selectLoadLevelIntrinsic(ResVReg, ResType, I);
5990 }
5991 case Intrinsic::spv_resource_getdimensions_x:
5992 case Intrinsic::spv_resource_getdimensions_xy:
5993 case Intrinsic::spv_resource_getdimensions_xyz: {
5994 return selectGetDimensionsIntrinsic(ResVReg, ResType, I);
5995 }
5996 case Intrinsic::spv_resource_getdimensions_levels_x:
5997 case Intrinsic::spv_resource_getdimensions_levels_xy:
5998 case Intrinsic::spv_resource_getdimensions_levels_xyz: {
5999 return selectGetDimensionsLevelsIntrinsic(ResVReg, ResType, I);
6000 }
6001 case Intrinsic::spv_resource_getdimensions_ms_xy:
6002 case Intrinsic::spv_resource_getdimensions_ms_xyz: {
6003 return selectGetDimensionsMSIntrinsic(ResVReg, ResType, I);
6004 }
6005 case Intrinsic::spv_resource_calculate_lod:
6006 case Intrinsic::spv_resource_calculate_lod_unclamped:
6007 return selectCalculateLodIntrinsic(ResVReg, ResType, I);
6008 case Intrinsic::spv_resource_sample:
6009 case Intrinsic::spv_resource_sample_clamp:
6010 return selectSampleBasicIntrinsic(ResVReg, ResType, I);
6011 case Intrinsic::spv_resource_samplebias:
6012 case Intrinsic::spv_resource_samplebias_clamp:
6013 return selectSampleBiasIntrinsic(ResVReg, ResType, I);
6014 case Intrinsic::spv_resource_samplegrad:
6015 case Intrinsic::spv_resource_samplegrad_clamp:
6016 return selectSampleGradIntrinsic(ResVReg, ResType, I);
6017 case Intrinsic::spv_resource_samplelevel:
6018 return selectSampleLevelIntrinsic(ResVReg, ResType, I);
6019 case Intrinsic::spv_resource_samplecmp:
6020 case Intrinsic::spv_resource_samplecmp_clamp:
6021 return selectSampleCmpIntrinsic(ResVReg, ResType, I);
6022 case Intrinsic::spv_resource_samplecmplevelzero:
6023 return selectSampleCmpLevelZeroIntrinsic(ResVReg, ResType, I);
6024 case Intrinsic::spv_resource_gather:
6025 case Intrinsic::spv_resource_gather_cmp:
6026 return selectGatherIntrinsic(ResVReg, ResType, I);
6027 case Intrinsic::spv_resource_getbasepointer:
6028 case Intrinsic::spv_resource_getpointer: {
6029 return selectResourceGetPointer(ResVReg, ResType, I);
6030 }
6031 case Intrinsic::spv_pushconstant_getpointer: {
6032 return selectPushConstantGetPointer(ResVReg, ResType, I);
6033 }
6034 case Intrinsic::spv_discard: {
6035 return selectDiscard(ResVReg, ResType, I);
6036 }
6037 case Intrinsic::spv_resource_nonuniformindex: {
6038 return selectResourceNonUniformIndex(ResVReg, ResType, I);
6039 }
6040 case Intrinsic::spv_unpackhalf2x16: {
6041 return selectExtInst(ResVReg, ResType, I, GL::UnpackHalf2x16);
6042 }
6043 case Intrinsic::spv_packhalf2x16: {
6044 return selectExtInst(ResVReg, ResType, I, GL::PackHalf2x16);
6045 }
6046 case Intrinsic::spv_ddx:
6047 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdx);
6048 case Intrinsic::spv_ddy:
6049 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdy);
6050 case Intrinsic::spv_ddx_coarse:
6051 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxCoarse);
6052 case Intrinsic::spv_ddy_coarse:
6053 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyCoarse);
6054 case Intrinsic::spv_ddx_fine:
6055 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxFine);
6056 case Intrinsic::spv_ddy_fine:
6057 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyFine);
6058 case Intrinsic::spv_fwidth:
6059 return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpFwidth);
6060 case Intrinsic::spv_masked_gather:
6061 if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
6062 return selectMaskedGather(ResVReg, ResType, I);
6063 return diagnoseUnsupported(
6064 I, "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
6065 case Intrinsic::spv_masked_scatter:
6066 if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
6067 return selectMaskedScatter(I);
6068 return diagnoseUnsupported(
6069 I, "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
6070 case Intrinsic::returnaddress:
6071 case Intrinsic::frameaddress: {
6072 // SPIR-V does not have a stack or return address. Lower to null.
6073 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
6074 .addDef(ResVReg)
6075 .addUse(GR.getSPIRVTypeID(ResType));
6076 MIB.constrainAllUses(TII, TRI, RBI);
6077 return true;
6078 }
6079 default:
6080 return diagnoseUnsupported(I, "intrinsic selection not implemented.");
6081 }
6082 return true;
6083}
6084
6085bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg,
6086 SPIRVTypeInst ResType,
6087 MachineInstr &I) const {
6088 // Image users reload the handle, so this load serves only other users.
6089 return loadHandleBeforePosition(ResVReg, ResType, *cast<GIntrinsic>(&I), I);
6090}
6091
6092bool SPIRVInstructionSelector::selectCounterHandleFromBinding(
6093 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6094 auto &Intr = cast<GIntrinsic>(I);
6095 assert(Intr.getIntrinsicID() ==
6096 Intrinsic::spv_resource_counterhandlefrombinding);
6097
6098 // Extract information from the intrinsic call.
6099 Register MainHandleReg = Intr.getOperand(2).getReg();
6100 auto *MainHandleDef = cast<GIntrinsic>(getVRegDef(*MRI, MainHandleReg));
6101 assert(MainHandleDef->getIntrinsicID() ==
6102 Intrinsic::spv_resource_handlefrombinding);
6103
6104 uint32_t Set = getIConstVal(Intr.getOperand(3).getReg(), MRI);
6105 uint32_t Binding = getIConstVal(Intr.getOperand(4).getReg(), MRI);
6106 uint32_t ArraySize = getIConstVal(MainHandleDef->getOperand(4).getReg(), MRI);
6107 Register IndexReg = MainHandleDef->getOperand(5).getReg();
6108 std::string CounterName =
6109 getStringValueFromReg(MainHandleDef->getOperand(6).getReg(), *MRI) +
6110 ".counter";
6111
6112 // Create the counter variable.
6113 MachineIRBuilder MIRBuilder(I);
6114 Register CounterVarReg =
6115 buildPointerToResource(SPIRVTypeInst(GR.getPointeeType(ResType)),
6116 GR.getPointerStorageClass(ResType), Set, Binding,
6117 ArraySize, IndexReg, CounterName, MIRBuilder);
6118
6119 return BuildCOPY(ResVReg, CounterVarReg, I);
6120}
6121
6122bool SPIRVInstructionSelector::selectUpdateCounter(Register &ResVReg,
6123 SPIRVTypeInst ResType,
6124 MachineInstr &I) const {
6125 auto &Intr = cast<GIntrinsic>(I);
6126 assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter);
6127
6128 Register CounterHandleReg = Intr.getOperand(2).getReg();
6129 int64_t IncrVal = Intr.getOperand(3).getImm();
6130
6131 // The counter handle is a pointer to the counter variable (which is a struct
6132 // containing an i32). We need to get a pointer to that i32 member to do the
6133 // atomic operation.
6134#ifndef NDEBUG
6135 SPIRVTypeInst CounterVarType = GR.getSPIRVTypeForVReg(CounterHandleReg);
6136 SPIRVTypeInst CounterVarPointeeType = GR.getPointeeType(CounterVarType);
6137 assert(CounterVarPointeeType &&
6138 CounterVarPointeeType->getOpcode() == SPIRV::OpTypeStruct &&
6139 "Counter variable must be a struct");
6140 assert(GR.getPointerStorageClass(CounterVarType) ==
6141 SPIRV::StorageClass::StorageBuffer &&
6142 "Counter variable must be in the storage buffer storage class");
6143 assert(CounterVarPointeeType->getNumOperands() == 2 &&
6144 "Counter variable must have exactly 1 member in the struct");
6145 const SPIRVTypeInst MemberType =
6146 GR.getSPIRVTypeForVReg(CounterVarPointeeType->getOperand(1).getReg());
6147 assert(MemberType->getOpcode() == SPIRV::OpTypeInt &&
6148 "Counter variable struct must have a single i32 member");
6149#endif
6150
6151 // The struct has a single i32 member.
6152 MachineIRBuilder MIRBuilder(I);
6153 const Type *LLVMIntType =
6154 Type::getInt32Ty(I.getMF()->getFunction().getContext());
6155
6156 SPIRVTypeInst IntPtrType = GR.getOrCreateSPIRVPointerType(
6157 LLVMIntType, MIRBuilder, SPIRV::StorageClass::StorageBuffer);
6158
6159 Register Zero = buildI32ConstantInEntryBlock(0, I);
6160
6161 Register PtrToCounter =
6162 MRI->createVirtualRegister(GR.getRegClass(IntPtrType));
6163 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain))
6164 .addDef(PtrToCounter)
6165 .addUse(GR.getSPIRVTypeID(IntPtrType))
6166 .addUse(CounterHandleReg)
6167 .addUse(Zero)
6168 .constrainAllUses(TII, TRI, RBI);
6169
6170 // For UAV/SSBO counters, the scope is Device. The counter variable is not
6171 // used as a flag. So the memory semantics can be None.
6172 Register Scope = buildI32ConstantInEntryBlock(SPIRV::Scope::Device, I);
6173 Register Semantics =
6174 buildI32ConstantInEntryBlock(SPIRV::MemorySemantics::None, I);
6175
6176 Register Incr =
6177 buildI32ConstantInEntryBlock(static_cast<uint32_t>(IncrVal), I);
6178
6179 Register AtomicRes = MRI->createVirtualRegister(GR.getRegClass(ResType));
6180 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAtomicIAdd))
6181 .addDef(AtomicRes)
6182 .addUse(GR.getSPIRVTypeID(ResType))
6183 .addUse(PtrToCounter)
6184 .addUse(Scope)
6185 .addUse(Semantics)
6186 .addUse(Incr)
6187 .constrainAllUses(TII, TRI, RBI);
6188 if (IncrVal >= 0) {
6189 return BuildCOPY(ResVReg, AtomicRes, I);
6190 }
6191
6192 // In HLSL, IncrementCounter returns the value *before* the increment, while
6193 // DecrementCounter returns the value *after* the decrement. Both are lowered
6194 // to the same atomic intrinsic which returns the value *before* the
6195 // operation. So for decrements (negative IncrVal), we must subtract the
6196 // increment value from the result to get the post-decrement value.
6197 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS))
6198 .addDef(ResVReg)
6199 .addUse(GR.getSPIRVTypeID(ResType))
6200 .addUse(AtomicRes)
6201 .addUse(Incr)
6202 .constrainAllUses(TII, TRI, RBI);
6203 return true;
6204}
6205bool SPIRVInstructionSelector::selectReadImageIntrinsic(Register &ResVReg,
6206 SPIRVTypeInst ResType,
6207 MachineInstr &I) const {
6208
6209 // If the load of the image is in a different basic block, then
6210 // this will generate invalid code. A proper solution is to move
6211 // the OpLoad from selectHandleFromBinding here. However, to do
6212 // that we will need to change the return type of the intrinsic.
6213 // We will do that when we can, but for now trying to move forward with other
6214 // issues.
6215 Register ImageReg = I.getOperand(2).getReg();
6216 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6217 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6218 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6219 *ImageDef, I)) {
6220 return false;
6221 }
6222
6223 Register IdxReg = I.getOperand(3).getReg();
6224 DebugLoc Loc = I.getDebugLoc();
6225 MachineInstr &Pos = I;
6226
6227 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, IdxReg, Loc,
6228 Pos);
6229}
6230
6231bool SPIRVInstructionSelector::generateSampleImage(
6232 Register ResVReg, SPIRVTypeInst ResType, Register ImageReg,
6233 Register SamplerReg, Register CoordinateReg, const ImageOperands &ImOps,
6234 DebugLoc Loc, MachineInstr &Pos) const {
6235 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6236 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6237 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6238 *ImageDef, Pos)) {
6239 return false;
6240 }
6241
6242 auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
6243 Register NewSamplerReg =
6245 if (!loadHandleBeforePosition(NewSamplerReg,
6246 GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef,
6247 Pos)) {
6248 return false;
6249 }
6250
6251 MachineIRBuilder MIRBuilder(Pos);
6252 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
6253 GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder);
6254 Register SampledImageReg =
6255 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
6256
6257 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpSampledImage))
6258 .addDef(SampledImageReg)
6259 .addUse(GR.getSPIRVTypeID(SampledImageType))
6260 .addUse(NewImageReg)
6261 .addUse(NewSamplerReg)
6262 .constrainAllUses(TII, TRI, RBI);
6263
6264 bool IsExplicitLod = ImOps.GradX.has_value() || ImOps.GradY.has_value() ||
6265 ImOps.Lod.has_value();
6266 unsigned Opcode = IsExplicitLod ? SPIRV::OpImageSampleExplicitLod
6267 : SPIRV::OpImageSampleImplicitLod;
6268 if (ImOps.Compare)
6269 Opcode = IsExplicitLod ? SPIRV::OpImageSampleDrefExplicitLod
6270 : SPIRV::OpImageSampleDrefImplicitLod;
6271
6272 auto MIB = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(Opcode))
6273 .addDef(ResVReg)
6274 .addUse(GR.getSPIRVTypeID(ResType))
6275 .addUse(SampledImageReg)
6276 .addUse(CoordinateReg);
6277
6278 if (ImOps.Compare)
6279 MIB.addUse(*ImOps.Compare);
6280
6281 uint32_t ImageOperands = 0;
6282 if (ImOps.Bias)
6283 ImageOperands |= SPIRV::ImageOperand::Bias;
6284 if (ImOps.Lod)
6285 ImageOperands |= SPIRV::ImageOperand::Lod;
6286 if (ImOps.GradX && ImOps.GradY)
6287 ImageOperands |= SPIRV::ImageOperand::Grad;
6288 if (ImOps.Offset && !isScalarOrVectorIntConstantZero(*ImOps.Offset)) {
6289 if (isConstReg(MRI, *ImOps.Offset))
6290 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
6291 else {
6292 Pos.emitGenericError(
6293 "Non-constant offsets are not supported in sample instructions.");
6294 return false;
6295 }
6296 }
6297 if (ImOps.MinLod)
6298 ImageOperands |= SPIRV::ImageOperand::MinLod;
6299
6300 if (ImageOperands != 0) {
6301 MIB.addImm(ImageOperands);
6302 if (ImageOperands & SPIRV::ImageOperand::Bias)
6303 MIB.addUse(*ImOps.Bias);
6304 if (ImageOperands & SPIRV::ImageOperand::Lod)
6305 MIB.addUse(*ImOps.Lod);
6306 if (ImageOperands & SPIRV::ImageOperand::Grad) {
6307 MIB.addUse(*ImOps.GradX);
6308 MIB.addUse(*ImOps.GradY);
6309 }
6310 if (ImageOperands &
6311 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
6312 MIB.addUse(*ImOps.Offset);
6313 if (ImageOperands & SPIRV::ImageOperand::MinLod)
6314 MIB.addUse(*ImOps.MinLod);
6315 }
6316
6317 MIB.constrainAllUses(TII, TRI, RBI);
6318 return true;
6319}
6320
6321bool SPIRVInstructionSelector::selectImageQuerySize(
6322 Register ImageReg, Register &ResVReg, MachineInstr &I,
6323 std::optional<Register> LodReg) const {
6324 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
6325 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
6326 "ImageReg is not an image type.");
6327
6328 auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm());
6329 bool IsArray = ImageType->getOperand(4).getImm() != 0;
6330 unsigned NumComponents = 0;
6331 switch (Dim) {
6332 case SPIRV::Dim::DIM_1D:
6333 case SPIRV::Dim::DIM_Buffer:
6334 NumComponents = IsArray ? 2 : 1;
6335 break;
6336 case SPIRV::Dim::DIM_2D:
6337 case SPIRV::Dim::DIM_Cube:
6338 case SPIRV::Dim::DIM_Rect:
6339 NumComponents = IsArray ? 3 : 2;
6340 break;
6341 case SPIRV::Dim::DIM_3D:
6342 NumComponents = 3;
6343 break;
6344 default:
6345 I.emitGenericError("Unsupported image dimension for OpImageQuerySize.");
6346 return false;
6347 }
6348
6349 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
6350 // Non-multisampled 1D, 2D, 3D, and Cube images with Sampled = 1 (compatible
6351 // with sampling operations) require an explicit LOD. Query mip level zero
6352 // when GetDimensions does not supply one.
6353 bool IsSampled = ImageType->getOperand(6).getImm() == 1;
6354 bool IsMultisampled = ImageType->getOperand(5).getImm() != 0;
6355 if (!LodReg && IsSampled && !IsMultisampled &&
6356 Dim != SPIRV::Dim::DIM_Buffer && Dim != SPIRV::Dim::DIM_Rect)
6357 LodReg = GR.getOrCreateConstInt(0, I, I32Ty, TII, !STI.isShader());
6358 unsigned Opcode =
6359 LodReg ? SPIRV::OpImageQuerySizeLod : SPIRV::OpImageQuerySize;
6360 SPIRVTypeInst ResType =
6361 NumComponents == 1
6362 ? I32Ty
6363 : GR.getOrCreateSPIRVVectorType(I32Ty, NumComponents, I, TII);
6364
6365 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
6366 .addDef(ResVReg)
6367 .addUse(GR.getSPIRVTypeID(ResType))
6368 .addUse(ImageReg);
6369 if (LodReg)
6370 MIB.addUse(*LodReg);
6371 MIB.constrainAllUses(TII, TRI, RBI);
6372 return true;
6373}
6374
6375bool SPIRVInstructionSelector::selectGetDimensionsIntrinsic(
6376 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6377 Register ImageReg = I.getOperand(2).getReg();
6378 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6379 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6380 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6381 *ImageDef, I)) {
6382 return false;
6383 }
6384 return selectImageQuerySize(NewImageReg, ResVReg, I);
6385}
6386
6387bool SPIRVInstructionSelector::selectGetDimensionsLevelsIntrinsic(
6388 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6389 Register ImageReg = I.getOperand(2).getReg();
6390 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6391 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6392 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6393 *ImageDef, I)) {
6394 return false;
6395 }
6396
6397 Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6398 Register LodReg = I.getOperand(3).getReg();
6399
6400 assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(6).getImm() == 1 &&
6401 "OpImageQuerySizeLod and OpImageQueryLevels require a sampled image");
6402
6403 if (!selectImageQuerySize(NewImageReg, SizeReg, I, LodReg)) {
6404 return false;
6405 }
6406
6407 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
6408 Register LevelsReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6409 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6410 TII.get(SPIRV::OpImageQueryLevels))
6411 .addDef(LevelsReg)
6412 .addUse(GR.getSPIRVTypeID(I32Ty))
6413 .addUse(NewImageReg)
6414 .constrainAllUses(TII, TRI, RBI);
6415
6416 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6417 TII.get(SPIRV::OpCompositeConstruct))
6418 .addDef(ResVReg)
6419 .addUse(GR.getSPIRVTypeID(ResType))
6420 .addUse(SizeReg)
6421 .addUse(LevelsReg)
6422 .constrainAllUses(TII, TRI, RBI);
6423
6424 return true;
6425}
6426
6427bool SPIRVInstructionSelector::selectGetDimensionsMSIntrinsic(
6428 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6429 Register ImageReg = I.getOperand(2).getReg();
6430 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6431 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6432 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6433 *ImageDef, I)) {
6434 return false;
6435 }
6436
6437 Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6438
6439 assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(5).getImm() == 1 &&
6440 "OpImageQuerySamples requires a multisampled image");
6441
6442 if (!selectImageQuerySize(NewImageReg, SizeReg, I)) {
6443 return false;
6444 }
6445
6446 Register SamplesReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
6447
6448 SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII);
6449 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6450 TII.get(SPIRV::OpImageQuerySamples))
6451 .addDef(SamplesReg)
6452 .addUse(GR.getSPIRVTypeID(I32Ty))
6453 .addUse(NewImageReg)
6454 .constrainAllUses(TII, TRI, RBI);
6455
6456 BuildMI(*I.getParent(), I, I.getDebugLoc(),
6457 TII.get(SPIRV::OpCompositeConstruct))
6458 .addDef(ResVReg)
6459 .addUse(GR.getSPIRVTypeID(ResType))
6460 .addUse(SizeReg)
6461 .addUse(SamplesReg)
6462 .constrainAllUses(TII, TRI, RBI);
6463
6464 return true;
6465}
6466
6467bool SPIRVInstructionSelector::selectCalculateLodIntrinsic(
6468 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6469 Register ImageReg = I.getOperand(2).getReg();
6470 Register SamplerReg = I.getOperand(3).getReg();
6471 Register CoordinateReg = I.getOperand(4).getReg();
6472
6473 auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6474 if (!ImageDef)
6475 return false;
6476 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6477 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6478 *ImageDef, I)) {
6479 return false;
6480 }
6481
6482 auto *SamplerDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
6483 if (!SamplerDef)
6484 return false;
6485 Register NewSamplerReg =
6487 if (!loadHandleBeforePosition(
6488 NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) {
6489 return false;
6490 }
6491
6492 MachineIRBuilder MIRBuilder(I);
6493 SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage(
6494 GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder);
6495 Register SampledImageReg =
6496 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
6497
6498 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage))
6499 .addDef(SampledImageReg)
6500 .addUse(GR.getSPIRVTypeID(SampledImageType))
6501 .addUse(NewImageReg)
6502 .addUse(NewSamplerReg)
6503 .constrainAllUses(TII, TRI, RBI);
6504
6505 SPIRVTypeInst Vec2Ty = GR.getOrCreateSPIRVVectorType(ResType, 2, I, TII);
6506 Register QueryResultReg = MRI->createVirtualRegister(GR.getRegClass(Vec2Ty));
6507
6508 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageQueryLod))
6509 .addDef(QueryResultReg)
6510 .addUse(GR.getSPIRVTypeID(Vec2Ty))
6511 .addUse(SampledImageReg)
6512 .addUse(CoordinateReg)
6513 .constrainAllUses(TII, TRI, RBI);
6514
6515 unsigned ExtractedIndex =
6516 cast<GIntrinsic>(I).getIntrinsicID() ==
6517 Intrinsic::spv_resource_calculate_lod_unclamped
6518 ? 1
6519 : 0;
6520
6521 MachineInstrBuilder MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
6522 TII.get(SPIRV::OpCompositeExtract))
6523 .addDef(ResVReg)
6524 .addUse(GR.getSPIRVTypeID(ResType))
6525 .addUse(QueryResultReg)
6526 .addImm(ExtractedIndex);
6527
6528 MIB.constrainAllUses(TII, TRI, RBI);
6529 return true;
6530}
6531
6532bool SPIRVInstructionSelector::selectSampleBasicIntrinsic(
6533 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6534 Register ImageReg = I.getOperand(2).getReg();
6535 Register SamplerReg = I.getOperand(3).getReg();
6536 Register CoordinateReg = I.getOperand(4).getReg();
6537 ImageOperands ImOps;
6538 if (I.getNumOperands() > 5)
6539 ImOps.Offset = I.getOperand(5).getReg();
6540 if (I.getNumOperands() > 6)
6541 ImOps.MinLod = I.getOperand(6).getReg();
6542 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6543 CoordinateReg, ImOps, I.getDebugLoc(), I);
6544}
6545
6546bool SPIRVInstructionSelector::selectSampleBiasIntrinsic(
6547 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6548 Register ImageReg = I.getOperand(2).getReg();
6549 Register SamplerReg = I.getOperand(3).getReg();
6550 Register CoordinateReg = I.getOperand(4).getReg();
6551 ImageOperands ImOps;
6552 ImOps.Bias = I.getOperand(5).getReg();
6553 if (I.getNumOperands() > 6)
6554 ImOps.Offset = I.getOperand(6).getReg();
6555 if (I.getNumOperands() > 7)
6556 ImOps.MinLod = I.getOperand(7).getReg();
6557 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6558 CoordinateReg, ImOps, I.getDebugLoc(), I);
6559}
6560
6561bool SPIRVInstructionSelector::selectSampleGradIntrinsic(
6562 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6563 Register ImageReg = I.getOperand(2).getReg();
6564 Register SamplerReg = I.getOperand(3).getReg();
6565 Register CoordinateReg = I.getOperand(4).getReg();
6566 ImageOperands ImOps;
6567 ImOps.GradX = I.getOperand(5).getReg();
6568 ImOps.GradY = I.getOperand(6).getReg();
6569 if (I.getNumOperands() > 7)
6570 ImOps.Offset = I.getOperand(7).getReg();
6571 if (I.getNumOperands() > 8)
6572 ImOps.MinLod = I.getOperand(8).getReg();
6573 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6574 CoordinateReg, ImOps, I.getDebugLoc(), I);
6575}
6576
6577bool SPIRVInstructionSelector::selectSampleLevelIntrinsic(
6578 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6579 Register ImageReg = I.getOperand(2).getReg();
6580 Register SamplerReg = I.getOperand(3).getReg();
6581 Register CoordinateReg = I.getOperand(4).getReg();
6582 ImageOperands ImOps;
6583 ImOps.Lod = I.getOperand(5).getReg();
6584 if (I.getNumOperands() > 6)
6585 ImOps.Offset = I.getOperand(6).getReg();
6586 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6587 CoordinateReg, ImOps, I.getDebugLoc(), I);
6588}
6589
6590bool SPIRVInstructionSelector::selectSampleCmpIntrinsic(Register &ResVReg,
6591 SPIRVTypeInst ResType,
6592 MachineInstr &I) const {
6593 Register ImageReg = I.getOperand(2).getReg();
6594 Register SamplerReg = I.getOperand(3).getReg();
6595 Register CoordinateReg = I.getOperand(4).getReg();
6596 ImageOperands ImOps;
6597 ImOps.Compare = I.getOperand(5).getReg();
6598 if (I.getNumOperands() > 6)
6599 ImOps.Offset = I.getOperand(6).getReg();
6600 if (I.getNumOperands() > 7)
6601 ImOps.MinLod = I.getOperand(7).getReg();
6602 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6603 CoordinateReg, ImOps, I.getDebugLoc(), I);
6604}
6605
6606bool SPIRVInstructionSelector::selectLoadLevelIntrinsic(Register &ResVReg,
6607 SPIRVTypeInst ResType,
6608 MachineInstr &I) const {
6609 Register ImageReg = I.getOperand(2).getReg();
6610 Register CoordinateReg = I.getOperand(3).getReg();
6611 Register LodReg = I.getOperand(4).getReg();
6612
6613 ImageOperands ImOps;
6614 ImOps.Lod = LodReg;
6615 if (I.getNumOperands() > 5)
6616 ImOps.Offset = I.getOperand(5).getReg();
6617
6618 auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6619 if (!ImageDef)
6620 return false;
6621
6622 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6623 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6624 *ImageDef, I)) {
6625 return false;
6626 }
6627
6628 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, CoordinateReg,
6629 I.getDebugLoc(), I, &ImOps);
6630}
6631
6632bool SPIRVInstructionSelector::selectSampleCmpLevelZeroIntrinsic(
6633 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6634 Register ImageReg = I.getOperand(2).getReg();
6635 Register SamplerReg = I.getOperand(3).getReg();
6636 Register CoordinateReg = I.getOperand(4).getReg();
6637 ImageOperands ImOps;
6638 ImOps.Compare = I.getOperand(5).getReg();
6639 if (I.getNumOperands() > 6)
6640 ImOps.Offset = I.getOperand(6).getReg();
6641 SPIRVTypeInst FloatTy = GR.getOrCreateSPIRVFloatType(32, I, TII);
6642 ImOps.Lod = GR.getOrCreateConstFP(APFloat(0.0f), I, FloatTy, TII);
6643 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6644 CoordinateReg, ImOps, I.getDebugLoc(), I);
6645}
6646
6647bool SPIRVInstructionSelector::selectGatherIntrinsic(Register &ResVReg,
6648 SPIRVTypeInst ResType,
6649 MachineInstr &I) const {
6650 Register ImageReg = I.getOperand(2).getReg();
6651 Register SamplerReg = I.getOperand(3).getReg();
6652 Register CoordinateReg = I.getOperand(4).getReg();
6653 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
6654 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
6655 "ImageReg is not an image type.");
6656
6657 Register ComponentOrCompareReg;
6658 Register OffsetReg;
6659
6660 ComponentOrCompareReg = I.getOperand(5).getReg();
6661 OffsetReg = I.getOperand(6).getReg();
6662 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6663 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6664 if (!loadHandleBeforePosition(NewImageReg, ImageType, *ImageDef, I)) {
6665 return false;
6666 }
6667
6668 auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm());
6669 if (Dim != SPIRV::Dim::DIM_2D && Dim != SPIRV::Dim::DIM_Cube &&
6670 Dim != SPIRV::Dim::DIM_Rect) {
6671 I.emitGenericError(
6672 "Gather operations are only supported for 2D, Cube, and Rect images.");
6673 return false;
6674 }
6675
6676 auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg));
6677 Register NewSamplerReg =
6679 if (!loadHandleBeforePosition(
6680 NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) {
6681 return false;
6682 }
6683
6684 MachineIRBuilder MIRBuilder(I);
6685 SPIRVTypeInst SampledImageType =
6686 GR.getOrCreateOpTypeSampledImage(ImageType, MIRBuilder);
6687 Register SampledImageReg =
6688 MRI->createVirtualRegister(GR.getRegClass(SampledImageType));
6689
6690 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage))
6691 .addDef(SampledImageReg)
6692 .addUse(GR.getSPIRVTypeID(SampledImageType))
6693 .addUse(NewImageReg)
6694 .addUse(NewSamplerReg)
6695 .constrainAllUses(TII, TRI, RBI);
6696
6697 auto IntrId = cast<GIntrinsic>(I).getIntrinsicID();
6698 bool IsGatherCmp = IntrId == Intrinsic::spv_resource_gather_cmp;
6699 unsigned Opcode =
6700 IsGatherCmp ? SPIRV::OpImageDrefGather : SPIRV::OpImageGather;
6701
6702 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
6703 .addDef(ResVReg)
6704 .addUse(GR.getSPIRVTypeID(ResType))
6705 .addUse(SampledImageReg)
6706 .addUse(CoordinateReg)
6707 .addUse(ComponentOrCompareReg);
6708
6709 uint32_t ImageOperands = 0;
6710 if (OffsetReg && !isScalarOrVectorIntConstantZero(OffsetReg)) {
6711 if (Dim == SPIRV::Dim::DIM_Cube) {
6712 I.emitGenericError(
6713 "Gather operations with offset are not supported for Cube images.");
6714 return false;
6715 }
6716 if (isConstReg(MRI, OffsetReg))
6717 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
6718 else {
6719 ImageOperands |= SPIRV::ImageOperand::Offset;
6720 }
6721 }
6722
6723 if (ImageOperands != 0) {
6724 MIB.addImm(ImageOperands);
6725 if (ImageOperands &
6726 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
6727 MIB.addUse(OffsetReg);
6728 }
6729
6730 MIB.constrainAllUses(TII, TRI, RBI);
6731 return true;
6732}
6733
6734bool SPIRVInstructionSelector::generateImageReadOrFetch(
6735 Register &ResVReg, SPIRVTypeInst ResType, Register ImageReg,
6736 Register IdxReg, DebugLoc Loc, MachineInstr &Pos,
6737 const ImageOperands *ImOps) const {
6738 SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg);
6739 assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
6740 "ImageReg is not an image type.");
6741
6742 bool IsSignedInteger =
6743 sampledTypeIsSignedInteger(GR.getTypeForSPIRVType(ImageType));
6744 // Check if the "sampled" operand of the image type is 1.
6745 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpImageFetch
6746 auto SampledOp = ImageType->getOperand(6);
6747 bool IsFetch = (SampledOp.getImm() == 1);
6748
6749 auto AddOperands = [&](MachineInstrBuilder &MIB) {
6750 uint32_t ImageOperandsMask = 0;
6751 if (IsSignedInteger)
6752 ImageOperandsMask |= 0x1000; // SignExtend
6753
6754 if (IsFetch && ImOps) {
6755 if (ImOps->Lod)
6756 ImageOperandsMask |= SPIRV::ImageOperand::Lod;
6757 if (ImOps->Offset && !isScalarOrVectorIntConstantZero(*ImOps->Offset)) {
6758 if (isConstReg(MRI, *ImOps->Offset))
6759 ImageOperandsMask |= SPIRV::ImageOperand::ConstOffset;
6760 else
6761 ImageOperandsMask |= SPIRV::ImageOperand::Offset;
6762 }
6763 }
6764
6765 if (ImageOperandsMask != 0) {
6766 MIB.addImm(ImageOperandsMask);
6767 if (IsFetch && ImOps) {
6768 if (ImOps->Lod)
6769 MIB.addUse(*ImOps->Lod);
6770 if (ImOps->Offset &&
6771 (ImageOperandsMask &
6772 (SPIRV::ImageOperand::Offset | SPIRV::ImageOperand::ConstOffset)))
6773 MIB.addUse(*ImOps->Offset);
6774 }
6775 }
6776 };
6777
6778 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
6779
6780 // A wide element (e.g. int64_t2) is emulated with a narrower packed image, so
6781 // its sampled type is different from the result.
6782 SPIRVTypeInst SampledType =
6783 GR.getSPIRVTypeForVReg(ImageType->getOperand(1).getReg());
6784 bool IsPacked = SampledType != GR.getScalarOrVectorComponentType(ResType);
6785 SPIRVTypeInst ReadType =
6786 widenTypeToVec4(IsPacked ? SampledType : ResType, Pos);
6787 bool ReadTypeMatchesResult = ReadType == ResType;
6788 // Read directly into the result, or into a temporary to bitcast/extract.
6789 Register ReadReg = ReadTypeMatchesResult
6790 ? ResVReg
6791 : MRI->createVirtualRegister(GR.getRegClass(ReadType));
6792
6793 auto BMI =
6794 BuildMI(*Pos.getParent(), Pos, Loc,
6795 TII.get(IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
6796 .addDef(ReadReg)
6797 .addUse(GR.getSPIRVTypeID(ReadType))
6798 .addUse(ImageReg)
6799 .addUse(IdxReg);
6800 AddOperands(BMI);
6801 BMI.constrainAllUses(TII, TRI, RBI);
6802
6803 if (ReadTypeMatchesResult)
6804 return true;
6805
6806 if (IsPacked) {
6807 // Reinterpret the packed vector as the wide result type.
6808 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpBitcast))
6809 .addDef(ResVReg)
6810 .addUse(GR.getSPIRVTypeID(ResType))
6811 .addUse(ReadReg)
6812 .constrainAllUses(TII, TRI, RBI);
6813 return true;
6814 }
6815
6816 if (ResultSize == 1) {
6817 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpCompositeExtract))
6818 .addDef(ResVReg)
6819 .addUse(GR.getSPIRVTypeID(ResType))
6820 .addUse(ReadReg)
6821 .addImm(0)
6822 .constrainAllUses(TII, TRI, RBI);
6823 return true;
6824 }
6825 return extractSubvector(ResVReg, ResType, ReadReg, Pos);
6826}
6827
6828bool SPIRVInstructionSelector::selectResourceGetPointer(Register &ResVReg,
6829 SPIRVTypeInst ResType,
6830 MachineInstr &I) const {
6831 Register ResourcePtr = I.getOperand(2).getReg();
6832 SPIRVTypeInst RegType = GR.getSPIRVTypeForVReg(ResourcePtr, I.getMF());
6833 if (RegType->getOpcode() == SPIRV::OpTypeImage) {
6834 // For texel buffers, the index into the image is part of the OpImageRead or
6835 // OpImageWrite instructions. So we will do nothing in this case. This
6836 // intrinsic will be combined with the load or store when selecting the load
6837 // or store.
6838 return true;
6839 }
6840
6841 assert(ResType->getOpcode() == SPIRV::OpTypePointer);
6842
6843 Register ZeroReg =
6844 buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
6845 auto MIB =
6846 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain))
6847 .addDef(ResVReg)
6848 .addUse(GR.getSPIRVTypeID(ResType))
6849 .addUse(ResourcePtr)
6850 .addUse(ZeroReg);
6851
6852 if (I.getNumExplicitOperands() > 3) {
6853 Register IndexReg = I.getOperand(3).getReg();
6854 MIB.addUse(IndexReg);
6855 }
6856 MIB.constrainAllUses(TII, TRI, RBI);
6857 return true;
6858}
6859
6860bool SPIRVInstructionSelector::selectPushConstantGetPointer(
6861 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6862 MRI->replaceRegWith(ResVReg, I.getOperand(2).getReg());
6863 return true;
6864}
6865
6866bool SPIRVInstructionSelector::selectResourceNonUniformIndex(
6867 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const {
6868 Register ObjReg = I.getOperand(2).getReg();
6869 if (!BuildCOPY(ResVReg, ObjReg, I))
6870 return false;
6871
6872 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {});
6873 // Check for the registers that use the index marked as non-uniform
6874 // and recursively mark them as non-uniform.
6875 // Per the spec, it's necessary that the final argument used for
6876 // load/store/sample/atomic must be decorated, so we need to propagate the
6877 // decoration through access chains and copies.
6878 // https://docs.vulkan.org/samples/latest/samples/extensions/descriptor_indexing/README.html#_when_to_use_non_uniform_indexing_qualifier
6879 decorateUsesAsNonUniform(ResVReg);
6880 return true;
6881}
6882
6883void SPIRVInstructionSelector::decorateUsesAsNonUniform(
6884 Register &NonUniformReg) const {
6886 {NonUniformReg, nullptr}};
6887 llvm::SmallSet<Register, 8> Visited;
6888 while (WorkList.size() > 0) {
6889 auto [CurrentReg, DefMI] = WorkList.pop_back_val();
6890
6891 if (!Visited.insert(CurrentReg).second)
6892 continue;
6893
6894 bool IsDecorated = false;
6895 for (MachineInstr &Use : MRI->use_instructions(CurrentReg)) {
6896 if (Use.getOpcode() == SPIRV::OpDecorate &&
6897 Use.getOperand(1).getImm() == SPIRV::Decoration::NonUniformEXT) {
6898 IsDecorated = true;
6899 continue;
6900 }
6901 // Check if the instruction has the result register and add it to the
6902 // worklist.
6903 if (Use.getOperand(0).isReg() && Use.getOperand(0).isDef()) {
6904 Register ResultReg = Use.getOperand(0).getReg();
6905 if (ResultReg == CurrentReg)
6906 continue;
6907 WorkList.push_back({ResultReg, &Use});
6908 }
6909 }
6910
6911 if (!IsDecorated) {
6912 MachineBasicBlock &MBB = *DefMI->getParent();
6913 MachineInstr &InsertPt =
6915 buildOpDecorate(CurrentReg, InsertPt, TII,
6916 SPIRV::Decoration::NonUniformEXT, {});
6917 }
6918 }
6919}
6920
6921bool SPIRVInstructionSelector::extractSubvector(
6922 Register &ResVReg, SPIRVTypeInst ResType, Register &ReadReg,
6923 MachineInstr &InsertionPoint) const {
6924 SPIRVTypeInst InputType = GR.getResultType(ReadReg);
6925 [[maybe_unused]] uint64_t InputSize =
6926 GR.getScalarOrVectorComponentCount(InputType);
6927 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
6928 [[maybe_unused]] bool IsLongVectorEXT =
6929 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector);
6930 assert((InputSize > 1 || IsLongVectorEXT) && "The input must be a vector.");
6931 assert((ResultSize > 1 || IsLongVectorEXT) && "The result must be a vector.");
6932 assert(ResultSize < InputSize &&
6933 "Cannot extract more element than there are in the input.");
6934 SmallVector<Register> ComponentRegisters;
6935 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(ResType);
6936 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType);
6937 for (uint64_t I = 0; I < ResultSize; I++) {
6938 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass);
6939 BuildMI(*InsertionPoint.getParent(), InsertionPoint,
6940 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
6941 .addDef(ComponentReg)
6942 .addUse(ScalarType->getOperand(0).getReg())
6943 .addUse(ReadReg)
6944 .addImm(I)
6945 .constrainAllUses(TII, TRI, RBI);
6946 ComponentRegisters.emplace_back(ComponentReg);
6947 }
6948
6949 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint,
6950 InsertionPoint.getDebugLoc(),
6951 TII.get(SPIRV::OpCompositeConstruct))
6952 .addDef(ResVReg)
6953 .addUse(GR.getSPIRVTypeID(ResType));
6954
6955 for (Register ComponentReg : ComponentRegisters)
6956 MIB.addUse(ComponentReg);
6957 MIB.constrainAllUses(TII, TRI, RBI);
6958 return true;
6959}
6960
6961bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
6962 MachineInstr &I) const {
6963 // If the load of the image is in a different basic block, then
6964 // this will generate invalid code. A proper solution is to move
6965 // the OpLoad from selectHandleFromBinding here. However, to do
6966 // that we will need to change the return type of the intrinsic.
6967 // We will do that when we can, but for now trying to move forward with other
6968 // issues.
6969 Register ImageReg = I.getOperand(1).getReg();
6970 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg));
6971 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg));
6972 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg),
6973 *ImageDef, I)) {
6974 return false;
6975 }
6976
6977 Register CoordinateReg = I.getOperand(2).getReg();
6978 Register DataReg = I.getOperand(3).getReg();
6979 assert(isVectorType(GR.getResultType(DataReg)));
6981 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite))
6982 .addUse(NewImageReg)
6983 .addUse(CoordinateReg)
6984 .addUse(DataReg)
6985 .constrainAllUses(TII, TRI, RBI);
6986 return true;
6987}
6988
6989Register SPIRVInstructionSelector::buildPointerToResource(
6990 SPIRVTypeInst SpirvResType, SPIRV::StorageClass::StorageClass SC,
6991 uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg,
6992 StringRef Name, MachineIRBuilder MIRBuilder) const {
6993 const Type *ResType = GR.getTypeForSPIRVType(SpirvResType);
6994 if (ArraySize == 1) {
6995 SPIRVTypeInst PtrType =
6996 GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
6997 assert(GR.getPointeeType(PtrType) == SpirvResType &&
6998 "SpirvResType did not have an explicit layout.");
6999 return GR.getOrCreateGlobalVariableWithBinding(PtrType, Set, Binding, Name,
7000 MIRBuilder);
7001 }
7002
7003 const Type *VarType = ArrayType::get(const_cast<Type *>(ResType), ArraySize);
7004 SPIRVTypeInst VarPointerType =
7005 GR.getOrCreateSPIRVPointerType(VarType, MIRBuilder, SC);
7007 VarPointerType, Set, Binding, Name, MIRBuilder);
7008
7009 SPIRVTypeInst ResPointerType =
7010 GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC);
7011 Register AcReg = MRI->createVirtualRegister(GR.getRegClass(ResPointerType));
7012
7013 MIRBuilder.buildInstr(SPIRV::OpAccessChain)
7014 .addDef(AcReg)
7015 .addUse(GR.getSPIRVTypeID(ResPointerType))
7016 .addUse(VarReg)
7017 .addUse(IndexReg);
7018
7019 return AcReg;
7020}
7021
7022bool SPIRVInstructionSelector::selectFirstBitSet16(
7023 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
7024 unsigned ExtendOpcode, unsigned BitSetOpcode) const {
7025 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
7026 if (!selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()},
7027 ExtendOpcode))
7028 return false;
7029
7030 return selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode);
7031}
7032
7033bool SPIRVInstructionSelector::selectFirstBitSet32(
7034 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
7035 unsigned BitSetOpcode) const {
7036 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
7037 .addDef(ResVReg)
7038 .addUse(GR.getSPIRVTypeID(ResType))
7039 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
7040 .addImm(BitSetOpcode)
7041 .addUse(SrcReg)
7042 .constrainAllUses(TII, TRI, RBI);
7043 return true;
7044}
7045
7046bool SPIRVInstructionSelector::selectFirstBitSet64(
7047 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg,
7048 unsigned BitSetOpcode, bool SwapPrimarySide) const {
7049 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType);
7050 SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType);
7051 bool ZeroAsNull = !STI.isShader();
7052 Register ConstIntZero =
7053 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull);
7054 Register ConstIntOne =
7055 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull);
7056
7057 // SPIRV doesn't support vectors with more than 4 components. Since the
7058 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only
7059 // operate on vectors with 2 or less components. When largers vectors are
7060 // seen. Split them, recurse, then recombine them.
7061 // TODO: handle the case where SPV_EXT_long_vector is enabled.
7062 if (ComponentCount > 2) {
7063 auto Func = [this, SwapPrimarySide](Register ResVReg, SPIRVTypeInst ResType,
7064 MachineInstr &I, Register SrcReg,
7065 unsigned Opcode) -> bool {
7066 return this->selectFirstBitSet64(ResVReg, ResType, I, SrcReg, Opcode,
7067 SwapPrimarySide);
7068 };
7069
7070 return handle64BitOverflow(ResVReg, ResType, I, SrcReg, BitSetOpcode, Func);
7071 }
7072
7073 // 1. Split int64 into 2 pieces using a bitcast
7074 MachineIRBuilder MIRBuilder(I);
7075 SPIRVTypeInst PostCastType = GR.getOrCreateSPIRVVectorType(
7076 BaseType, 2 * ComponentCount, MIRBuilder, false);
7077 Register BitcastReg =
7078 MRI->createVirtualRegister(GR.getRegClass(PostCastType));
7079
7080 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg},
7081 SPIRV::OpBitcast))
7082 return false;
7083
7084 // 2. Find the first set bit from the primary side for all the pieces in #1
7085 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType));
7086 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode))
7087 return false;
7088
7089 // 3. Split result vector into high bits and low bits
7090 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
7091 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
7092
7093 bool IsScalarRes = !isVectorType(ResType);
7094 if (IsScalarRes) {
7095 // if scalar do a vector extract
7096 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntOne},
7097 SPIRV::OpVectorExtractDynamic))
7098 return false;
7099 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntZero},
7100 SPIRV::OpVectorExtractDynamic))
7101 return false;
7102 } else {
7103 // if vector do a shufflevector
7104 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
7105 TII.get(SPIRV::OpVectorShuffle))
7106 .addDef(HighReg)
7107 .addUse(GR.getSPIRVTypeID(ResType))
7108 .addUse(FBSReg)
7109 // Per the spec, repeat the vector if only one vec is needed
7110 .addUse(FBSReg);
7111
7112 // high bits are stored in even natural indexes. Extract them from FBSReg
7113 for (unsigned J = 1; J < ComponentCount * 2; J += 2) {
7114 MIB.addImm(J);
7115 }
7116
7117 MIB.constrainAllUses(TII, TRI, RBI);
7118
7119 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
7120 TII.get(SPIRV::OpVectorShuffle))
7121 .addDef(LowReg)
7122 .addUse(GR.getSPIRVTypeID(ResType))
7123 .addUse(FBSReg)
7124 // Per the spec, repeat the vector if only one vec is needed
7125 .addUse(FBSReg);
7126
7127 // low bits are stored in odd natural indices. Extract them from FBSReg
7128 for (unsigned J = 0; J < ComponentCount * 2; J += 2) {
7129 MIB.addImm(J);
7130 }
7131 MIB.constrainAllUses(TII, TRI, RBI);
7132 }
7133
7134 // 4. Check the result. When primary bits == -1 use secondary, otherwise use
7135 // primary
7136 SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
7137 Register NegOneReg;
7138 Register Reg0;
7139 Register Reg32;
7140 unsigned SelectOp;
7141 unsigned AddOp;
7142
7143 if (IsScalarRes) {
7144 NegOneReg =
7145 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull);
7146 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
7147 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull);
7148 SelectOp = SPIRV::OpSelectSISCond;
7149 AddOp = SPIRV::OpIAddS;
7150 } else {
7151 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount,
7152 MIRBuilder, false);
7153 NegOneReg =
7154 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull);
7155 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull);
7156 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull);
7157 SelectOp = SPIRV::OpSelectVIVCond;
7158 AddOp = SPIRV::OpIAddV;
7159 }
7160
7161 Register PrimaryReg = HighReg;
7162 Register SecondaryReg = LowReg;
7163 Register RegPrimaryOffset = Reg32;
7164 Register RegSecondaryOffset = Reg0;
7165
7166 // By default the emitted opcodes check for the set bit from the MSB side.
7167 // Setting SwapPrimarySide checks the set bit from the LSB side
7168 if (SwapPrimarySide) {
7169 PrimaryReg = LowReg;
7170 SecondaryReg = HighReg;
7171 RegPrimaryOffset = Reg0;
7172 RegSecondaryOffset = Reg32;
7173 }
7174
7175 Register RegSecondaryHasVal =
7176 MRI->createVirtualRegister(GR.getRegClass(BoolType));
7177 if (!selectOpWithSrcs(RegSecondaryHasVal, BoolType, I,
7178 {SecondaryReg, NegOneReg}, SPIRV::OpINotEqual))
7179 return false;
7180
7181 Register RegPrimaryHasVal =
7182 MRI->createVirtualRegister(GR.getRegClass(BoolType));
7183 if (!selectOpWithSrcs(RegPrimaryHasVal, BoolType, I, {PrimaryReg, NegOneReg},
7184 SPIRV::OpINotEqual))
7185 return false;
7186
7187 // Pass 1: seed with secondary (lower-priority fallback)
7188 // ReturnBits = secondaryHasVal ? SecondaryBits : -1
7189 // Add = secondaryHasVal ? SecondaryOffset : 0
7190 Register RegReturnBits = MRI->createVirtualRegister(GR.getRegClass(ResType));
7191 if (!selectOpWithSrcs(RegReturnBits, ResType, I,
7192 {RegSecondaryHasVal, SecondaryReg, NegOneReg},
7193 SelectOp))
7194 return false;
7195
7196 Register RegAdd;
7197 if (SwapPrimarySide) {
7198 RegAdd = MRI->createVirtualRegister(GR.getRegClass(ResType));
7199 if (!selectOpWithSrcs(RegAdd, ResType, I,
7200 {RegSecondaryHasVal, RegSecondaryOffset, Reg0},
7201 SelectOp))
7202 return false;
7203 } else {
7204 RegAdd = Reg0;
7205 }
7206
7207 // Pass 2: override with primary (higher priority) if it has a valid result
7208 // ReturnBits2 = primaryHasVal ? PrimaryBits : ReturnBits
7209 // Add2 = primaryHasVal ? PrimaryOffset : Add
7210 Register RegReturnBits2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
7211 if (!selectOpWithSrcs(RegReturnBits2, ResType, I,
7212 {RegPrimaryHasVal, PrimaryReg, RegReturnBits},
7213 SelectOp))
7214 return false;
7215
7216 Register RegAdd2 = MRI->createVirtualRegister(GR.getRegClass(ResType));
7217 if (!selectOpWithSrcs(RegAdd2, ResType, I,
7218 {RegPrimaryHasVal, RegPrimaryOffset, RegAdd}, SelectOp))
7219 return false;
7220
7221 return selectOpWithSrcs(ResVReg, ResType, I, {RegReturnBits2, RegAdd2},
7222 AddOp);
7223}
7224
7225bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg,
7226 SPIRVTypeInst ResType,
7227 MachineInstr &I,
7228 bool IsSigned) const {
7229 // FindUMsb and FindSMsb intrinsics only support 32 bit integers
7230 Register OpReg = I.getOperand(2).getReg();
7231 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
7232 // zero or sign extend
7233 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
7234 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
7235
7236 switch (GR.getScalarOrVectorBitWidth(OpType)) {
7237 case 16:
7238 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
7239 case 32:
7240 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
7241 case 64:
7242 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
7243 /*SwapPrimarySide=*/false);
7244 default:
7245 return diagnoseUnsupported(
7246 I,
7247 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
7248 }
7249}
7250
7251bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg,
7252 SPIRVTypeInst ResType,
7253 MachineInstr &I) const {
7254 // FindILsb intrinsic only supports 32 bit integers
7255 Register OpReg = I.getOperand(2).getReg();
7256 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg);
7257 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it
7258 // to an unsigned i32. As this leaves all the least significant bits unchanged
7259 // so the first set bit from the LSB side doesn't change.
7260 unsigned ExtendOpcode = SPIRV::OpUConvert;
7261 unsigned BitSetOpcode = GL::FindILsb;
7262
7263 switch (GR.getScalarOrVectorBitWidth(OpType)) {
7264 case 16:
7265 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode);
7266 case 32:
7267 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode);
7268 case 64:
7269 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode,
7270 /*SwapPrimarySide=*/true);
7271 default:
7272 return diagnoseUnsupported(I,
7273 "spv_firstbitlow only supports 16,32,64 bits.");
7274 }
7275}
7276
7277bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
7278 SPIRVTypeInst ResType,
7279 MachineInstr &I) const {
7280 // there was an allocation size parameter to the allocation instruction
7281 // that is not 1
7282 MachineBasicBlock &BB = *I.getParent();
7283
7284 bool UseUntypedPointers =
7285 ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7286 unsigned Opcode = UseUntypedPointers
7287 ? SPIRV::OpUntypedVariableLengthArrayINTEL
7288 : SPIRV::OpVariableLengthArrayINTEL;
7289
7290 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
7291 .addDef(ResVReg)
7292 .addUse(GR.getSPIRVTypeID(ResType));
7293
7294 // OpUntypedVariableLengthArrayINTEL takes an explicit Element Type <id>
7295 // right after the result type
7296 if (UseUntypedPointers) {
7297 SPIRVTypeInst ElementType = GR.getUntypedPtrElementType(ResVReg);
7298 assert(ElementType &&
7299 "untyped variable length array result must have a recorded element "
7300 "type");
7301 MIB.addUse(GR.getSPIRVTypeID(ElementType));
7302 }
7303
7304 MIB.addUse(I.getOperand(2).getReg());
7305 MIB.constrainAllUses(TII, TRI, RBI);
7306
7307 if (!STI.isShader()) {
7308 unsigned Alignment = I.getOperand(3).getImm();
7309 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment});
7310 }
7311 return true;
7312}
7313
7314// Returns true iff `Ty` is a concrete SPIR-V type per the SPV_KHR_abort
7315// definition: a numerical scalar (int/float), a (physical) pointer, a vector,
7316// matrix or any aggregate (array/struct) recursively containing only such
7317// types. OpTypeBool, OpTypeVoid, opaque handles and similar abstract
7318// non-concrete types are rejected.
7320 const SPIRVGlobalRegistry &GR) {
7321 SmallVector<SPIRVTypeInst, 4> Worklist{Ty};
7322 while (!Worklist.empty()) {
7323 SPIRVTypeInst T = Worklist.pop_back_val();
7324 switch (T->getOpcode()) {
7325 case SPIRV::OpTypeInt:
7326 case SPIRV::OpTypeFloat:
7327 case SPIRV::OpTypePointer:
7328 break;
7329 case SPIRV::OpTypeVector:
7330 case SPIRV::OpTypeVectorIdEXT:
7331 case SPIRV::OpTypeMatrix:
7332 case SPIRV::OpTypeArray: {
7333 Register OperandReg = T->getOperand(1).getReg();
7334 SPIRVTypeInst ElementT = GR.getSPIRVTypeForVReg(OperandReg);
7335 Worklist.push_back(ElementT);
7336 } break;
7337 case SPIRV::OpTypeStruct:
7338 for (unsigned Idx = 1, E = T->getNumOperands(); Idx < E; ++Idx) {
7339 Register OperandReg = T->getOperand(Idx).getReg();
7340 SPIRVTypeInst ElementT = GR.getSPIRVTypeForVReg(OperandReg);
7341 Worklist.push_back(ElementT);
7342 }
7343 break;
7344 default:
7345 return false;
7346 }
7347 }
7348 return true;
7349}
7350
7352 const SPIRVGlobalRegistry &GR,
7353 SmallSet<Register, 8> &Visited) {
7354 Register TypeReg = Ty->getOperand(0).getReg();
7355 if (!Visited.insert(TypeReg).second)
7356 return false;
7357
7358 switch (Ty->getOpcode()) {
7359 case SPIRV::OpTypePointer:
7360 if (Ty->getOperand(1).getImm() == SPIRV::StorageClass::StorageBuffer)
7361 return true;
7363 GR.getSPIRVTypeForVReg(Ty->getOperand(2).getReg()), GR, Visited);
7364 case SPIRV::OpTypeArray:
7365 case SPIRV::OpTypeRuntimeArray:
7367 GR.getSPIRVTypeForVReg(Ty->getOperand(1).getReg()), GR, Visited);
7368 case SPIRV::OpTypeStruct:
7369 for (unsigned I = 1; I < Ty->getNumOperands(); ++I)
7371 GR.getSPIRVTypeForVReg(Ty->getOperand(I).getReg()), GR, Visited))
7372 return true;
7373 return false;
7374 default:
7375 return false;
7376 }
7377}
7378
7380 const SPIRVGlobalRegistry &GR) {
7381 SmallSet<Register, 8> Visited;
7382 return containsStorageBufferPointer(Ty, GR, Visited);
7383}
7384
7385bool SPIRVInstructionSelector::selectAbort(MachineInstr &I) const {
7386 assert(I.getNumExplicitOperands() == 2);
7387
7388 Register MsgReg = I.getOperand(1).getReg();
7389 SPIRVTypeInst MsgType = GR.getSPIRVTypeForVReg(MsgReg);
7390 assert(MsgType && "Message argument of llvm.spv.abort has no SPIR-V type");
7391
7392 if (!isConcreteSPIRVType(MsgType, GR))
7393 return diagnoseUnsupported(
7394 I,
7395 "llvm.spv.abort message type must be a concrete SPIR-V type (numerical "
7396 "scalar, pointer, vector, matrix, or aggregate of such types)");
7397
7398 MachineBasicBlock &BB = *I.getParent();
7399 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAbortKHR))
7400 .addUse(GR.getSPIRVTypeID(MsgType))
7401 .addUse(MsgReg)
7402 .constrainAllUses(TII, TRI, RBI);
7403 return true;
7404}
7405
7406bool SPIRVInstructionSelector::selectTrap(MachineInstr &I) const {
7407 // When the SPV_KHR_abort extension is disabled, drop the G_TRAP and
7408 // G_UBSANTRAP silently. Also drop the no-op placeholder of the service
7409 // function.
7410 if (!STI.canUseExtension(SPIRV::Extension::SPV_KHR_abort) ||
7411 I.getMF()->getFunction().hasFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME))
7412 return true;
7413
7414 // Use the 32-bit integer constant for the abort "message" argument:
7415 // - G_UBSANTRAP operand is zero-extended to 32 bits.
7416 // - "All ones" constant is used for G_TRAP.
7417 uint32_t MsgVal = ~0u;
7418 if (I.getOpcode() == TargetOpcode::G_UBSANTRAP)
7419 MsgVal = static_cast<uint32_t>(I.getOperand(0).getImm());
7420
7421 SPIRVTypeInst MsgType = GR.getOrCreateSPIRVIntegerType(32, I, TII);
7422 Register MsgReg = buildI32ConstantInEntryBlock(MsgVal, I, MsgType);
7423
7424 MachineBasicBlock &BB = *I.getParent();
7425 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAbortKHR))
7426 .addUse(GR.getSPIRVTypeID(MsgType))
7427 .addUse(MsgReg)
7428 .constrainAllUses(TII, TRI, RBI);
7429 return true;
7430}
7431
7432bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
7433 SPIRVTypeInst ResType,
7434 MachineInstr &I) const {
7435 // Change order of instructions if needed: all OpVariable instructions in a
7436 // function must be the first instructions in the first block
7437 auto It = getOpVariableMBBIt(*I.getMF());
7438
7439 // Pointers to opaque types stay typed even with the extension on, so emit the
7440 // untyped variant only when the result is actually an untyped pointer.
7441 bool UseUntypedPointers =
7442 ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7443 unsigned Opcode =
7444 UseUntypedPointers ? SPIRV::OpUntypedVariableKHR : SPIRV::OpVariable;
7445
7446 if (!UseUntypedPointers && containsStorageBufferPointer(ResType, GR)) {
7447 MachineIRBuilder MIRBuilder(I);
7448 if (!STI.isAtLeastSPIRVVer(VersionTuple(1, 3)))
7449 MIRBuilder.buildInstr(SPIRV::OpExtension)
7450 .addImm(SPIRV::Extension::SPV_KHR_variable_pointers);
7451 MIRBuilder.buildInstr(SPIRV::OpCapability)
7452 .addImm(SPIRV::Capability::VariablePointersStorageBuffer);
7453 }
7454
7455 auto MIB = BuildMI(*It->getParent(), It, It->getDebugLoc(), TII.get(Opcode))
7456 .addDef(ResVReg)
7457 .addUse(GR.getSPIRVTypeID(ResType))
7458 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
7459
7460 // OpUntypedVariableKHR takes an extra Data Type operand right after the
7461 // storage class.
7462 if (UseUntypedPointers) {
7463 // Get the element type that was stored when processing spv_assign_ptr_type.
7464 SPIRVTypeInst DataType = GR.getUntypedPtrElementType(ResVReg);
7465 if (!DataType)
7466 return diagnoseUnsupported(
7467 I, "could not deduce the data type of an untyped variable");
7468 MIB.addUse(GR.getSPIRVTypeID(DataType));
7469 }
7470 MIB.constrainAllUses(TII, TRI, RBI);
7471
7472 if (!STI.isShader()) {
7473 unsigned Alignment = I.getOperand(2).getImm();
7474 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment,
7475 {Alignment});
7476 }
7477 return true;
7478}
7479
7480bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
7481 // InstructionSelector walks backwards through the instructions. We can use
7482 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
7483 // first, so can generate an OpBranchConditional here. If there is no
7484 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
7485 const MachineInstr *PrevI = I.getPrevNode();
7486 MachineBasicBlock &MBB = *I.getParent();
7487 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
7488 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
7489 .addUse(PrevI->getOperand(0).getReg())
7490 .addMBB(PrevI->getOperand(1).getMBB())
7491 .addMBB(I.getOperand(0).getMBB())
7492 .constrainAllUses(TII, TRI, RBI);
7493 return true;
7494 }
7495 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
7496 .addMBB(I.getOperand(0).getMBB())
7497 .constrainAllUses(TII, TRI, RBI);
7498 return true;
7499}
7500
7501bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
7502 // InstructionSelector walks backwards through the instructions. For an
7503 // explicit conditional branch with no fallthrough, we use both a G_BR and a
7504 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
7505 // generate the OpBranchConditional in selectBranch above.
7506 //
7507 // If an OpBranchConditional has been generated, we simply return, as the work
7508 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
7509 // implicit fallthrough to the next basic block, so we need to create an
7510 // OpBranchConditional with an explicit "false" argument pointing to the next
7511 // basic block that LLVM would fall through to.
7512 const MachineInstr *NextI = I.getNextNode();
7513 // Check if this has already been successfully selected.
7514 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
7515 return true;
7516 // Must be relying on implicit block fallthrough, so generate an
7517 // OpBranchConditional with the "next" basic block as the "false" target.
7518 MachineBasicBlock &MBB = *I.getParent();
7519 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
7520 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
7521 BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
7522 .addUse(I.getOperand(0).getReg())
7523 .addMBB(I.getOperand(1).getMBB())
7524 .addMBB(NextMBB)
7525 .constrainAllUses(TII, TRI, RBI);
7526 return true;
7527}
7528
7529bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
7530 MachineInstr &I) const {
7531 auto MIB =
7532 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::PHI))
7533 .addDef(ResVReg);
7534 const unsigned NumOps = I.getNumOperands();
7535 for (unsigned i = 1; i < NumOps; i += 2) {
7536 MIB.addUse(I.getOperand(i + 0).getReg());
7537 MIB.addMBB(I.getOperand(i + 1).getMBB());
7538 }
7539 MIB.constrainAllUses(TII, TRI, RBI);
7540 return true;
7541}
7542
7543bool SPIRVInstructionSelector::selectGlobalValue(
7544 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
7545 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
7546 MachineIRBuilder MIRBuilder(I);
7547 const GlobalValue *GV = I.getOperand(1).getGlobal();
7549
7550 std::string GlobalIdent;
7551 if (!GV->hasName()) {
7552 unsigned &ID = UnnamedGlobalIDs[GV];
7553 if (ID == 0)
7554 ID = UnnamedGlobalIDs.size();
7555 GlobalIdent = "__unnamed_" + Twine(ID).str();
7556 } else {
7557 GlobalIdent = GV->getName();
7558 }
7559
7560 // Behaviour of functions as operands depends on availability of the
7561 // corresponding extension (SPV_INTEL_function_pointers):
7562 // - If there is an extension to operate with functions as operands:
7563 // We create a proper constant operand and evaluate a correct type for a
7564 // function pointer.
7565 // - Without the required extension:
7566 // We have functions as operands in tests with blocks of instruction e.g. in
7567 // transcoding/global_block.ll. These operands are not used and should be
7568 // substituted by zero constants. Their type is expected to be always
7569 // OpTypePointer Function %uchar.
7570 if (isa<Function>(GV)) {
7571 const Constant *ConstVal = GV;
7572 MachineBasicBlock &BB = *I.getParent();
7573 Register NewReg = GR.find(ConstVal, GR.CurMF);
7574 if (!NewReg.isValid()) {
7575 const Function *GVFun =
7576 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)
7577 ? dyn_cast<Function>(GV)
7578 : nullptr;
7579 SPIRVTypeInst ResType = GR.getOrCreateSPIRVPointerType(
7580 GVType, I,
7581 GVFun ? SPIRV::StorageClass::CodeSectionINTEL
7583 if (GVFun) {
7584 // References to a function via function pointers generate virtual
7585 // registers without a definition. We will resolve it later, during
7586 // module analysis stage.
7587 Register ResTypeReg = GR.getSPIRVTypeID(ResType);
7588 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
7589 Register FuncVReg =
7591 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass);
7592 GR.assignSPIRVTypeToVReg(ResType, FuncVReg, *GR.CurMF);
7593 MachineInstrBuilder MIB1 =
7594 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
7595 .addDef(FuncVReg)
7596 .addUse(ResTypeReg);
7597 MachineInstrBuilder MIB2 =
7598 BuildMI(BB, I, I.getDebugLoc(),
7599 TII.get(SPIRV::OpConstantFunctionPointerINTEL))
7600 .addDef(ResVReg)
7601 .addUse(ResTypeReg)
7602 .addUse(FuncVReg);
7603 GR.add(ConstVal, MIB2);
7604 // mapping the function pointer to the used Function
7605 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun);
7606 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
7607 MIB1.constrainAllUses(TII, TRI, RBI);
7608 MIB2.constrainAllUses(TII, TRI, RBI);
7609 return true;
7610 }
7611 MachineInstrBuilder MIB3 =
7612 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
7613 .addDef(ResVReg)
7614 .addUse(GR.getSPIRVTypeID(ResType));
7615 GR.add(ConstVal, MIB3);
7617 cast<Function>(GV));
7618 MIB3.constrainAllUses(TII, TRI, RBI);
7619 return true;
7620 }
7621 assert(NewReg != ResVReg);
7622 return BuildCOPY(ResVReg, NewReg, I);
7623 }
7625 assert(GlobalVar->getName() != "llvm.global.annotations");
7626
7627 // Skip empty declaration for GVs with initializers till we get the decl with
7628 // passed initializer.
7629 if (hasInitializer(GlobalVar) && !Init)
7630 return true;
7631
7632 const std::optional<SPIRV::LinkageType::LinkageType> LnkType =
7633 getSpirvLinkageTypeFor(STI, *GV);
7634
7635 if (LnkType && *LnkType == SPIRV::LinkageType::Import)
7636 Init = nullptr;
7637
7638 const unsigned AddrSpace = GV->getAddressSpace();
7639 SPIRV::StorageClass::StorageClass StorageClass =
7640 addressSpaceToStorageClass(AddrSpace, STI);
7641 SPIRVTypeInst ResType =
7644 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init,
7645 GlobalVar->isConstant(), LnkType, MIRBuilder, true);
7646 // TODO: For AMDGCN, we pipe externally_initialized through via
7647 // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during
7648 // reverse translation. We should remove this once SPIR-V gains the ability to
7649 // express the concept.
7650 if (GlobalVar->isExternallyInitialized() &&
7651 STI.getTargetTriple().getVendor() == Triple::AMD) {
7652 constexpr unsigned ReadWriteINTEL = 3u;
7653 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL,
7654 {ReadWriteINTEL});
7655 MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt());
7656 addStringImm(GV->getName(), MIB);
7657 }
7658 return Reg.isValid();
7659}
7660
7661bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
7662 SPIRVTypeInst ResType,
7663 MachineInstr &I) const {
7664 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
7665 return selectExtInst(ResVReg, ResType, I, CL::log10);
7666 }
7667
7668 // There is no log10 instruction in the GLSL Extended Instruction set, so it
7669 // is implemented as:
7670 // log10(x) = log2(x) * (1 / log2(10))
7671 // = log2(x) * 0.30103
7672
7673 MachineIRBuilder MIRBuilder(I);
7674 MachineBasicBlock &BB = *I.getParent();
7675
7676 // Build log2(x).
7677 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
7678 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
7679 .addDef(VarReg)
7680 .addUse(GR.getSPIRVTypeID(ResType))
7681 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
7682 .addImm(GL::Log2)
7683 .add(I.getOperand(1))
7684 .constrainAllUses(TII, TRI, RBI);
7685
7686 // Build 0.30103.
7687 assert(isVectorType(ResType) || ResType.isAnyTypeFloat());
7688 // TODO: Add matrix implementation once supported by the HLSL frontend.
7689 SPIRVTypeInst SpirvScalarType = GR.getScalarOrVectorComponentType(ResType);
7690 // The literal must match the precision of the scalar type, otherwise the
7691 // OpConstant will contain non-zero high-order bits and fail SPIR-V
7692 // validation when the type is narrower than 32 bits (e.g. half).
7693 APFloat ScaleVal(0.30103);
7694 bool LosesInfo;
7695 ScaleVal.convert(
7696 getZeroFP(GR.getTypeForSPIRVType(SpirvScalarType)).getSemantics(),
7697 APFloat::rmNearestTiesToEven, &LosesInfo);
7698 Register ScaleReg = GR.buildConstantFP(ScaleVal, MIRBuilder, SpirvScalarType);
7699
7700 // Multiply log2(x) by 0.30103 to get log10(x) result.
7701 auto Opcode =
7702 isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS;
7703 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
7704 .addDef(ResVReg)
7705 .addUse(GR.getSPIRVTypeID(ResType))
7706 .addUse(VarReg)
7707 .addUse(ScaleReg)
7708 .constrainAllUses(TII, TRI, RBI);
7709 return true;
7710}
7711
7712bool SPIRVInstructionSelector::selectFpowi(Register ResVReg,
7713 SPIRVTypeInst ResType,
7714 MachineInstr &I) const {
7715 // On OpenCL targets, pown(gentype x, intn n) maps directly.
7716 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std))
7717 return selectExtInst(ResVReg, ResType, I, CL::pown);
7718
7719 // On GLSL (Vulkan) targets, there is no integer-exponent power instruction.
7720 // Lower as: Pow(base, OpConvertSToF(exp)).
7721 if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
7722 Register BaseReg = I.getOperand(1).getReg();
7723 Register ExpReg = I.getOperand(2).getReg();
7724 Register FloatExpReg = MRI->createVirtualRegister(GR.getRegClass(ResType));
7725 if (!selectOpWithSrcs(FloatExpReg, ResType, I, {ExpReg},
7726 SPIRV::OpConvertSToF))
7727 return false;
7728 return selectExtInst(ResVReg, ResType, I, GL::Pow,
7729 /*setMIFlags=*/true, /*useMISrc=*/false,
7730 {BaseReg, FloatExpReg});
7731 }
7732 return false;
7733}
7734
7735bool SPIRVInstructionSelector::selectModf(Register ResVReg,
7736 SPIRVTypeInst ResType,
7737 MachineInstr &I) const {
7738 // llvm.modf has a single arg --the number to be decomposed-- and returns a
7739 // struct { restype, restype }, while OpenCLLIB::modf has two args --the
7740 // number to be decomposed and a pointer--, returns the fractional part and
7741 // the integral part is stored in the pointer argument. Therefore, we can't
7742 // use directly the OpenCLLIB::modf intrinsic. However, we can do some
7743 // scaffolding to make it work. The idea is to create an alloca instruction
7744 // to get a ptr, pass this ptr to OpenCL::modf, and then load the value
7745 // from this ptr to place it in the struct. llvm.modf returns the fractional
7746 // part as the first element of the result, and the integral part as the
7747 // second element of the result.
7748
7749 // At this point, the return type is not a struct anymore, but rather two
7750 // independent elements of SPIRVResType. We can get each independent element
7751 // from I.getDefs() or I.getOperands().
7752 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
7753 MachineIRBuilder MIRBuilder(I);
7754 SPIRVTypeInst FloatType =
7755 GR.getSPIRVTypeForVReg(I.getOperand(I.getNumExplicitDefs()).getReg());
7756 // Get pointer type for alloca variable.
7757 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
7758 FloatType, MIRBuilder, SPIRV::StorageClass::Function);
7759 // Create new register for the pointer type of alloca variable.
7760 Register PtrTyReg =
7761 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
7762 MIRBuilder.getMRI()->setType(
7763 PtrTyReg,
7764 LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
7765 GR.getPointerSize()));
7766
7767 // Assign SPIR-V type of the pointer type of the alloca variable to the
7768 // new register.
7769 GR.assignSPIRVTypeToVReg(PtrType, PtrTyReg, MIRBuilder.getMF());
7771 MachineBasicBlock &EntryBB = I.getMF()->front();
7772 const bool IsUntyped =
7773 PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7774 auto AllocaMIB =
7775 BuildMI(EntryBB, VarPos, I.getDebugLoc(),
7776 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
7777 : SPIRV::OpVariable))
7778 .addDef(PtrTyReg)
7779 .addUse(GR.getSPIRVTypeID(PtrType))
7780 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function));
7781 if (IsUntyped)
7782 AllocaMIB.addUse(GR.getSPIRVTypeID(ResType)); // Data Type
7783 Register Variable = AllocaMIB->getOperand(0).getReg();
7784
7785 MachineBasicBlock &BB = *I.getParent();
7786 // Create the OpenCLLIB::modf instruction.
7787 auto MIB =
7788 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
7789 .addDef(ResVReg)
7790 .addUse(GR.getSPIRVTypeID(FloatType))
7791 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
7792 .addImm(CL::modf)
7793 .setMIFlags(I.getFlags())
7794 .add(I.getOperand(I.getNumExplicitDefs())) // Floating point value.
7795 .addUse(Variable); // Pointer to integral part.
7796 // Assign the integral part stored in the ptr to the second element of the
7797 // result.
7798 Register IntegralPartReg = I.getOperand(1).getReg();
7799 if (IntegralPartReg.isValid() && !MRI->use_nodbg_empty(IntegralPartReg)) {
7800 // Load the value from the pointer to integral part.
7801 auto LoadMIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
7802 .addDef(IntegralPartReg)
7803 .addUse(GR.getSPIRVTypeID(FloatType))
7804 .addUse(Variable);
7805 LoadMIB.constrainAllUses(TII, TRI, RBI);
7806 }
7807
7808 MIB.constrainAllUses(TII, TRI, RBI);
7809 return true;
7810 } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) {
7811 assert(false && "GLSL::Modf is deprecated.");
7812 // FIXME: GL::Modf is deprecated, use Modfstruct instead.
7813 return false;
7814 }
7815 return false;
7816}
7817
7818// Generate the instructions to load 3-element vector builtin input
7819// IDs/Indices.
7820// Like: GlobalInvocationId, LocalInvocationId, etc....
7821
7822bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
7823 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
7824 SPIRVTypeInst ResType, MachineInstr &I) const {
7825 MachineIRBuilder MIRBuilder(I);
7826 const SPIRVTypeInst Vec3Ty =
7827 GR.getOrCreateSPIRVVectorType(ResType, 3, MIRBuilder, false);
7828 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
7829 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
7830
7831 // Create new register for the input ID builtin variable.
7832 Register NewRegister =
7833 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
7834 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64));
7835 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
7836
7837 // Build global variable with the necessary decorations for the input ID
7838 // builtin variable.
7840 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
7841 SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder,
7842 false);
7843
7844 // Create new register for loading value.
7845 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
7846 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass);
7847 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64));
7848 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF());
7849
7850 // Load v3uint value from the global variable.
7851 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
7852 .addDef(LoadedRegister)
7853 .addUse(GR.getSPIRVTypeID(Vec3Ty))
7854 .addUse(Variable);
7855
7856 // Get the input ID index. Expecting operand is a constant immediate value,
7857 // wrapped in a type assignment.
7858 assert(I.getOperand(2).isReg());
7859 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI);
7860
7861 // Extract the input ID from the loaded vector value.
7862 MachineBasicBlock &BB = *I.getParent();
7863 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
7864 .addDef(ResVReg)
7865 .addUse(GR.getSPIRVTypeID(ResType))
7866 .addUse(LoadedRegister)
7867 .addImm(ThreadId);
7868 MIB.constrainAllUses(TII, TRI, RBI);
7869 return true;
7870}
7871
7872// Generate the instructions to load 32-bit integer builtin input IDs/Indices.
7873// Like LocalInvocationIndex
7874bool SPIRVInstructionSelector::loadBuiltinInputID(
7875 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg,
7876 SPIRVTypeInst ResType, MachineInstr &I) const {
7877 MachineIRBuilder MIRBuilder(I);
7878 const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType(
7879 ResType, MIRBuilder, SPIRV::StorageClass::Input);
7880
7881 // Create new register for the input ID builtin variable.
7882 Register NewRegister =
7883 MIRBuilder.getMRI()->createVirtualRegister(GR.getRegClass(PtrType));
7884 MIRBuilder.getMRI()->setType(
7885 NewRegister,
7886 LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Input),
7887 GR.getPointerSize()));
7888 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
7889
7890 // Build global variable with the necessary decorations for the input ID
7891 // builtin variable.
7893 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr,
7894 SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder,
7895 false);
7896
7897 // Load uint value from the global variable.
7898 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
7899 .addDef(ResVReg)
7900 .addUse(GR.getSPIRVTypeID(ResType))
7901 .addUse(Variable);
7902
7903 MIB.constrainAllUses(TII, TRI, RBI);
7904 return true;
7905}
7906
7907SPIRVTypeInst SPIRVInstructionSelector::widenTypeToVec4(SPIRVTypeInst Type,
7908 MachineInstr &I) const {
7909 MachineIRBuilder MIRBuilder(I);
7910 if (!isVectorType(Type))
7911 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder, false);
7912
7914 return Type;
7915
7916 SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(Type);
7917 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder, false);
7918}
7919
7920bool SPIRVInstructionSelector::loadHandleBeforePosition(
7921 Register &HandleReg, SPIRVTypeInst ResType, GIntrinsic &HandleDef,
7922 MachineInstr &Pos) const {
7923
7924 assert(HandleDef.getIntrinsicID() ==
7925 Intrinsic::spv_resource_handlefrombinding);
7926 uint32_t Set = foldImm(HandleDef.getOperand(2), MRI);
7927 uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI);
7928 uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI);
7929 Register IndexReg = HandleDef.getOperand(5).getReg();
7930 std::string Name =
7931 getStringValueFromReg(HandleDef.getOperand(6).getReg(), *MRI);
7932
7933 bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer;
7934 MachineIRBuilder MIRBuilder(HandleDef);
7935 SPIRVTypeInst VarType = ResType;
7936 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
7937
7938 if (IsStructuredBuffer) {
7939 VarType = GR.getPointeeType(ResType);
7940 SC = GR.getPointerStorageClass(ResType);
7941 }
7942
7943 // ArraySize 0 means an unbounded array and we need to set to required
7944 // capability.
7945 if (ArraySize == 0)
7946 MIRBuilder.buildInstr(SPIRV::OpCapability)
7947 .addImm(SPIRV::Capability::RuntimeDescriptorArrayEXT);
7948
7949 Register VarReg =
7950 buildPointerToResource(SPIRVTypeInst(VarType), SC, Set, Binding,
7951 ArraySize, IndexReg, Name, MIRBuilder);
7952
7953 // The handle for the buffer is the pointer to the resource. For an image, the
7954 // handle is the image object. So images get an extra load.
7955 uint32_t LoadOpcode =
7956 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
7957 GR.assignSPIRVTypeToVReg(ResType, HandleReg, *Pos.getMF());
7958 BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(), TII.get(LoadOpcode))
7959 .addDef(HandleReg)
7960 .addUse(GR.getSPIRVTypeID(ResType))
7961 .addUse(VarReg)
7962 .constrainAllUses(TII, TRI, RBI);
7963 return true;
7964}
7965
7966bool SPIRVInstructionSelector::errorIfInstrOutsideShader(
7967 MachineInstr &I) const {
7968 if (!STI.isShader())
7969 return diagnoseUnsupported(
7970 I, "this instruction is only supported in shaders.");
7971 return true;
7972}
7973
7974namespace llvm {
7975InstructionSelector *
7978 const RegisterBankInfo &RBI) {
7979 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
7980}
7981} // namespace llvm
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
const AArch64Subtarget & Subtarget
SDValue Pred
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo MachineBasicBlock::iterator InsertPt
const AArch64InstrInfo & TII
uint64_t uint64_t
uint64_t Imm
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static uint8_t SwapBits(uint8_t Val)
basic Basic Alias true
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
DXIL Resource Implicit Binding
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
static MaybeAlign getAlign(Value *Ptr)
IRTranslator LLVM IR MI
LLVMTypeRef LLVMIntType(unsigned NumBits)
Definition Core.cpp:750
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
Loop::LoopBounds::Direction Direction
Definition LoopInfo.cpp:249
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
Type::TypeID TypeID
#define T
#define T1
uint64_t IntrinsicInst * II
if(PassOpts->AAPipeline)
static StringRef getName(Value *V)
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
MachineRegisterInfo & MRI
static bool isConcreteSPIRVType(SPIRVTypeInst Ty, const SPIRVGlobalRegistry &GR)
static APFloat getOneFP(const Type *LLVMFloatTy)
static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC)
static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg)
static bool mayApplyGenericSelection(unsigned Opcode)
static APFloat getZeroFP(const Type *LLVMFloatTy)
std::vector< std::pair< SPIRV::InstructionSet::InstructionSet, uint32_t > > ExtInstList
static bool intrinsicHasSideEffects(Intrinsic::ID ID)
static unsigned getBoolCmpOpcode(unsigned PredNum)
static unsigned getICmpOpcode(unsigned PredNum)
static bool isOpcodeWithNoSideEffects(unsigned Opcode)
static bool containsStorageBufferPointer(SPIRVTypeInst Ty, const SPIRVGlobalRegistry &GR, SmallSet< Register, 8 > &Visited)
static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef)
static unsigned getPtrCmpOpcode(unsigned Pred)
static void addMemoryOperands(const MachineMemOperand *MemOp, MachineInstrBuilder &MIB, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry &GR, std::optional< Align > AlignOverride=std::nullopt)
bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType, LegalizerHelper &Helper, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR)
const char * Msg
#define SPIRV_BACKEND_SERVICE_FUN_NAME
Definition SPIRVUtils.h:558
This file contains some templates that are useful if you are working with the STL at all.
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
This file defines the SmallSet class.
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static ManagedStatic< cl::opt< FnT >, OptCreatorT > CallbackFunction
BinaryOperator * Mul
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static const fltSemantics & BFloat()
Definition APFloat.h:303
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
const fltSemantics & getSemantics() const
Definition APFloat.h:1591
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition APFloat.h:1192
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1183
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
unsigned size() const
Definition DenseMap.h:717
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
static LLVM_ABI std::optional< GFConstant > getConstant(Register Const, const MachineRegisterInfo &MRI)
Definition Utils.cpp:2205
static LLVM_ABI std::optional< GIConstant > getConstant(Register Const, const MachineRegisterInfo &MRI)
Definition Utils.cpp:2165
Represents a call to an intrinsic.
Intrinsic::ID getIntrinsicID() const
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
@ InternalLinkage
Rename collisions when linking (static functions).
Definition GlobalValue.h:60
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
static constexpr LLT float64()
Get a 64-bit IEEE double value.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
LLVM_ABI void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo)
Replace all occurrences of FromReg with ToReg:SubIdx, properly composing subreg indices where necessa...
LLVM_ABI void emitGenericError(const Twine &ErrMsg) const
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
@ MOVolatile
The memory access is volatile.
@ MONonTemporal
The memory access is non-temporal.
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
defusechain_instr_iterator< true, false, false, true > use_instr_iterator
use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the specified register,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
use_instr_iterator use_instr_begin(Register RegNo) const
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
static def_instr_iterator def_instr_end()
defusechain_instr_iterator< false, true, false, true > def_instr_iterator
def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the specified register,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
def_instr_iterator def_instr_begin(Register RegNo) const
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
static use_instr_iterator use_instr_end()
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
const MachineFunction & getMF() const
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Analysis providing profile information.
Holds all the information related to register banks.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
const SDValue & getOperand(unsigned i) const
bool isScalarOrVectorSigned(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateOpTypeSampledImage(SPIRVTypeInst ImageType, MachineIRBuilder &MIRBuilder)
void assignSPIRVTypeToVReg(SPIRVTypeInst Type, Register VReg, const MachineFunction &MF)
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
const TargetRegisterClass * getRegClass(SPIRVTypeInst SpvType) const
MachineInstr * getOrAddMemAliasingINTELInst(MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD)
bool isAggregateType(SPIRVTypeInst Type) const
unsigned getScalarOrVectorBitWidth(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
Register buildGlobalVariable(Register Reg, SPIRVTypeInst BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
unsigned getScalarOrVectorComponentCount(Register VReg) const
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
bool isBitcastCompatible(SPIRVTypeInst Type1, SPIRVTypeInst Type2) const
Register getOrCreateConstFP(APFloat Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
LLT getRegType(SPIRVTypeInst SpvType) const
SPIRVTypeInst getOpTypeVoid(MachineIRBuilder &MIRBuilder)
void invalidateMachineInstr(MachineInstr *MI)
SPIRVTypeInst getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder, bool EmitIR)
bool isScalarOfType(Register VReg, unsigned TypeOpcode) const
Register getSPIRVTypeID(SPIRVTypeInst SpirvType) const
Register getOrCreateConstInt(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
bool findValueAttrs(const MachineInstr *Key, Type *&Ty, StringRef &Name)
SPIRVTypeInst retrieveScalarOrVectorIntType(SPIRVTypeInst Type) const
Register getOrCreateGlobalVariableWithBinding(SPIRVTypeInst VarType, uint32_t Set, uint32_t Binding, StringRef Name, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst changePointerStorageClass(SPIRVTypeInst PtrType, SPIRV::StorageClass::StorageClass SC, MachineInstr &I)
Register getOrCreateConstVector(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Register buildConstantFP(APFloat Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType=nullptr)
void addGlobalObject(const Value *V, const MachineFunction *MF, Register R)
SPIRVTypeInst getScalarOrVectorComponentType(SPIRVTypeInst Type) const
void recordFunctionPointer(const MachineOperand *MO, const Function *F)
SPIRVTypeInst getOrCreateSPIRVFloatType(unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII)
SPIRVTypeInst getPointeeType(SPIRVTypeInst PtrType)
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
MachineFunction * setCurrentFunc(MachineFunction &MF)
Register getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType)
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
Type * getDeducedGlobalValueType(const GlobalValue *Global)
Register getOrCreateUndef(MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const
SPIRVTypeInst getUntypedPtrElementType(Register Reg) const
bool erase(const MachineInstr *MI)
bool add(SPIRV::IRHandle Handle, const MachineInstr *MI)
Register find(SPIRV::IRHandle Handle, const MachineFunction *MF)
bool isPhysicalSPIRV() const
bool isAtLeastSPIRVVer(VersionTuple VerToCompareTo) const
bool canUseExtInstSet(SPIRV::InstructionSet::InstructionSet E) const
bool isLogicalSPIRV() const
bool canUseExtension(SPIRV::Extension::Extension E) const
bool isTypeIntOrFloat() const
bool isAnyTypeFloat() const
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition SmallSet.h:134
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:462
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
@ HalfTyID
16-bit floating point type
Definition Type.h:57
@ FloatTyID
32-bit floating point type
Definition Type.h:59
@ BFloatTyID
16-bit floating point type (7-bit significand)
Definition Type.h:58
@ DoubleTyID
64-bit floating point type
Definition Type.h:60
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:271
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition Type.h:314
TypeID getTypeID() const
Return the type id for the type.
Definition Type.h:138
Value * getOperand(unsigned i) const
Definition User.h:207
bool hasName() const
Definition Value.h:263
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char IsConst[]
Key for Kernel::Arg::Metadata::mIsConst.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
ElementType
The element type of an SRV or UAV resource.
Definition DXILABI.h:68
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
NodeAddr< InstrNode * > Instr
Definition RDFGraph.h:389
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
void addStringImm(StringRef Str, MCInst &Inst)
MachineBasicBlock::iterator getOpVariableMBBIt(MachineFunction &MF)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isTypeFoldingSupported(unsigned Opcode)
uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem, uint32_t StorageClassSem)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
MachineInstr * getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI)
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition Utils.cpp:1837
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition Utils.cpp:163
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
unsigned getArrayComponentCount(const MachineRegisterInfo *MRI, const MachineInstr *ResType)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, ArrayRef< uint32_t > DecArgs, StringRef StrImm)
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition Utils.cpp:1699
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx, SyncScope::ID Id)
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.h:238
bool isVectorType(SPIRVTypeInst SPVTy)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
void buildOpName(Register Target, StringRef Name, MachineIRBuilder &MIRBuilder)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
Type * toTypedPointer(Type *Ty)
Definition SPIRVUtils.h:469
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.h:222
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
MachineInstr * passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI)
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
std::optional< SPIRV::LinkageType::LinkageType > getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV)
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)
AtomicOrdering
Atomic ordering for LLVM's memory model.
InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &Subtarget, const RegisterBankInfo &RBI)
std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI)
int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
bool hasInitializer(const GlobalVariable *GV)
Definition SPIRVUtils.h:354
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)
LLVM_ABI bool isTriviallyDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Check whether an instruction MI is dead: it only defines dead virtual registers, and doesn't have oth...
Definition Utils.cpp:228
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
#define N
bool isVolatile() const