LLVM 24.0.0git
SPIRVISelLowering.cpp
Go to the documentation of this file.
1//===- SPIRVISelLowering.cpp - SPIR-V DAG Lowering Impl ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SPIRVTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVISelLowering.h"
14#include "SPIRV.h"
15#include "SPIRVInstrInfo.h"
17#include "SPIRVRegisterInfo.h"
18#include "SPIRVSubtarget.h"
23#include "llvm/IR/IntrinsicsSPIRV.h"
24
25#define DEBUG_TYPE "spirv-lower"
26
27using namespace llvm;
28
30 const SPIRVSubtarget &ST)
31 : TargetLowering(TM, ST), STI(ST) {
32 // Even with SPV_ALTERA_arbitrary_precision_integers enabled, atomic sizes are
33 // limited by atomicrmw xchg operation, which only supports operand up to 64
34 // bits wide, as defined in SPIR-V legalizer. Currently, spirv-val doesn't
35 // consider 128-bit OpTypeInt as valid either.
38}
39
40// Returns true of the types logically match, as defined in
41// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpCopyLogical.
42static bool typesLogicallyMatch(const SPIRVTypeInst Ty1,
43 const SPIRVTypeInst Ty2,
45 if (Ty1->getOpcode() != Ty2->getOpcode())
46 return false;
47
48 if (Ty1->getNumOperands() != Ty2->getNumOperands())
49 return false;
50
51 if (Ty1->getOpcode() == SPIRV::OpTypeArray) {
52 // Array must have the same size.
53 if (Ty1->getOperand(2).getReg() != Ty2->getOperand(2).getReg())
54 return false;
55
56 SPIRVTypeInst ElemType1 =
58 SPIRVTypeInst ElemType2 =
60 return ElemType1 == ElemType2 ||
61 typesLogicallyMatch(ElemType1, ElemType2, GR);
62 }
63
64 if (Ty1->getOpcode() == SPIRV::OpTypeStruct) {
65 for (unsigned I = 1; I < Ty1->getNumOperands(); I++) {
66 SPIRVTypeInst ElemType1 =
68 SPIRVTypeInst ElemType2 =
70 if (ElemType1 != ElemType2 &&
71 !typesLogicallyMatch(ElemType1, ElemType2, GR))
72 return false;
73 }
74 return true;
75 }
76 return false;
77}
78
80 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
81 // This code avoids CallLowering fail inside getVectorTypeBreakdown
82 // on v3i1 arguments. Maybe we need to return 1 for all types.
83 // TODO: remove it once this case is supported by the default implementation.
84 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
85 (VT.getVectorElementType() == MVT::i1 ||
86 VT.getVectorElementType() == MVT::i8))
87 return 1;
88 if (!VT.isVector() && VT.isInteger() && VT.getSizeInBits() <= 64)
89 return 1;
90 return getNumRegisters(Context, VT);
91}
92
95 EVT VT) const {
96 // This code avoids CallLowering fail inside getVectorTypeBreakdown
97 // on v3i1 arguments. Maybe we need to return i32 for all types.
98 // TODO: remove it once this case is supported by the default implementation.
99 if (VT.isVector()) {
100 if (VT.getVectorNumElements() == 3) {
101 if (VT.getVectorElementType() == MVT::i1)
102 return MVT::v4i1;
103 else if (VT.getVectorElementType() == MVT::i8)
104 return MVT::v4i8;
105 } else if (!isPowerOf2_32(VT.getVectorNumElements()) &&
106 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)) {
107 // Non POT element counts are not yet supported by GISEL.
108 return MVT::getVectorVT(
111 }
112 }
113 return getRegisterType(Context, VT);
114}
115
118 MachineFunction &MF, unsigned Intrinsic) const {
119 IntrinsicInfo Info;
120
121 unsigned AlignIdx = 0;
122 unsigned OrderingIdx = 0;
123 unsigned FlagsIdx;
124
125 switch (Intrinsic) {
126 case Intrinsic::spv_load:
127 FlagsIdx = 1;
128 AlignIdx = 2;
129 break;
130 case Intrinsic::spv_store:
131 FlagsIdx = 2;
132 AlignIdx = 3;
133 break;
134 case Intrinsic::spv_atomic_load:
135 FlagsIdx = 1;
136 OrderingIdx = 2;
137 break;
138 case Intrinsic::spv_atomic_store:
139 FlagsIdx = 2;
140 OrderingIdx = 3;
141 break;
142 default:
143 return;
144 }
145
146 Info.flags = static_cast<MachineMemOperand::Flags>(
147 cast<ConstantInt>(I.getOperand(FlagsIdx))->getZExtValue());
148 Info.memVT = MVT::i64;
149 // TODO: take into account opaque pointers (don't use getElementType).
150 // MVT::getVT(PtrTy->getElementType());
151
152 if (AlignIdx) {
153 auto *AlignOp = cast<ConstantInt>(I.getOperand(AlignIdx));
154 Info.align = Align(AlignOp->getZExtValue());
155 }
156
157 if (OrderingIdx) {
158 Info.order = static_cast<AtomicOrdering>(
159 cast<ConstantInt>(I.getOperand(OrderingIdx))->getZExtValue());
160 }
161 Infos.push_back(Info);
162}
163
166 // SPIR-V represents inline assembly via OpAsmINTEL where constraints are
167 // passed through as literals defined by client API. Return C_RegisterClass
168 // for non-memory constraints since SPIR-V does not distinguish between
169 // register, immediate, or memory operands at this level. We do have to return
170 // C_Memory for memory constraints as otherwise IRTranslator gets confused
171 // trying to allocate registers for them.
172 if (Constraint == "m")
173 return C_Memory;
174 return C_RegisterClass;
175}
176
177std::pair<unsigned, const TargetRegisterClass *>
179 StringRef Constraint,
180 MVT VT) const {
181 const TargetRegisterClass *RC = nullptr;
182 if (Constraint.starts_with("{"))
183 return std::make_pair(0u, RC);
184
185 if (VT.isFloatingPoint())
186 RC = VT.isVector() ? &SPIRV::vfIDRegClass : &SPIRV::fIDRegClass;
187 else if (VT.isInteger())
188 RC = VT.isVector() ? &SPIRV::viIDRegClass : &SPIRV::iIDRegClass;
189 else
190 RC = &SPIRV::iIDRegClass;
191
192 return std::make_pair(0u, RC);
193}
194
196 const MachineInstr *Inst = MRI->getVRegDef(OpReg);
197 return Inst && Inst->getOpcode() == SPIRV::OpFunctionParameter
198 ? Inst->getOperand(1).getReg()
199 : OpReg;
200}
201
204 Register OpReg, unsigned OpIdx,
205 SPIRVTypeInst NewPtrType) {
206 MachineIRBuilder MIB(I);
207 Register NewReg = createVirtualRegister(NewPtrType, &GR, MRI, MIB.getMF());
208 MIB.buildInstr(SPIRV::OpBitcast)
209 .addDef(NewReg)
210 .addUse(GR.getSPIRVTypeID(NewPtrType))
211 .addUse(OpReg)
213 *STI.getRegBankInfo());
214 I.getOperand(OpIdx).setReg(NewReg);
215}
216
218 SPIRVTypeInst OpType, bool ReuseType,
219 SPIRVTypeInst ResType,
220 const Type *ResTy) {
221 SPIRV::StorageClass::StorageClass SC =
222 static_cast<SPIRV::StorageClass::StorageClass>(
223 OpType->getOperand(1).getImm());
224 MachineIRBuilder MIB(I);
225 SPIRVTypeInst NewBaseType =
226 ReuseType ? ResType
228 ResTy, MIB, SPIRV::AccessQualifier::ReadWrite, false);
229 return GR.getOrCreateSPIRVPointerType(NewBaseType, MIB, SC);
230}
231
232// Insert a bitcast before the instruction to keep SPIR-V code valid
233// when there is a type mismatch between results and operand types.
234static void validatePtrTypes(const SPIRVSubtarget &STI,
236 MachineInstr &I, unsigned OpIdx,
237 SPIRVTypeInst ResType,
238 const Type *ResTy = nullptr) {
239 // Get operand type
240 MachineFunction *MF = I.getParent()->getParent();
241 Register OpReg = I.getOperand(OpIdx).getReg();
242 Register OpTypeReg = getTypeReg(MRI, OpReg);
243 const MachineInstr *OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);
244 if (!ResType || !OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
245 return;
246 // Get operand's pointee type
247 Register ElemTypeReg = OpType->getOperand(2).getReg();
248 SPIRVTypeInst ElemType = GR.getSPIRVTypeForVReg(ElemTypeReg, MF);
249 if (!ElemType)
250 return;
251 // Check if we need a bitcast to make a statement valid
252 bool IsSameMF = MF == ResType->getParent()->getParent();
253 bool IsEqualTypes = IsSameMF ? ElemType == ResType
254 : GR.getTypeForSPIRVType(ElemType) == ResTy;
255 if (IsEqualTypes)
256 return;
257 // There is a type mismatch between results and operand types
258 // and we insert a bitcast before the instruction to keep SPIR-V code valid
259 SPIRVTypeInst NewPtrType =
260 createNewPtrType(GR, I, OpType, IsSameMF, ResType, ResTy);
261 if (!GR.isBitcastCompatible(NewPtrType, OpType))
263 "insert validation bitcast: incompatible result and operand types");
264 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
265}
266
267// Insert a bitcast before OpGroupWaitEvents if the last argument is a pointer
268// that doesn't point to OpTypeEvent.
272 MachineInstr &I) {
273 constexpr unsigned OpIdx = 2;
274 MachineFunction *MF = I.getParent()->getParent();
275 Register OpReg = I.getOperand(OpIdx).getReg();
276 Register OpTypeReg = getTypeReg(MRI, OpReg);
277 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);
278 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
279 return;
280 SPIRVTypeInst ElemType =
281 GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg());
282 if (!ElemType || ElemType->getOpcode() == SPIRV::OpTypeEvent)
283 return;
284 // Insert a bitcast before the instruction to keep SPIR-V code valid.
285 LLVMContext &Context = MF->getFunction().getContext();
286 SPIRVTypeInst NewPtrType =
287 createNewPtrType(GR, I, OpType, false, nullptr,
288 TargetExtType::get(Context, "spirv.Event"));
289 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
290}
291
292// Per the SPIR-V spec, the Size operand of OpLifetimeStart/OpLifetimeStop must
293// be 0 if Pointer points to a non-void type.
296 Register PtrReg = I.getOperand(0).getReg();
297 MachineFunction *MF = I.getParent()->getParent();
298 Register PtrTypeReg = getTypeReg(MRI, PtrReg);
299 SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrTypeReg, MF);
300 SPIRVTypeInst PointeeElemType =
301 PtrType ? GR.getPointeeType(PtrType) : nullptr;
302 if (!PointeeElemType || PointeeElemType->getOpcode() == SPIRV::OpTypeVoid ||
303 (PointeeElemType->getOpcode() == SPIRV::OpTypeInt &&
304 PointeeElemType->getOperand(1).getImm() == 8))
305 return;
306 I.getOperand(1).setImm(0);
307}
308
312 MachineInstr &I, unsigned OpIdx) {
313 MachineFunction *MF = I.getParent()->getParent();
314 Register OpReg = I.getOperand(OpIdx).getReg();
315 Register OpTypeReg = getTypeReg(MRI, OpReg);
316 SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);
317 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
318 return;
319 SPIRVTypeInst ElemType =
320 GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg());
321 if (!ElemType || ElemType->getOpcode() != SPIRV::OpTypeStruct ||
322 ElemType->getNumOperands() != 2)
323 return;
324 // It's a structure-wrapper around another type with a single member field.
325 SPIRVTypeInst MemberType =
326 GR.getSPIRVTypeForVReg(ElemType->getOperand(1).getReg());
327 if (!MemberType)
328 return;
329 unsigned MemberTypeOp = MemberType->getOpcode();
330 if (!isVectorType(MemberType) && MemberTypeOp != SPIRV::OpTypeInt &&
331 MemberTypeOp != SPIRV::OpTypeFloat && MemberTypeOp != SPIRV::OpTypeBool)
332 return;
333 // It's a structure-wrapper around a valid type. Insert a bitcast before the
334 // instruction to keep SPIR-V code valid.
335 SPIRV::StorageClass::StorageClass SC =
336 static_cast<SPIRV::StorageClass::StorageClass>(
337 OpType->getOperand(1).getImm());
338 MachineIRBuilder MIB(I);
339 SPIRVTypeInst NewPtrType =
340 GR.getOrCreateSPIRVPointerType(MemberType, MIB, SC);
341 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
342}
343
344// Insert a bitcast before the function call instruction to keep SPIR-V code
345// valid when there is a type mismatch between actual and expected types of an
346// argument:
347// %formal = OpFunctionParameter %formal_type
348// ...
349// %res = OpFunctionCall %ty %fun %actual ...
350// implies that %actual is of %formal_type, and in case of opaque pointers.
351// We may need to insert a bitcast to ensure this.
353 MachineRegisterInfo *DefMRI,
354 MachineRegisterInfo *CallMRI,
355 SPIRVGlobalRegistry &GR, MachineInstr &FunCall,
356 MachineInstr *FunDef) {
357 if (FunDef->getOpcode() != SPIRV::OpFunction)
358 return;
359 unsigned OpIdx = 3;
360 for (FunDef = FunDef->getNextNode();
361 FunDef && FunDef->getOpcode() == SPIRV::OpFunctionParameter &&
362 OpIdx < FunCall.getNumOperands();
363 FunDef = FunDef->getNextNode(), OpIdx++) {
364 SPIRVTypeInst DefPtrType =
365 DefMRI->getVRegDef(FunDef->getOperand(1).getReg());
366 SPIRVTypeInst DefElemType =
367 DefPtrType && DefPtrType->getOpcode() == SPIRV::OpTypePointer
368 ? GR.getSPIRVTypeForVReg(DefPtrType->getOperand(2).getReg(),
369 DefPtrType->getParent()->getParent())
370 : nullptr;
371 if (DefElemType) {
372 const Type *DefElemTy = GR.getTypeForSPIRVType(DefElemType);
373 // validatePtrTypes() works in the context if the call site
374 // When we process historical records about forward calls
375 // we need to switch context to the (forward) call site and
376 // then restore it back to the current machine function.
377 MachineFunction *CurMF =
378 GR.setCurrentFunc(*FunCall.getParent()->getParent());
379 validatePtrTypes(STI, CallMRI, GR, FunCall, OpIdx, DefElemType,
380 DefElemTy);
381 GR.setCurrentFunc(*CurMF);
382 }
383 }
384}
385
386// Ensure there is no mismatch between actual and expected arg types: calls
387// with a processed definition. Return Function pointer if it's a forward
388// call (ahead of definition), and nullptr otherwise.
390 MachineRegisterInfo *CallMRI,
392 MachineInstr &FunCall) {
393 const GlobalValue *GV = FunCall.getOperand(2).getGlobal();
394 const Function *F = dyn_cast<Function>(GV);
395 MachineInstr *FunDef =
396 const_cast<MachineInstr *>(GR.getFunctionDefinition(F));
397 if (!FunDef)
398 return F;
399 MachineRegisterInfo *DefMRI = &FunDef->getParent()->getParent()->getRegInfo();
400 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall, FunDef);
401 return nullptr;
402}
403
404// Ensure there is no mismatch between actual and expected arg types: calls
405// ahead of a processed definition.
408 MachineInstr &FunDef) {
409 const Function *F = GR.getFunctionByDefinition(&FunDef);
411 for (MachineInstr *FunCall : *FwdCalls) {
412 MachineRegisterInfo *CallMRI =
413 &FunCall->getParent()->getParent()->getRegInfo();
414 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, *FunCall, &FunDef);
415 }
416}
417
418// Validation of an access chain.
421 SPIRVTypeInst BaseTypeInst = GR.getSPIRVTypeForVReg(I.getOperand(0).getReg());
422 if (BaseTypeInst && BaseTypeInst->getOpcode() == SPIRV::OpTypePointer) {
423 SPIRVTypeInst BaseElemType =
424 GR.getSPIRVTypeForVReg(BaseTypeInst->getOperand(2).getReg());
425 validatePtrTypes(STI, MRI, GR, I, 2, BaseElemType);
426 }
427}
428
431 // IRTranslator does not believe that rank-1 vectors exist, unlike upstream
432 // LLVM which happily creates <1 x T> vectors. This leads to operations over
433 // <1 x T> vectors getting translated as their scalar counterparts, which is
434 // wrong if we used SPV_EXT_long_vector to preserve the actual vector-ness.
435 switch (MI.getOpcode()) {
436 case SPIRV::OpBitwiseAndS:
437 case SPIRV::OpBitwiseOrS:
438 case SPIRV::OpBitwiseXorS:
439 case SPIRV::OpFAddS:
440 case SPIRV::OpFDivS:
441 case SPIRV::OpFMulS:
442 case SPIRV::OpFNegate:
443 case SPIRV::OpFRemS:
444 case SPIRV::OpFSubS:
445 case SPIRV::OpIAddCarryS:
446 case SPIRV::OpIAddS:
447 case SPIRV::OpIMulS:
448 case SPIRV::OpISubBorrowS:
449 case SPIRV::OpISubS:
450 case SPIRV::OpSDivS:
451 case SPIRV::OpSRemS:
452 case SPIRV::OpShiftLeftLogicalS:
453 case SPIRV::OpShiftRightArithmeticS:
454 case SPIRV::OpShiftRightLogicalS:
455 case SPIRV::OpStrictFAddS:
456 case SPIRV::OpStrictFDivS:
457 case SPIRV::OpStrictFMulS:
458 case SPIRV::OpStrictFRemS:
459 case SPIRV::OpStrictFSubS:
460 case SPIRV::OpUDivS:
461 case SPIRV::OpUModS: {
462 SPIRVTypeInst ResTy = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());
463
464 if (!isVectorType(ResTy))
465 return;
466
467 // Restore original Vec1 type.
468 Register NewResultReg = createVirtualRegister(ResTy, &GR, MRI, *MI.getMF());
469 MRI->replaceRegWith(MI.getOperand(0).getReg(), NewResultReg);
470 // Vector opcodes are always next after scalar (if this ceases to hold we
471 // will have to adapt).
472 MI.setDesc(STI.getInstrInfo()->get(MI.getOpcode() + 1));
473 // IRTranslator would've inserted COPYs from the vector into a scalar, which
474 // are spurious and have to be walked through.
475 for (unsigned I = 2; I != MI.getNumOperands(); ++I) {
476 MachineOperand &Op = MI.getOperand(I);
477 if (!Op.isReg())
478 continue;
479
481 if (OpTy == ResTy)
482 continue;
483
484 MachineInstr *OpDef = getDef(Op, MRI);
485 assert(OpDef &&
486 GR.getSPIRVTypeForVReg(OpDef->getOperand(0).getReg()) == ResTy &&
487 "Expected to find Result Type (Vec1)!");
488 MI.substituteRegister(Op.getReg(), OpDef->getOperand(0).getReg(), 0,
489 *STI.getRegisterInfo());
490 }
491 break;
492 }
493 case TargetOpcode::COPY: {
494 Register ResVReg = MI.getOperand(0).getReg();
495 SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());
496 SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(ResVReg);
497
498 if (!SrcTy || !DstTy || isVectorType(DstTy) || !isVectorType(SrcTy))
499 return;
500
501 Register ExtractReg = createVirtualRegister(DstTy, &GR, MRI, *MI.getMF());
502 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
503 STI.getInstrInfo()->get(SPIRV::OpCompositeExtract))
504 .addDef(ExtractReg)
505 .addUse(GR.getSPIRVTypeID(DstTy))
506 .addUse(MI.getOperand(1).getReg())
507 .addImm(0);
508 for (auto &&U : MRI->use_instructions(ResVReg))
509 U.substituteRegister(ResVReg, ExtractReg, 0, *STI.getRegisterInfo());
510 break;
511 }
512 default:
513 break;
514 }
515}
516
517// TODO: the logic of inserting additional bitcast's is to be moved
518// to pre-IRTranslation passes eventually
521 return;
522
524 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
525 GR.setCurrentFunc(MF);
526 for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
528 for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end();
529 MBBI != MBBE;) {
530 MachineInstr &MI = *MBBI++;
531 validateVec1Ops(STI, MRI, GR, MI);
532 switch (MI.getOpcode()) {
533 case SPIRV::OpAtomicLoad:
534 case SPIRV::OpAtomicExchange:
535 case SPIRV::OpAtomicCompareExchange:
536 case SPIRV::OpAtomicCompareExchangeWeak:
537 case SPIRV::OpAtomicIIncrement:
538 case SPIRV::OpAtomicIDecrement:
539 case SPIRV::OpAtomicIAdd:
540 case SPIRV::OpAtomicISub:
541 case SPIRV::OpAtomicSMin:
542 case SPIRV::OpAtomicUMin:
543 case SPIRV::OpAtomicSMax:
544 case SPIRV::OpAtomicUMax:
545 case SPIRV::OpAtomicAnd:
546 case SPIRV::OpAtomicOr:
547 case SPIRV::OpAtomicXor:
548 // for the above listed instructions
549 // OpAtomicXXX <ResType>, ptr %Op, ...
550 // implies that %Op is a pointer to <ResType>
551 case SPIRV::OpLoad:
552 // OpLoad <ResType>, ptr %Op implies that %Op is a pointer to <ResType>
554 break;
555
556 validatePtrTypes(STI, MRI, GR, MI, 2,
557 GR.getSPIRVTypeForVReg(MI.getOperand(0).getReg()));
558 break;
559 case SPIRV::OpAtomicStore:
560 // OpAtomicStore ptr %Op, <Scope>, <Mem>, <Obj>
561 // implies that %Op points to the <Obj>'s type
562 validatePtrTypes(STI, MRI, GR, MI, 0,
563 GR.getSPIRVTypeForVReg(MI.getOperand(3).getReg()));
564 break;
565 case SPIRV::OpStore:
566 // OpStore ptr %Op, <Obj> implies that %Op points to the <Obj>'s type
567 validatePtrTypes(STI, MRI, GR, MI, 0,
568 GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg()));
569 break;
570 case SPIRV::OpPtrCastToGeneric:
571 case SPIRV::OpGenericCastToPtr:
572 case SPIRV::OpGenericCastToPtrExplicit:
573 validateAccessChain(STI, MRI, GR, MI);
574 break;
575 case SPIRV::OpPtrAccessChain:
576 case SPIRV::OpInBoundsPtrAccessChain:
577 if (MI.getNumOperands() == 4)
578 validateAccessChain(STI, MRI, GR, MI);
579 break;
580
581 case SPIRV::OpFunctionCall:
582 // ensure there is no mismatch between actual and expected arg types:
583 // calls with a processed definition
584 if (MI.getNumOperands() > 3)
585 if (const Function *F = validateFunCall(STI, MRI, GR, MI))
586 GR.addForwardCall(F, &MI);
587 break;
588 case SPIRV::OpFunction:
589 // ensure there is no mismatch between actual and expected arg types:
590 // calls ahead of a processed definition
591 validateForwardCalls(STI, MRI, GR, MI);
592 break;
593
594 // ensure that LLVM IR add/sub instructions result in logical SPIR-V
595 // instructions when applied to bool type
596 case SPIRV::OpIAddS:
597 case SPIRV::OpIAddV:
598 case SPIRV::OpISubS:
599 case SPIRV::OpISubV:
600 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
601 SPIRV::OpTypeBool))
602 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
603 break;
604 // multiplication of bool operands is equivalent to a logical AND
605 case SPIRV::OpIMulS:
606 case SPIRV::OpIMulV:
607 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
608 SPIRV::OpTypeBool))
609 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd));
610 break;
611
612 // ensure that LLVM IR bitwise instructions result in logical SPIR-V
613 // instructions when applied to bool type
614 case SPIRV::OpBitwiseOrS:
615 case SPIRV::OpBitwiseOrV:
616 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
617 SPIRV::OpTypeBool))
618 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalOr));
619 break;
620 case SPIRV::OpBitwiseAndS:
621 case SPIRV::OpBitwiseAndV:
622 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
623 SPIRV::OpTypeBool))
624 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd));
625 break;
626 case SPIRV::OpBitwiseXorS:
627 case SPIRV::OpBitwiseXorV:
628 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
629 SPIRV::OpTypeBool))
630 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
631 break;
632 case SPIRV::OpLifetimeStart:
633 case SPIRV::OpLifetimeStop:
634 if (MI.getOperand(1).getImm() > 0)
636 break;
637 case SPIRV::OpGroupAsyncCopy:
638 validatePtrUnwrapStructField(STI, MRI, GR, MI, 3);
639 validatePtrUnwrapStructField(STI, MRI, GR, MI, 4);
640 break;
641 case SPIRV::OpGroupWaitEvents:
642 // OpGroupWaitEvents ..., ..., <pointer to OpTypeEvent>
644 break;
645 case SPIRV::OpConstantI: {
646 SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());
647 if (Type->getOpcode() != SPIRV::OpTypeInt && MI.getOperand(2).isImm() &&
648 MI.getOperand(2).getImm() == 0) {
649 // Validate the null constant of a target extension type
650 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull));
651 for (unsigned i = MI.getNumOperands() - 1; i > 1; --i)
652 MI.removeOperand(i);
653 }
654 } break;
655 case SPIRV::OpExtInst: {
656 // prefetch
657 if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isImm() ||
658 MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std)
659 continue;
660 switch (MI.getOperand(3).getImm()) {
661 case SPIRV::OpenCLExtInst::frexp:
662 case SPIRV::OpenCLExtInst::lgamma_r:
663 case SPIRV::OpenCLExtInst::remquo: {
664 // The last operand must be of a pointer to i32 or vector of i32
665 // values.
666 MachineIRBuilder MIB(MI);
667 SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(32, MIB);
668 SPIRVTypeInst RetType = MRI->getVRegDef(MI.getOperand(1).getReg());
669 assert(RetType && "Expected return type");
671 STI, MRI, GR, MI, MI.getNumOperands() - 1,
672 (!isVectorType(RetType))
673 ? Int32Type
675 Int32Type, GR.getScalarOrVectorComponentCount(RetType),
676 MIB, false));
677 } break;
678 case SPIRV::OpenCLExtInst::fract:
679 case SPIRV::OpenCLExtInst::modf:
680 case SPIRV::OpenCLExtInst::sincos:
681 // The last operand must be of a pointer to the base type represented
682 // by the previous operand.
683 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
684 "Expected v-reg");
686 STI, MRI, GR, MI, MI.getNumOperands() - 1,
688 MI.getOperand(MI.getNumOperands() - 2).getReg()));
689 break;
690 case SPIRV::OpenCLExtInst::prefetch:
691 // Expected `ptr` type is a pointer to float, integer or vector, but
692 // the pontee value can be wrapped into a struct.
693 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
694 "Expected v-reg");
696 MI.getNumOperands() - 2);
697 break;
698 }
699 } break;
700 }
701 }
702 }
704}
705
706// Modifies either operand PtrOpIdx or OpIdx so that the pointee type of
707// PtrOpIdx matches the type for operand OpIdx. Returns true if they already
708// match or if the instruction was modified to make them match.
710 MachineInstr &I, unsigned int PtrOpIdx, unsigned int OpIdx) const {
711 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
712 SPIRVTypeInst PtrType = GR.getResultType(I.getOperand(PtrOpIdx).getReg());
713
714 if (PtrType && PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
715 return true;
716
717 SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
718 SPIRVTypeInst OpType = GR.getResultType(I.getOperand(OpIdx).getReg());
719
720 if (PointeeType == OpType)
721 return true;
722
723 // getPointeeType yields nullptr for anything that is not an OpTypePointer.
724 // The early return above does not cover an untyped pointer nested in another
725 // type, such as a vector of pointers built for a scalarized vector GEP.
726 // typesLogicallyMatch dereferences both of its arguments, so bail out before
727 // calling it.
728 if (PointeeType && OpType && typesLogicallyMatch(PointeeType, OpType, GR)) {
729 // Apply OpCopyLogical to OpIdx.
730 if (I.getOperand(OpIdx).isDef() &&
731 insertLogicalCopyOnResult(I, PointeeType)) {
732 return true;
733 }
734
735 llvm_unreachable("Unable to add OpCopyLogical yet.");
736 return false;
737 }
738
739 return false;
740}
741
743 MachineInstr &I, SPIRVTypeInst NewResultType) const {
744 MachineRegisterInfo *MRI = &I.getMF()->getRegInfo();
745 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
746
747 Register NewResultReg =
748 createVirtualRegister(NewResultType, &GR, MRI, *I.getMF());
749 Register NewTypeReg = GR.getSPIRVTypeID(NewResultType);
750
751 assert(llvm::size(I.defs()) == 1 && "Expected only one def");
752 MachineOperand &OldResult = *I.defs().begin();
753 Register OldResultReg = OldResult.getReg();
754 MachineOperand &OldType = *I.uses().begin();
755 Register OldTypeReg = OldType.getReg();
756
757 OldResult.setReg(NewResultReg);
758 OldType.setReg(NewTypeReg);
759
760 MachineIRBuilder MIB(*I.getNextNode());
761 MIB.buildInstr(SPIRV::OpCopyLogical)
762 .addDef(OldResultReg)
763 .addUse(OldTypeReg)
764 .addUse(NewResultReg)
765 .constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(),
766 *STI.getRegBankInfo());
767 return true;
768}
769
772 switch (RMW->getOperation()) {
780 // AMD targets lower these to a helper call in SPIRVEmitIntrinsics, so keep
781 // them unexpanded there; other targets need the generic expansion.
782 return STI.getTargetTriple().getVendor() == Triple::AMD
787 default:
789 }
790}
791
794 // TODO: Pointer operand should be cast to integer in atomicrmw xchg, since
795 // SPIR-V only supports atomic exchange for integer and floating-point types.
797}
798
801 // TODO: pointer load should return CastToInteger, but
802 // convertAtomicLoadToIntegerType uses BitCast which asserts on pointer types.
804}
805
808 // TODO: pointer store should return CastToInteger, but
809 // convertAtomicStoreToIntegerType uses BitCast which asserts on pointer
810 // types.
812}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator MBBI
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register const TargetRegisterInfo * TRI
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
static void validateLifetimeStart(MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I)
static bool typesLogicallyMatch(const SPIRVTypeInst Ty1, const SPIRVTypeInst Ty2, SPIRVGlobalRegistry &GR)
static void validateVec1Ops(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &MI)
static void validatePtrTypes(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I, unsigned OpIdx, SPIRVTypeInst ResType, const Type *ResTy=nullptr)
static void validateGroupWaitEventsPtr(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I)
static void validatePtrUnwrapStructField(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I, unsigned OpIdx)
Register getTypeReg(MachineRegisterInfo *MRI, Register OpReg)
void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I)
void validateFunCallMachineDef(const SPIRVSubtarget &STI, MachineRegisterInfo *DefMRI, MachineRegisterInfo *CallMRI, SPIRVGlobalRegistry &GR, MachineInstr &FunCall, MachineInstr *FunDef)
void validateForwardCalls(const SPIRVSubtarget &STI, MachineRegisterInfo *DefMRI, SPIRVGlobalRegistry &GR, MachineInstr &FunDef)
const Function * validateFunCall(const SPIRVSubtarget &STI, MachineRegisterInfo *CallMRI, SPIRVGlobalRegistry &GR, MachineInstr &FunCall)
static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I, Register OpReg, unsigned OpIdx, SPIRVTypeInst NewPtrType)
static SPIRVTypeInst createNewPtrType(SPIRVGlobalRegistry &GR, MachineInstr &I, SPIRVTypeInst OpType, bool ReuseType, SPIRVTypeInst ResType, const Type *ResTy)
This file describes how to lower LLVM code to machine code.
an instruction that atomically reads a memory location, combines it with another value,...
@ FAdd
*p = old + v
@ FSub
*p = old - v
@ UIncWrap
Increment one up to a maximum value.
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ Nand
*p = ~(old & v)
BinOp getOperation() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
An instruction for reading from memory.
Machine Value Type.
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Helper class to build MachineInstr.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
const MachineOperand & getOperand(unsigned i) const
Flags
Flags values. These may be or'd together.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
int64_t getImm() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
Wrapper class representing virtual and physical registers.
Definition Register.h:20
void addForwardCall(const Function *F, MachineInstr *MI)
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
SPIRVTypeInst getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
unsigned getScalarOrVectorComponentCount(Register VReg) const
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
bool isBitcastCompatible(SPIRVTypeInst Type1, SPIRVTypeInst Type2) const
const MachineInstr * getFunctionDefinition(const Function *F)
Register getSPIRVTypeID(SPIRVTypeInst SpirvType) const
SPIRVTypeInst getPointeeType(SPIRVTypeInst PtrType)
SmallPtrSet< MachineInstr *, 8 > * getForwardCalls(const Function *F)
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
MachineFunction * setCurrentFunc(MachineFunction &MF)
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
const Function * getFunctionByDefinition(const MachineInstr *MI)
const SPIRVInstrInfo * getInstrInfo() const override
const SPIRVRegisterInfo * getRegisterInfo() const override
const RegisterBankInfo * getRegBankInfo() const override
AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const override
Returns how the given atomic atomicrmw should be cast by the IR-level AtomicExpand pass.
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
bool enforcePtrTypeCompatibility(MachineInstr &I, unsigned PtrOpIdx, unsigned OpIdx) const
unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const override
Return the number of registers that this ValueType will eventually require.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
bool insertLogicalCopyOnResult(MachineInstr &I, SPIRVTypeInst NewResultType) const
AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
SPIRVTargetLowering(const TargetMachine &TM, const SPIRVSubtarget &ST)
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
static LLVM_ABI TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})
Return a target extension type having the specified name and optional type and integer parameters.
Definition Type.cpp:931
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
TargetLowering(const TargetLowering &)=delete
Primary interface to the complete machine description for the target machine.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
MachineInstr * getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
bool isVectorType(SPIRVTypeInst SPVTy)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
AtomicOrdering
Atomic ordering for LLVM's memory model.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175