LLVM 24.0.0git
SPIRVGlobalRegistry.cpp
Go to the documentation of this file.
1//===-- SPIRVGlobalRegistry.cpp - SPIR-V Global Registry --------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the SPIRVGlobalRegistry class,
10// which is used to maintain rich type information required for SPIR-V even
11// after lowering from LLVM IR to GMIR. It can convert an llvm::Type into
12// an OpTypeXXX instruction, and map it to a virtual register. Also it builds
13// and supports consistency of constants and global variables.
14//
15//===----------------------------------------------------------------------===//
16
17#include "SPIRVGlobalRegistry.h"
18#include "SPIRV.h"
19#include "SPIRVBuiltins.h"
20#include "SPIRVSubtarget.h"
21#include "SPIRVUtils.h"
22#include "llvm/ADT/APInt.h"
23#include "llvm/IR/Constants.h"
25#include "llvm/IR/Function.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/IR/Type.h"
32#include <cassert>
33#include <functional>
34
35using namespace llvm;
36
37static bool allowEmitFakeUse(const Value *Arg) {
38 if (isSpvIntrinsic(Arg))
39 return false;
41 return false;
42 if (const auto *LI = dyn_cast<LoadInst>(Arg))
43 if (LI->getType()->isAggregateType())
44 return false;
45 return true;
46}
47
48static unsigned typeToAddressSpace(const Type *Ty) {
49 if (auto PType = dyn_cast<TypedPointerType>(Ty))
50 return PType->getAddressSpace();
51 if (auto PType = dyn_cast<PointerType>(Ty))
52 return PType->getAddressSpace();
53 if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);
54 ExtTy && isTypedPointerWrapper(ExtTy))
55 return ExtTy->getIntParameter(0);
56 reportFatalInternalError("Unable to convert LLVM type to SPIRVType");
57}
58
59static bool
60storageClassRequiresExplictLayout(SPIRV::StorageClass::StorageClass SC) {
61 switch (SC) {
62 case SPIRV::StorageClass::Uniform:
63 case SPIRV::StorageClass::PushConstant:
64 case SPIRV::StorageClass::StorageBuffer:
65 case SPIRV::StorageClass::PhysicalStorageBufferEXT:
66 return true;
67 case SPIRV::StorageClass::UniformConstant:
68 case SPIRV::StorageClass::Input:
69 case SPIRV::StorageClass::Output:
70 case SPIRV::StorageClass::Workgroup:
71 case SPIRV::StorageClass::CrossWorkgroup:
72 case SPIRV::StorageClass::Private:
73 case SPIRV::StorageClass::Function:
74 case SPIRV::StorageClass::Generic:
75 case SPIRV::StorageClass::AtomicCounter:
76 case SPIRV::StorageClass::Image:
77 case SPIRV::StorageClass::CallableDataNV:
78 case SPIRV::StorageClass::IncomingCallableDataNV:
79 case SPIRV::StorageClass::RayPayloadNV:
80 case SPIRV::StorageClass::HitAttributeNV:
81 case SPIRV::StorageClass::IncomingRayPayloadNV:
82 case SPIRV::StorageClass::ShaderRecordBufferNV:
83 case SPIRV::StorageClass::CodeSectionINTEL:
84 case SPIRV::StorageClass::DeviceOnlyINTEL:
85 case SPIRV::StorageClass::HostOnlyINTEL:
86 return false;
87 }
88 llvm_unreachable("Unknown SPIRV::StorageClass enum");
89}
90
92 : DL(DL), Bound(0), CurMF(nullptr) {}
93
94void SPIRVGlobalRegistry::constrainSelectedInstRegOperands(
95 MachineInstrBuilder &MIB) const {
96 const auto &ST = CurMF->getSubtarget();
97 MIB.constrainAllUses(*ST.getInstrInfo(), *ST.getRegisterInfo(),
98 *ST.getRegBankInfo());
99}
100
104 const SPIRVInstrInfo &TII) {
106 assignSPIRVTypeToVReg(SpirvType, VReg, *CurMF);
107 return SpirvType;
108}
109
111 const Type *Type, Register VReg, MachineIRBuilder &MIRBuilder,
112 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
113 SPIRVTypeInst SpirvType =
114 getOrCreateSPIRVType(Type, MIRBuilder, AccessQual, EmitIR);
115 assignSPIRVTypeToVReg(SpirvType, VReg, MIRBuilder.getMF());
116 return SpirvType;
117}
118
120 Register VReg,
121 const MachineFunction &MF) {
122 VRegToTypeMap[&MF][VReg] = SpirvType;
123}
124
126 auto Res = MRI.createGenericVirtualRegister(LLT::scalar(64));
127 MRI.setRegClass(Res, &SPIRV::TYPERegClass);
128 return Res;
129}
130
132 return createTypeVReg(MIRBuilder.getMF().getRegInfo());
133}
134
135SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeBool(MachineIRBuilder &MIRBuilder) {
136 return createConstOrTypeAtFunctionEntry(
137 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
138 return MIRBuilder.buildInstr(SPIRV::OpTypeBool)
139 .addDef(createTypeVReg(MIRBuilder));
140 });
141}
142
143unsigned SPIRVGlobalRegistry::adjustOpTypeIntWidth(unsigned Width) const {
144 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(CurMF->getSubtarget());
145 if (ST.canUseExtension(
146 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||
147 (Width == 4 && ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4)))
148 return Width;
149 if (Width <= 8)
150 return 8;
151 else if (Width <= 16)
152 return 16;
153 else if (Width <= 32)
154 return 32;
155 else if (Width <= 64)
156 return 64;
157 else if (Width <= 128)
158 return 128;
159 reportFatalUsageError("Unsupported Integer width!");
160}
161
162SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeInt(unsigned Width,
163 MachineIRBuilder &MIRBuilder,
164 bool IsSigned) {
165 Width = adjustOpTypeIntWidth(Width);
166 const SPIRVSubtarget &ST =
168 return createConstOrTypeAtFunctionEntry(MIRBuilder, [&](MachineIRBuilder
169 &MIRBuilder) {
170 if (Width == 4 && ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4)) {
171 MIRBuilder.buildInstr(SPIRV::OpExtension)
172 .addImm(SPIRV::Extension::SPV_INTEL_int4);
173 MIRBuilder.buildInstr(SPIRV::OpCapability)
174 .addImm(SPIRV::Capability::Int4TypeINTEL);
175 } else if ((!isPowerOf2_32(Width) || Width < 8) &&
176 ST.canUseExtension(
177 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers)) {
178 MIRBuilder.buildInstr(SPIRV::OpExtension)
179 .addImm(SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
180 MIRBuilder.buildInstr(SPIRV::OpCapability)
181 .addImm(SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
182 }
183 return MIRBuilder.buildInstr(SPIRV::OpTypeInt)
184 .addDef(createTypeVReg(MIRBuilder))
185 .addImm(Width)
186 .addImm(IsSigned ? 1 : 0);
187 });
188}
189
191SPIRVGlobalRegistry::getOpTypeFloat(uint32_t Width,
192 MachineIRBuilder &MIRBuilder) {
193 return createConstOrTypeAtFunctionEntry(MIRBuilder, [&](MachineIRBuilder
194 &MIRBuilder) {
195 return MIRBuilder.buildInstr(SPIRV::OpTypeFloat)
196 .addDef(createTypeVReg(MIRBuilder))
197 .addImm(Width);
198 });
199}
200
202SPIRVGlobalRegistry::getOpTypeFloat(uint32_t Width,
203 MachineIRBuilder &MIRBuilder,
204 SPIRV::FPEncoding::FPEncoding FPEncode) {
205 return createConstOrTypeAtFunctionEntry(MIRBuilder, [&](MachineIRBuilder
206 &MIRBuilder) {
207 return MIRBuilder.buildInstr(SPIRV::OpTypeFloat)
208 .addDef(createTypeVReg(MIRBuilder))
209 .addImm(Width)
210 .addImm(FPEncode);
211 });
212}
213
215 return createConstOrTypeAtFunctionEntry(
216 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
217 return MIRBuilder.buildInstr(SPIRV::OpTypeVoid)
218 .addDef(createTypeVReg(MIRBuilder));
219 });
220}
221
223 // Other maps that may hold MachineInstr*:
224 // - VRegToTypeMap: We cannot remove the definitions of `MI` from
225 // VRegToTypeMap because some calls to invalidateMachineInstr are replacing MI
226 // with another instruction defining the same register. We expect that if MI
227 // is a type instruction, and it is still referenced in VRegToTypeMap, then
228 // those registers are dead or the VRegToTypeMap is out-of-date. We do not
229 // expect passes to ask for the SPIR-V type of a dead register. If the
230 // VRegToTypeMap is out-of-date already, then there was an error before. We
231 // cannot add an assert to verify this because the VRegToTypeMap can be
232 // out-of-date.
233 // - FunctionToInstr & FunctionToInstrRev: At this point, we should not be
234 // deleting functions. No need to update.
235 // - AliasInstMDMap: Would require a linear search, and the Intel Alias
236 // instruction are not instructions instruction selection will be able to
237 // remove.
238
239 const SPIRVSubtarget &ST = MI->getMF()->getSubtarget<SPIRVSubtarget>();
240 [[maybe_unused]] const SPIRVInstrInfo *TII = ST.getInstrInfo();
241 assert(!TII->isAliasingInstr(*MI) &&
242 "Cannot invalidate aliasing instructions.");
243 assert(MI->getOpcode() != SPIRV::OpFunction &&
244 "Cannot invalidate OpFunction.");
245
246 if (MI->getOpcode() == SPIRV::OpFunctionCall) {
247 if (const auto *F = dyn_cast<Function>(MI->getOperand(2).getGlobal())) {
248 auto It = ForwardCalls.find(F);
249 if (It != ForwardCalls.end()) {
250 It->second.erase(MI);
251 if (It->second.empty())
252 ForwardCalls.erase(It);
253 }
254 }
255 }
256
257 const MachineFunction *MF = MI->getMF();
258 auto It = LastInsertedTypeMap.find(MF);
259 if (It != LastInsertedTypeMap.end() && It->second == MI)
260 LastInsertedTypeMap.erase(MF);
261 // remove from the duplicate tracker to avoid incorrect reuse
262 erase(MI);
263}
264
265const MachineInstr *SPIRVGlobalRegistry::createConstOrTypeAtFunctionEntry(
266 MachineIRBuilder &MIRBuilder,
267 std::function<MachineInstr *(MachineIRBuilder &)> Op) {
268 auto oldInsertPoint = MIRBuilder.getInsertPt();
269 assert(oldInsertPoint.isValid() && "Builder must have insertion point set!");
270 MachineBasicBlock *OldMBB = &MIRBuilder.getMBB();
271 MachineBasicBlock *NewMBB = &*MIRBuilder.getMF().begin();
272
273 auto LastInsertedType = LastInsertedTypeMap.find(CurMF);
274 if (LastInsertedType != LastInsertedTypeMap.end()) {
275 auto It = LastInsertedType->second->getIterator();
276 // It might happen that this instruction was removed from the first MBB,
277 // hence the Parent's check.
279 if (It->getParent() != NewMBB)
280 InsertAt = oldInsertPoint->getParent() == NewMBB
281 ? oldInsertPoint
282 : getInsertPtValidEnd(NewMBB);
283 else if (It->getNextNode())
284 InsertAt = It->getNextNode()->getIterator();
285 else
286 InsertAt = getInsertPtValidEnd(NewMBB);
287 MIRBuilder.setInsertPt(*NewMBB, InsertAt);
288 } else {
289 MIRBuilder.setInsertPt(*NewMBB, NewMBB->begin());
290 auto Result = LastInsertedTypeMap.try_emplace(CurMF, nullptr);
291 assert(Result.second);
292 LastInsertedType = Result.first;
293 }
294
295 MachineInstr *ConstOrType = Op(MIRBuilder);
296 // We expect all users of this function to insert definitions at the insertion
297 // point set above that is always the first MBB.
298 assert(ConstOrType->getParent() == NewMBB);
299 LastInsertedType->second = ConstOrType;
300 // Advance past any continued instructions so that the next type/constant
301 // is inserted after the full group, preserving required adjacency.
302 while (auto *Next = LastInsertedType->second->getNextNode()) {
303 unsigned Opc = Next->getOpcode();
304 if (Opc == SPIRV::OpTypeStructContinuedINTEL ||
305 Opc == SPIRV::OpConstantCompositeContinuedINTEL ||
306 Opc == SPIRV::OpSpecConstantCompositeContinuedINTEL ||
307 Opc == SPIRV::OpCompositeConstructContinuedINTEL)
308 LastInsertedType->second = Next;
309 else
310 break;
311 }
312
313 MIRBuilder.setInsertPt(*OldMBB, oldInsertPoint);
314 return ConstOrType;
315}
316
317SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeVectorImpl(
318 uint32_t NumElems, SPIRVTypeInst ElemType, MachineIRBuilder &MIRBuilder,
319 bool IsLongVectorEXT) {
320 if (ElemType.isPointer()) {
321 if (!cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget())
322 .canUseExtension(
323 SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
324 const Function &F = MIRBuilder.getMF().getFunction();
325 F.getContext().diagnose(DiagnosticInfoUnsupported(
326 F,
327 "Vector of pointers requires SPV_INTEL_masked_gather_scatter "
328 "extension",
329 DebugLoc(), DS_Error));
330 }
331 } else {
332 [[maybe_unused]] auto EleOpc = ElemType->getOpcode();
333 assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||
334 EleOpc == SPIRV::OpTypeBool) &&
335 "Invalid vector element type");
336 }
337
338 return createConstOrTypeAtFunctionEntry(
339 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
340 unsigned Op =
341 IsLongVectorEXT ? SPIRV::OpTypeVectorIdEXT : SPIRV::OpTypeVector;
342 Register VTy = createTypeVReg(MIRBuilder);
343 Register Ty = getSPIRVTypeID(ElemType);
344 auto MIB = MIRBuilder.buildInstr(Op).addDef(VTy).addUse(Ty);
345 if (!IsLongVectorEXT)
346 return MIB.addImm(NumElems);
347
348 const auto &ST = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
349 SPIRVTypeInst Int32Ty = getOrCreateSPIRVIntegerType(32, MIRBuilder);
350 return MIB.addUse(getOrCreateConstInt(NumElems, *MIB.getInstr(),
351 Int32Ty, *ST.getInstrInfo()));
352 });
353}
354
356SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVTypeInst ElemType,
357 MachineIRBuilder &MIRBuilder) {
358 assert(NumElems >= 2 && "SPIR-V OpTypeVector requires at least 2 components");
359 return getOpTypeVectorImpl(NumElems, ElemType, MIRBuilder);
360}
361
362SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeVectorIdEXT(
363 uint32_t NumElems, SPIRVTypeInst ElemType, MachineIRBuilder &MIRBuilder) {
364 assert(NumElems != 2 && NumElems != 3 && NumElems != 4 &&
365 (!MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>().isKernel() ||
366 (NumElems != 8 && NumElems != 16)) &&
367 "SPIR-V OpTypeVectorIdExt should only be used for extended vectors");
368 return getOpTypeVectorImpl(NumElems, ElemType, MIRBuilder, true);
369}
370
372 SPIRVTypeInst SpvType,
373 const SPIRVInstrInfo &TII,
374 bool ZeroAsNull) {
375 LLVMContext &Ctx = CurMF->getFunction().getContext();
376 auto *const CF = ConstantFP::get(Ctx, Val);
377 const MachineInstr *MI = findMI(CF, CurMF);
378 if (MI && (MI->getOpcode() == SPIRV::OpConstantNull ||
379 MI->getOpcode() == SPIRV::OpConstantF))
380 return MI->getOperand(0).getReg();
381 return createConstFP(CF, I, SpvType, TII, ZeroAsNull);
382}
383
386 SPIRVTypeInst SpvType,
387 const SPIRVInstrInfo &TII,
388 bool ZeroAsNull) {
389 unsigned BitWidth = getScalarOrVectorBitWidth(SpvType);
390 LLT LLTy = LLT::scalar(BitWidth);
391 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
392 CurMF->getRegInfo().setRegClass(Res, &SPIRV::fIDRegClass);
393 assignSPIRVTypeToVReg(SpvType, Res, *CurMF);
394
395 MachineInstr *DepMI =
396 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
397 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
398 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
399 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
401 // In OpenCL OpConstantNull - Scalar floating point: +0.0 (all bits 0)
402 if (CF->getValue().isPosZero() && ZeroAsNull) {
403 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
404 .addDef(Res)
405 .addUse(getSPIRVTypeID(SpvType));
406 } else {
407 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantF)
408 .addDef(Res)
409 .addUse(getSPIRVTypeID(SpvType));
412 MIB);
413 }
414 constrainSelectedInstRegOperands(MIB);
415 return MIB;
416 });
417 add(CF, Const);
418 return Res;
419}
420
422 SPIRVTypeInst SpvType,
423 const SPIRVInstrInfo &TII,
424 bool ZeroAsNull) {
426 SpvType, TII, ZeroAsNull);
427}
428
431 SPIRVTypeInst SpvType,
432 const SPIRVInstrInfo &TII,
433 bool ZeroAsNull) {
434 auto *const CI = ConstantInt::get(
436 const MachineInstr *MI = findMI(CI, CurMF);
437 if (MI && (MI->getOpcode() == SPIRV::OpConstantNull ||
438 MI->getOpcode() == SPIRV::OpConstantI))
439 return MI->getOperand(0).getReg();
440 return createConstInt(CI, I, SpvType, TII, ZeroAsNull);
441}
442
445 SPIRVTypeInst SpvType,
446 const SPIRVInstrInfo &TII,
447 bool ZeroAsNull) {
448 unsigned BitWidth = getScalarOrVectorBitWidth(SpvType);
449 LLT LLTy = LLT::scalar(BitWidth);
450 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
451 CurMF->getRegInfo().setRegClass(Res, &SPIRV::iIDRegClass);
453
454 MachineInstr *DepMI =
455 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
456 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
457 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
458 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
460 if (BitWidth == 1) {
461 MIB = MIRBuilder
462 .buildInstr(CI->isZero() ? SPIRV::OpConstantFalse
463 : SPIRV::OpConstantTrue)
464 .addDef(Res)
465 .addUse(getSPIRVTypeID(SpvType));
466 } else if (!CI->isZero() || !ZeroAsNull) {
467 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantI)
468 .addDef(Res)
469 .addUse(getSPIRVTypeID(SpvType));
470 addNumImm(CI->getValue(), MIB);
471 } else {
472 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
473 .addDef(Res)
474 .addUse(getSPIRVTypeID(SpvType));
475 }
476 constrainSelectedInstRegOperands(MIB);
477 return MIB;
478 });
479 add(CI, Const);
480 return Res;
481}
482
484 MachineIRBuilder &MIRBuilder,
485 SPIRVTypeInst SpvType,
486 bool EmitIR, bool ZeroAsNull) {
487 assert(SpvType);
488 auto &MF = MIRBuilder.getMF();
490 // TODO: Avoid implicit trunc?
491 // See https://github.com/llvm/llvm-project/issues/112510.
492 auto *const CI = ConstantInt::get(const_cast<IntegerType *>(Ty), Val,
493 /*IsSigned=*/false, /*ImplicitTrunc=*/true);
494 Register Res = find(CI, &MF);
495 if (Res.isValid())
496 return Res;
497
498 unsigned BitWidth = getScalarOrVectorBitWidth(SpvType);
499 LLT LLTy = LLT::scalar(BitWidth);
500 MachineRegisterInfo &MRI = MF.getRegInfo();
501 Res = MRI.createGenericVirtualRegister(LLTy);
502 MRI.setRegClass(Res, &SPIRV::iIDRegClass);
503 assignTypeToVReg(Ty, Res, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
504 EmitIR);
505
506 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
507 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
508 if (EmitIR)
509 return MIRBuilder.buildConstant(Res, *CI);
510 Register SpvTypeReg = getSPIRVTypeID(SpvType);
512 if (Val || !ZeroAsNull) {
513 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantI)
514 .addDef(Res)
515 .addUse(SpvTypeReg);
516 addNumImm(APInt(BitWidth, Val), MIB);
517 } else {
518 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
519 .addDef(Res)
520 .addUse(SpvTypeReg);
521 }
522 constrainSelectedInstRegOperands(MIB);
523 return MIB;
524 });
525 add(CI, Const);
526 return Res;
527}
528
530 MachineIRBuilder &MIRBuilder,
531 SPIRVTypeInst SpvType) {
532 auto &MF = MIRBuilder.getMF();
533 LLVMContext &Ctx = MF.getFunction().getContext();
534 if (!SpvType)
535 SpvType = getOrCreateSPIRVType(Type::getFloatTy(Ctx), MIRBuilder,
536 SPIRV::AccessQualifier::ReadWrite, true);
537 auto *const CF = ConstantFP::get(Ctx, Val);
538 Register Res = find(CF, &MF);
539 if (Res.isValid())
540 return Res;
541
543 Res = MF.getRegInfo().createGenericVirtualRegister(LLTy);
544 MF.getRegInfo().setRegClass(Res, &SPIRV::fIDRegClass);
545 assignSPIRVTypeToVReg(SpvType, Res, MF);
546
547 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
548 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
550 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantF)
551 .addDef(Res)
552 .addUse(getSPIRVTypeID(SpvType));
553 addNumImm(CF->getValueAPF().bitcastToAPInt(), MIB);
554 return MIB;
555 });
556 add(CF, Const);
557 return Res;
558}
559
560Register SPIRVGlobalRegistry::getOrCreateBaseRegister(
561 Constant *Val, MachineInstr &I, SPIRVTypeInst SpvType,
562 const SPIRVInstrInfo &TII, unsigned BitWidth, bool ZeroAsNull) {
563 SPIRVTypeInst Type = SpvType;
564 if (isVectorType(SpvType) || SpvType->getOpcode() == SPIRV::OpTypeArray) {
565 auto EleTypeReg = SpvType->getOperand(1).getReg();
566 Type = getSPIRVTypeForVReg(EleTypeReg);
567 }
568 if (Type->getOpcode() == SPIRV::OpTypeFloat) {
570 return getOrCreateConstFP(cast<ConstantFP>(Val)->getValue(), I, SpvBaseType,
571 TII, ZeroAsNull);
572 }
573 assert(Type->getOpcode() == SPIRV::OpTypeInt);
574 SPIRVTypeInst SpvBaseType = getOrCreateSPIRVIntegerType(BitWidth, I, TII);
575 return getOrCreateConstInt(Val->getUniqueInteger(), I, SpvBaseType, TII,
576 ZeroAsNull);
577}
578
579Register SPIRVGlobalRegistry::getOrCreateCompositeOrNull(
580 Constant *Val, MachineInstr &I, SPIRVTypeInst SpvType,
581 const SPIRVInstrInfo &TII, Constant *CA, unsigned BitWidth,
582 unsigned ElemCnt, bool ZeroAsNull) {
583 if (Register R = find(CA, CurMF); R.isValid())
584 return R;
585
586 bool IsNull = Val->isNullValue() && ZeroAsNull;
587 Register ElemReg;
588 if (!IsNull)
589 ElemReg =
590 getOrCreateBaseRegister(Val, I, SpvType, TII, BitWidth, ZeroAsNull);
591
592 LLT LLTy = LLT::scalar(64);
593 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
594 CurMF->getRegInfo().setRegClass(Res, getRegClass(SpvType));
595 assignSPIRVTypeToVReg(SpvType, Res, *CurMF);
596
597 MachineInstr *DepMI =
598 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
599 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
600 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
601 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
602 MachineInstrBuilder MIB;
603 if (!IsNull) {
604 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantComposite)
605 .addDef(Res)
606 .addUse(getSPIRVTypeID(SpvType));
607 for (unsigned i = 0; i < ElemCnt; ++i)
608 MIB.addUse(ElemReg);
609 } else {
610 MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)
611 .addDef(Res)
612 .addUse(getSPIRVTypeID(SpvType));
613 }
614 constrainSelectedInstRegOperands(MIB);
615 return MIB;
616 });
617 add(CA, NewMI);
618 return Res;
619}
620
623 SPIRVTypeInst SpvType,
624 const SPIRVInstrInfo &TII,
625 bool ZeroAsNull) {
627 I, SpvType, TII, ZeroAsNull);
628}
629
632 SPIRVTypeInst SpvType,
633 const SPIRVInstrInfo &TII,
634 bool ZeroAsNull) {
635 const Type *LLVMTy = getTypeForSPIRVType(SpvType);
636 assert(LLVMTy->isVectorTy() &&
637 "Expected vector type for constant vector creation");
638 const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(LLVMTy);
639 Type *LLVMBaseTy = LLVMVecTy->getElementType();
640 [[maybe_unused]] const auto &ST = I.getMF()->getSubtarget<SPIRVSubtarget>();
641 assert((LLVMBaseTy->isIntegerTy() ||
642 (LLVMBaseTy->isPointerTy() &&
643 ST.canUseExtension(
644 SPIRV::Extension::SPV_INTEL_masked_gather_scatter))) &&
645 "Expected either integer element type for APInt constant vector or "
646 "pointer type if the SPV_INTEL_masked_gather_scatter extension is "
647 "enabled!");
648 Constant *ConstVal = nullptr;
649 if (LLVMBaseTy->isIntegerTy()) {
650 ConstVal = ConstantInt::get(LLVMBaseTy, Val);
651 } else {
652 if (Val.isZero())
653 ConstVal = ConstantPointerNull::get(LLVMBaseTy);
654 else
655 llvm_unreachable("Vectors of non-null constant pointers unimplemented!");
656 }
657 auto *ConstVec =
658 ConstantVector::getSplat(LLVMVecTy->getElementCount(), ConstVal);
659 unsigned BW = getScalarOrVectorBitWidth(SpvType);
660 return getOrCreateCompositeOrNull(ConstVal, I, SpvType, TII, ConstVec, BW,
662 ZeroAsNull);
663}
664
667 SPIRVTypeInst SpvType,
668 const SPIRVInstrInfo &TII,
669 bool ZeroAsNull) {
670 const Type *LLVMTy = getTypeForSPIRVType(SpvType);
671 assert(LLVMTy->isVectorTy());
672 const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(LLVMTy);
673 Type *LLVMBaseTy = LLVMVecTy->getElementType();
674 assert(LLVMBaseTy->isFloatingPointTy());
675 auto *ConstVal = ConstantFP::get(LLVMBaseTy, Val);
676 auto *ConstVec =
677 ConstantVector::getSplat(LLVMVecTy->getElementCount(), ConstVal);
678 unsigned BW = getScalarOrVectorBitWidth(SpvType);
679 return getOrCreateCompositeOrNull(ConstVal, I, SpvType, TII, ConstVec, BW,
681 ZeroAsNull);
682}
683
685 uint64_t Val, size_t Num, MachineInstr &I, SPIRVTypeInst SpvType,
686 const SPIRVInstrInfo &TII) {
687 const Type *LLVMTy = getTypeForSPIRVType(SpvType);
688 assert(LLVMTy->isArrayTy());
689 const ArrayType *LLVMArrTy = cast<ArrayType>(LLVMTy);
690 Type *LLVMBaseTy = LLVMArrTy->getElementType();
691 Constant *CI = ConstantInt::get(LLVMBaseTy, Val);
692 SPIRVTypeInst SpvBaseTy =
694 unsigned BW = getScalarOrVectorBitWidth(SpvBaseTy);
695 // The following is reasonably unique key that is better that [Val]. The naive
696 // alternative would be something along the lines of:
697 // SmallVector<Constant *> NumCI(Num, CI);
698 // Constant *UniqueKey =
699 // ConstantArray::get(const_cast<ArrayType*>(LLVMArrTy), NumCI);
700 // that would be a truly unique but dangerous key, because it could lead to
701 // the creation of constants of arbitrary length (that is, the parameter of
702 // memset) which were missing in the original module.
703 Type *I64Ty = Type::getInt64Ty(LLVMBaseTy->getContext());
705 {PoisonValue::get(const_cast<ArrayType *>(LLVMArrTy)),
706 ConstantInt::get(LLVMBaseTy, Val), ConstantInt::get(I64Ty, Num)});
707 return getOrCreateCompositeOrNull(CI, I, SpvType, TII, UniqueKey, BW,
708 LLVMArrTy->getNumElements());
709}
710
711Register SPIRVGlobalRegistry::getOrCreateIntCompositeOrNull(
712 uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType,
713 bool EmitIR, Constant *CA, unsigned BitWidth, unsigned ElemCnt) {
714 if (Register R = find(CA, CurMF); R.isValid())
715 return R;
716
717 Register ElemReg;
718 if (Val || EmitIR) {
719 SPIRVTypeInst SpvBaseType =
721 ElemReg = buildConstantInt(Val, MIRBuilder, SpvBaseType, EmitIR);
722 }
723 LLT LLTy = EmitIR ? LLT::fixed_vector(ElemCnt, BitWidth) : LLT::scalar(64);
724 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
725 CurMF->getRegInfo().setRegClass(Res, &SPIRV::iIDRegClass);
726 assignSPIRVTypeToVReg(SpvType, Res, *CurMF);
727
728 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
729 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
730 if (EmitIR)
731 return MIRBuilder.buildSplatBuildVector(Res, ElemReg);
732
733 if (Val) {
734 auto MIB = MIRBuilder.buildInstr(SPIRV::OpConstantComposite)
735 .addDef(Res)
736 .addUse(getSPIRVTypeID(SpvType));
737 for (unsigned i = 0; i < ElemCnt; ++i)
738 MIB.addUse(ElemReg);
739 return MIB;
740 }
741
742 return MIRBuilder.buildInstr(SPIRV::OpConstantNull)
743 .addDef(Res)
744 .addUse(getSPIRVTypeID(SpvType));
745 });
746 add(CA, NewMI);
747 return Res;
748}
749
751 uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType,
752 bool EmitIR) {
753 const Type *LLVMTy = getTypeForSPIRVType(SpvType);
754 assert(LLVMTy->isVectorTy());
755 const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(LLVMTy);
756 Type *LLVMBaseTy = LLVMVecTy->getElementType();
757 const auto ConstInt = ConstantInt::get(LLVMBaseTy, Val);
758 auto ConstVec =
759 ConstantVector::getSplat(LLVMVecTy->getElementCount(), ConstInt);
760 unsigned BW = getScalarOrVectorBitWidth(SpvType);
761 return getOrCreateIntCompositeOrNull(
762 Val, MIRBuilder, SpvType, EmitIR, ConstVec, BW,
764}
765
768 SPIRVTypeInst SpvType) {
769 const Type *Ty = getTypeForSPIRVType(SpvType);
770 unsigned AddressSpace = typeToAddressSpace(Ty);
772 assert(ElemTy);
775 Register Res = find(CP, CurMF);
776 if (Res.isValid())
777 return Res;
778
780 Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
781 CurMF->getRegInfo().setRegClass(Res, &SPIRV::pIDRegClass);
782 assignSPIRVTypeToVReg(SpvType, Res, *CurMF);
783
784 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
785 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
786 return MIRBuilder.buildInstr(SPIRV::OpConstantNull)
787 .addDef(Res)
788 .addUse(getSPIRVTypeID(SpvType));
789 });
790 add(CP, NewMI);
791 return Res;
792}
793
796 unsigned Param, unsigned FilerMode,
797 MachineIRBuilder &MIRBuilder) {
798 auto Sampler =
799 ResReg.isValid()
800 ? ResReg
801 : MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
802 SPIRVTypeInst TypeSampler = getOrCreateOpTypeSampler(MIRBuilder);
803 Register TypeSamplerReg = getSPIRVTypeID(TypeSampler);
804 // We cannot use createOpType() logic here, because of the
805 // GlobalISel/IRTranslator.cpp check for a tail call that expects that
806 // MIRBuilder.getInsertPt() has a previous instruction. If this constant is
807 // inserted as a result of "__translate_sampler_initializer()" this would
808 // break this IRTranslator assumption.
809 MIRBuilder.buildInstr(SPIRV::OpConstantSampler)
811 .addUse(TypeSamplerReg)
813 .addImm(Param)
814 .addImm(FilerMode);
815 return Sampler;
816}
817
820 const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage,
821 const MachineInstr *Init, bool IsConst,
822 const std::optional<SPIRV::LinkageType::LinkageType> &LinkageType,
823 MachineIRBuilder &MIRBuilder, bool IsInstSelector) {
824 const GlobalVariable *GVar = nullptr;
825 if (GV) {
827 } else {
828 // If GV is not passed explicitly, use the name to find or construct
829 // the global variable.
830 Module *M = MIRBuilder.getMF().getFunction().getParent();
831 GVar = M->getGlobalVariable(Name);
832 if (GVar == nullptr) {
833 const Type *Ty = getTypeForSPIRVType(BaseType); // TODO: check type.
834 if (auto *TPTy = dyn_cast<TypedPointerType>(Ty))
835 Ty = PointerType::get(M->getContext(), TPTy->getAddressSpace());
836 // Module takes ownership of the global var.
837 GVar = new GlobalVariable(*M, const_cast<Type *>(Ty), false,
839 Twine(Name));
840 }
841 GV = GVar;
842 }
843
844 const MachineFunction *MF = &MIRBuilder.getMF();
845 Register Reg = find(GVar, MF);
846 if (Reg.isValid()) {
847 if (Reg != ResVReg)
848 MIRBuilder.buildCopy(ResVReg, Reg);
849 return ResVReg;
850 }
851
852 // Emit the OpVariable into the entry block to ensure the def dominates
853 // all uses across all MBBs.
854 MachineBasicBlock &EntryBB = MIRBuilder.getMF().front();
855 MachineIRBuilder GVBuilder(MIRBuilder.getState());
856 if (&GVBuilder.getMBB() != &EntryBB)
857 GVBuilder.setInsertPt(EntryBB, EntryBB.getFirstTerminator());
858
859 // Pointers to opaque types stay typed even with the extension on, so emit the
860 // untyped variant only when the result is actually an untyped pointer.
861 const bool UseUntypedPointers =
862 BaseType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
863 const unsigned VariableOpcode =
864 UseUntypedPointers ? SPIRV::OpUntypedVariableKHR : SPIRV::OpVariable;
865
866 auto MIB = GVBuilder.buildInstr(VariableOpcode)
867 .addDef(ResVReg)
869 .addImm(static_cast<uint32_t>(Storage));
870
871 // OpUntypedVariableKHR takes an extra Data Type operand right after the
872 // storage class, holding the global's value type.
873 if (UseUntypedPointers) {
875 if (!DataType)
876 DataType = getOrCreateSPIRVType(GV->getValueType(), GVBuilder,
877 SPIRV::AccessQualifier::ReadWrite,
878 /*EmitIR=*/false);
879 if (!DataType) {
880 const Function &F = MIRBuilder.getMF().getFunction();
881 F.getContext().diagnose(DiagnosticInfoUnsupported(
882 F,
883 "Could not deduce the data type of untyped global variable '" +
884 GVar->getName() + "'",
885 DebugLoc(), DS_Error));
886 // Recover with i8 so that codegen can finish and report the error.
887 DataType =
888 getOrCreateSPIRVType(Type::getInt8Ty(F.getContext()), GVBuilder,
889 SPIRV::AccessQualifier::ReadWrite,
890 /*EmitIR=*/false);
891 }
892 MIB.addUse(getSPIRVTypeID(DataType));
893 }
894
895 if (Init)
896 MIB.addUse(Init->getOperand(0).getReg());
897 // ISel may introduce a new register on this step, so we need to add it to
898 // DT and correct its type avoiding fails on the next stage.
899 if (IsInstSelector) {
900 constrainSelectedInstRegOperands(MIB);
901 }
902 add(GVar, MIB);
903
904 Reg = MIB->getOperand(0).getReg();
905 addGlobalObject(GVar, MF, Reg);
906
907 // Set to Reg the same type as ResVReg has.
908 auto MRI = MIRBuilder.getMRI();
909 if (Reg != ResVReg) {
910 LLT RegLLTy =
911 LLT::pointer(MRI->getType(ResVReg).getAddressSpace(), getPointerSize());
912 MRI->setType(Reg, RegLLTy);
913 assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF());
914 } else {
915 // Our knowledge about the type may be updated.
916 // If that's the case, we need to update a type
917 // associated with the register.
918 SPIRVTypeInst DefType = getSPIRVTypeForVReg(ResVReg);
919 if (!DefType || DefType != SPIRVTypeInst(BaseType))
920 assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF());
921 }
922
923 // If it's a global variable with name, output OpName for it.
924 if (GVar && GVar->hasName())
925 buildOpName(Reg, GVar->getName(), MIRBuilder);
926
927 // Output decorations for the GV.
928 // TODO: maybe move to GenerateDecorations pass.
929 const SPIRVSubtarget &ST =
931 if (IsConst && !ST.isShader())
932 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Constant, {});
933
934 if (GVar && GVar->getAlign().valueOrOne().value() != 1 && !ST.isShader()) {
935 unsigned Alignment = (unsigned)GVar->getAlign().valueOrOne().value();
936 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Alignment, {Alignment});
937 }
938
939 if (LinkageType)
940 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::LinkageAttributes,
941 {static_cast<uint32_t>(*LinkageType)}, Name);
942
943 SPIRV::BuiltIn::BuiltIn BuiltInId;
944 if (getSpirvBuiltInIdByName(Name, BuiltInId))
945 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::BuiltIn,
946 {static_cast<uint32_t>(BuiltInId)});
947
948 // If it's a global variable with "spirv.Decorations" metadata node
949 // recognize it as a SPIR-V friendly LLVM IR and parse "spirv.Decorations"
950 // arguments.
951 MDNode *GVarMD = nullptr;
952 if (GVar && (GVarMD = GVar->getMetadata("spirv.Decorations")) != nullptr)
953 buildOpSpirvDecorations(Reg, MIRBuilder, GVarMD, ST);
954
955 return Reg;
956}
957
958// Returns a name based on the Type. Notes that this does not look at
959// decorations, and will return the same string for two types that are the same
960// except for decorations.
962 SPIRVTypeInst VarType, uint32_t Set, uint32_t Binding, StringRef Name,
963 MachineIRBuilder &MIRBuilder) {
964 Register VarReg =
965 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
966
967 buildGlobalVariable(VarReg, VarType, Name, nullptr,
968 getPointerStorageClass(VarType), nullptr, false,
969 std::nullopt, MIRBuilder, false);
970
971 buildOpDecorate(VarReg, MIRBuilder, SPIRV::Decoration::DescriptorSet, {Set});
972 buildOpDecorate(VarReg, MIRBuilder, SPIRV::Decoration::Binding, {Binding});
973 return VarReg;
974}
975
976// TODO: Double check the calls to getOpTypeArray to make sure that `ElemType`
977// is explicitly laid out when required.
978SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeArray(uint32_t NumElems,
979 SPIRVTypeInst ElemType,
980 MachineIRBuilder &MIRBuilder,
981 bool ExplicitLayoutRequired,
982 bool EmitIR) {
983 assert((ElemType->getOpcode() != SPIRV::OpTypeVoid) &&
984 "Invalid array element type");
985 SPIRVTypeInst SpvTypeInt32 = getOrCreateSPIRVIntegerType(32, MIRBuilder);
986 SPIRVTypeInst ArrayType = nullptr;
987 const SPIRVSubtarget &ST =
989 if (NumElems != 0) {
990 Register NumElementsVReg =
991 buildConstantInt(NumElems, MIRBuilder, SpvTypeInt32, EmitIR);
992 ArrayType = createConstOrTypeAtFunctionEntry(
993 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
994 return MIRBuilder.buildInstr(SPIRV::OpTypeArray)
995 .addDef(createTypeVReg(MIRBuilder))
996 .addUse(getSPIRVTypeID(ElemType))
997 .addUse(NumElementsVReg);
998 });
999 } else if (ST.getTargetTriple().getVendor() == Triple::VendorType::AMD) {
1000 // We set the array size to the token UINT64_MAX value, which is generally
1001 // illegal (the maximum legal size is 61-bits) for the foreseeable future.
1002 SPIRVTypeInst SpvTypeInt64 = getOrCreateSPIRVIntegerType(64, MIRBuilder);
1003 Register NumElementsVReg =
1004 buildConstantInt(UINT64_MAX, MIRBuilder, SpvTypeInt64, EmitIR);
1005 ArrayType = createConstOrTypeAtFunctionEntry(
1006 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1007 return MIRBuilder.buildInstr(SPIRV::OpTypeArray)
1008 .addDef(createTypeVReg(MIRBuilder))
1009 .addUse(getSPIRVTypeID(ElemType))
1010 .addUse(NumElementsVReg);
1011 });
1012 } else {
1013 if (!ST.isShader()) {
1015 "Runtime arrays are not allowed in non-shader "
1016 "SPIR-V modules");
1017 return nullptr;
1018 }
1019 ArrayType = createConstOrTypeAtFunctionEntry(
1020 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1021 return MIRBuilder.buildInstr(SPIRV::OpTypeRuntimeArray)
1022 .addDef(createTypeVReg(MIRBuilder))
1023 .addUse(getSPIRVTypeID(ElemType));
1024 });
1025 }
1026
1027 if (ExplicitLayoutRequired && !isResourceType(ElemType)) {
1028 Type *ET = const_cast<Type *>(getTypeForSPIRVType(ElemType));
1029 addArrayStrideDecorations(ArrayType->defs().begin()->getReg(), ET,
1030 MIRBuilder);
1031 }
1032
1033 return ArrayType;
1034}
1035
1037SPIRVGlobalRegistry::getOpTypeOpaque(const StructType *Ty,
1038 MachineIRBuilder &MIRBuilder) {
1039 assert(Ty->hasName());
1040 StringRef Name = Ty->hasName() ? Ty->getName() : "";
1041 Register ResVReg = createTypeVReg(MIRBuilder);
1042 return createConstOrTypeAtFunctionEntry(
1043 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1044 auto MIB = MIRBuilder.buildInstr(SPIRV::OpTypeOpaque).addDef(ResVReg);
1045 addStringImm(Name, MIB);
1046 buildOpName(ResVReg, Name, MIRBuilder);
1047 return MIB;
1048 });
1049}
1050
1051SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeStruct(
1052 const StructType *Ty, MachineIRBuilder &MIRBuilder,
1053 SPIRV::AccessQualifier::AccessQualifier AccQual,
1054 StructOffsetDecorator Decorator, bool EmitIR) {
1055 Type *OriginalElementType = nullptr;
1056 uint64_t TotalSize = 0;
1057 if (matchPeeledArrayPattern(Ty, OriginalElementType, TotalSize)) {
1058 SPIRVTypeInst ElementSPIRVType = findSPIRVType(
1059 OriginalElementType, MIRBuilder, AccQual,
1060 /* ExplicitLayoutRequired= */ Decorator != nullptr, EmitIR);
1061 return getOpTypeArray(TotalSize, ElementSPIRVType, MIRBuilder,
1062 /*ExplicitLayoutRequired=*/Decorator != nullptr,
1063 EmitIR);
1064 }
1065
1066 const SPIRVSubtarget &ST =
1067 cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());
1068 SmallVector<Register, 4> FieldTypes;
1069 constexpr unsigned MaxWordCount = UINT16_MAX;
1070 const size_t NumElements = Ty->getNumElements();
1071
1072 size_t MaxNumElements = MaxWordCount - 2;
1073 size_t SPIRVStructNumElements = NumElements;
1074 if (NumElements > MaxNumElements) {
1075 // Do adjustments for continued instructions.
1076 SPIRVStructNumElements = MaxNumElements;
1077 MaxNumElements = MaxWordCount - 1;
1078 }
1079
1080 for (const auto &Elem : Ty->elements()) {
1081 SPIRVTypeInst ElemTy = findSPIRVType(
1082 toTypedPointer(Elem), MIRBuilder, AccQual,
1083 /* ExplicitLayoutRequired= */ Decorator != nullptr, EmitIR);
1084 assert(ElemTy && ElemTy->getOpcode() != SPIRV::OpTypeVoid &&
1085 "Invalid struct element type");
1086 FieldTypes.push_back(getSPIRVTypeID(ElemTy));
1087 }
1088 Register ResVReg = createTypeVReg(MIRBuilder);
1089 if (Ty->hasName())
1090 buildOpName(ResVReg, Ty->getName(), MIRBuilder);
1091 if (Ty->isPacked() && !ST.isShader())
1092 buildOpDecorate(ResVReg, MIRBuilder, SPIRV::Decoration::CPacked, {});
1093
1094 SPIRVTypeInst SPVType = createConstOrTypeAtFunctionEntry(
1095 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1096 auto MIBStruct =
1097 MIRBuilder.buildInstr(SPIRV::OpTypeStruct).addDef(ResVReg);
1098 for (size_t I = 0; I < SPIRVStructNumElements; ++I)
1099 MIBStruct.addUse(FieldTypes[I]);
1100 for (size_t I = SPIRVStructNumElements; I < NumElements;
1101 I += MaxNumElements) {
1102 auto MIBCont =
1103 MIRBuilder.buildInstr(SPIRV::OpTypeStructContinuedINTEL);
1104 for (size_t J = I; J < std::min(I + MaxNumElements, NumElements); ++J)
1105 MIBCont.addUse(FieldTypes[J]);
1106 }
1107 return MIBStruct;
1108 });
1109
1110 if (Decorator)
1111 Decorator(SPVType->defs().begin()->getReg());
1112
1113 return SPVType;
1114}
1115
1116SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSpecialType(
1117 const Type *Ty, MachineIRBuilder &MIRBuilder,
1118 SPIRV::AccessQualifier::AccessQualifier AccQual) {
1119 assert(isSpecialOpaqueType(Ty) && "Not a special opaque builtin type");
1120 return SPIRV::lowerBuiltinType(Ty, AccQual, MIRBuilder, this);
1121}
1122
1123SPIRVTypeInst SPIRVGlobalRegistry::getOpTypePointer(
1124 SPIRV::StorageClass::StorageClass SC, SPIRVTypeInst ElemType,
1125 MachineIRBuilder &MIRBuilder, Register Reg) {
1126 // Check if we should use untyped pointers.
1127 const SPIRVSubtarget &ST =
1128 cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());
1129 if (shouldUseUntypedPointer(ElemType, ST))
1130 return getOrCreateSPIRVUntypedPointerType(SC, MIRBuilder);
1131
1132 if (!Reg.isValid())
1133 Reg = createTypeVReg(MIRBuilder);
1134
1135 return createConstOrTypeAtFunctionEntry(MIRBuilder, [&](MachineIRBuilder
1136 &MIRBuilder) {
1137 return MIRBuilder.buildInstr(SPIRV::OpTypePointer)
1138 .addDef(Reg)
1139 .addImm(static_cast<uint32_t>(SC))
1140 .addUse(getSPIRVTypeID(ElemType));
1141 });
1142}
1143
1144SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeFunction(
1145 const FunctionType *Ty, SPIRVTypeInst RetType,
1146 const SmallVectorImpl<SPIRVTypeInst> &ArgTypes,
1147 MachineIRBuilder &MIRBuilder) {
1148 const SPIRVSubtarget *ST =
1149 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1150 if (Ty->isVarArg() && ST->isShader()) {
1151 Function &Fn = MIRBuilder.getMF().getFunction();
1152 Ty->getContext().diagnose(DiagnosticInfoUnsupported(
1153 Fn, "SPIR-V shaders do not support variadic functions",
1154 MIRBuilder.getDebugLoc()));
1155 }
1156 return createConstOrTypeAtFunctionEntry(MIRBuilder, [&](MachineIRBuilder
1157 &MIRBuilder) {
1158 auto MIB = MIRBuilder.buildInstr(SPIRV::OpTypeFunction)
1159 .addDef(createTypeVReg(MIRBuilder))
1160 .addUse(getSPIRVTypeID(RetType));
1161 for (auto &ArgType : ArgTypes)
1162 MIB.addUse(getSPIRVTypeID(ArgType));
1163 return MIB;
1164 });
1165}
1166
1168 const Type *Ty, SPIRVTypeInst RetType,
1169 const SmallVectorImpl<SPIRVTypeInst> &ArgTypes,
1170 MachineIRBuilder &MIRBuilder) {
1171 if (const MachineInstr *MI = findMI(Ty, false, &MIRBuilder.getMF()))
1172 return MI;
1173 const MachineInstr *NewMI =
1174 getOpTypeFunction(cast<FunctionType>(Ty), RetType, ArgTypes, MIRBuilder);
1175 add(Ty, false, NewMI);
1176 return finishCreatingSPIRVType(Ty, NewMI);
1177}
1178
1179SPIRVTypeInst SPIRVGlobalRegistry::findSPIRVType(
1180 const Type *Ty, MachineIRBuilder &MIRBuilder,
1181 SPIRV::AccessQualifier::AccessQualifier AccQual,
1182 bool ExplicitLayoutRequired, bool EmitIR) {
1183 const auto &STI = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
1184 // Treat <1 x T> as T if the SPV_EXT_long_vector extension is not available.
1185 if (auto *FVT = dyn_cast<FixedVectorType>(Ty);
1186 FVT && FVT->getNumElements() == 1 &&
1187 !STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))
1188 return findSPIRVType(FVT->getElementType(), MIRBuilder, AccQual,
1189 ExplicitLayoutRequired, EmitIR);
1190 Ty = adjustIntTypeByWidth(Ty);
1191 // TODO: findMI needs to know if a layout is required.
1192 if (const MachineInstr *MI =
1193 findMI(Ty, ExplicitLayoutRequired, &MIRBuilder.getMF()))
1194 return MI;
1195 if (auto It = ForwardPointerTypes.find(Ty); It != ForwardPointerTypes.end())
1196 return It->second;
1197 return restOfCreateSPIRVType(Ty, MIRBuilder, AccQual, ExplicitLayoutRequired,
1198 EmitIR);
1199}
1200
1202 assert(SpirvType && "Attempting to get type id for nullptr type.");
1203 if (SpirvType->getOpcode() == SPIRV::OpTypeForwardPointer ||
1204 SpirvType->getOpcode() == SPIRV::OpTypeStructContinuedINTEL)
1205 return SpirvType->uses().begin()->getReg();
1206 return SpirvType->defs().begin()->getReg();
1207}
1208
1209// We need to use a new LLVM integer type if there is a mismatch between
1210// number of bits in LLVM and SPIRV integer types to let DuplicateTracker
1211// ensure uniqueness of a SPIRV type by the corresponding LLVM type. Without
1212// such an adjustment SPIRVGlobalRegistry::getOpTypeInt() could create the
1213// same "OpTypeInt 8" type for a series of LLVM integer types with number of
1214// bits less than 8. This would lead to duplicate type definitions
1215// eventually due to the method that DuplicateTracker utilizes to reason
1216// about uniqueness of type records.
1217const Type *SPIRVGlobalRegistry::adjustIntTypeByWidth(const Type *Ty) const {
1218 if (auto IType = dyn_cast<IntegerType>(Ty)) {
1219 unsigned SrcBitWidth = IType->getBitWidth();
1220 if (SrcBitWidth > 1) {
1221 unsigned BitWidth = adjustOpTypeIntWidth(SrcBitWidth);
1222 // Maybe change source LLVM type to keep DuplicateTracker consistent.
1223 if (SrcBitWidth != BitWidth)
1224 Ty = IntegerType::get(Ty->getContext(), BitWidth);
1225 }
1226 }
1227 return Ty;
1228}
1229
1230SPIRVTypeInst SPIRVGlobalRegistry::createSPIRVType(
1231 const Type *Ty, MachineIRBuilder &MIRBuilder,
1232 SPIRV::AccessQualifier::AccessQualifier AccQual,
1233 bool ExplicitLayoutRequired, bool EmitIR) {
1234 if (isSpecialOpaqueType(Ty))
1235 return getOrCreateSpecialType(Ty, MIRBuilder, AccQual);
1236
1237 if (const MachineInstr *MI =
1238 findMI(Ty, ExplicitLayoutRequired, &MIRBuilder.getMF()))
1239 return MI;
1240
1241 if (auto IType = dyn_cast<IntegerType>(Ty)) {
1242 const unsigned Width = IType->getBitWidth();
1243 return Width == 1 ? getOpTypeBool(MIRBuilder)
1244 : getOpTypeInt(Width, MIRBuilder, false);
1245 }
1246 if (Ty->isFloatingPointTy()) {
1247 if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty())
1248 llvm::reportFatalUsageError("fp128 is not supported in SPIR-V");
1249 if (Ty->isBFloatTy()) {
1250 return getOpTypeFloat(Ty->getPrimitiveSizeInBits(), MIRBuilder,
1251 SPIRV::FPEncoding::BFloat16KHR);
1252 } else {
1253 return getOpTypeFloat(Ty->getPrimitiveSizeInBits(), MIRBuilder);
1254 }
1255 }
1256 if (Ty->isVoidTy())
1257 return getOpTypeVoid(MIRBuilder);
1258 if (Ty->isVectorTy()) {
1259 SPIRVTypeInst El =
1260 findSPIRVType(cast<FixedVectorType>(Ty)->getElementType(), MIRBuilder,
1261 AccQual, ExplicitLayoutRequired, EmitIR);
1262 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1263 const auto &STI = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
1264 if (isLongVectorEXT(Ty, STI.isKernel()) &&
1265 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))
1266 return getOpTypeVectorIdEXT(NumElts, El, MIRBuilder);
1267 return getOpTypeVector(NumElts, El, MIRBuilder);
1268 }
1269 if (Ty->isArrayTy()) {
1270 SPIRVTypeInst El = findSPIRVType(Ty->getArrayElementType(), MIRBuilder,
1271 AccQual, ExplicitLayoutRequired, EmitIR);
1272 return getOpTypeArray(Ty->getArrayNumElements(), El, MIRBuilder,
1273 ExplicitLayoutRequired, EmitIR);
1274 }
1275 if (auto SType = dyn_cast<StructType>(Ty)) {
1276 if (SType->isOpaque())
1277 return getOpTypeOpaque(SType, MIRBuilder);
1278
1279 StructOffsetDecorator Decorator = nullptr;
1280 if (ExplicitLayoutRequired) {
1281 Decorator = [&MIRBuilder, SType, this](Register Reg) {
1282 addStructOffsetDecorations(Reg, const_cast<StructType *>(SType),
1283 MIRBuilder);
1284 };
1285 }
1286 return getOpTypeStruct(SType, MIRBuilder, AccQual, std::move(Decorator),
1287 EmitIR);
1288 }
1289 if (auto FType = dyn_cast<FunctionType>(Ty)) {
1290 SPIRVTypeInst RetTy =
1291 findSPIRVType(FType->getReturnType(), MIRBuilder, AccQual,
1292 ExplicitLayoutRequired, EmitIR);
1294 for (const auto &ParamTy : FType->params())
1295 ParamTypes.push_back(findSPIRVType(ParamTy, MIRBuilder, AccQual,
1296 ExplicitLayoutRequired, EmitIR));
1297 return getOpTypeFunction(FType, RetTy, ParamTypes, MIRBuilder);
1298 }
1299
1300 unsigned AddrSpace = typeToAddressSpace(Ty);
1301
1302 // Get access to information about available extensions
1303 const SPIRVSubtarget *ST =
1304 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1305 auto SC = addressSpaceToStorageClass(AddrSpace, *ST);
1306
1307 SPIRVTypeInst SpvElementType = nullptr;
1308 Type *ElemTy = ::getPointeeType(Ty);
1309 if (ElemTy && isa<FunctionType>(ElemTy) &&
1310 !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers))
1311 ElemTy = nullptr;
1312 if (ElemTy)
1313 SpvElementType = getOrCreateSPIRVType(ElemTy, MIRBuilder, AccQual, EmitIR);
1314 else
1315 SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder);
1316
1317 if (!ElemTy) {
1318 ElemTy = Type::getInt8Ty(MIRBuilder.getContext());
1319 }
1320
1321 // If we have forward pointer associated with this type, use its register
1322 // operand to create OpTypePointer.
1323 if (auto It = ForwardPointerTypes.find(Ty); It != ForwardPointerTypes.end()) {
1324 Register Reg = getSPIRVTypeID(It->second);
1325 // TODO: what does getOpTypePointer do?
1326 return getOpTypePointer(SC, SpvElementType, MIRBuilder, Reg);
1327 }
1328
1329 return getOrCreateSPIRVPointerType(ElemTy, MIRBuilder, SC);
1330}
1331
1332SPIRVTypeInst SPIRVGlobalRegistry::restOfCreateSPIRVType(
1333 const Type *Ty, MachineIRBuilder &MIRBuilder,
1334 SPIRV::AccessQualifier::AccessQualifier AccessQual,
1335 bool ExplicitLayoutRequired, bool EmitIR) {
1336 // TODO: Could this create a problem if one requires an explicit layout, and
1337 // the next time it does not?
1338 if (TypesInProcessing.count(Ty) && !isPointerTyOrWrapper(Ty))
1339 return nullptr;
1340 TypesInProcessing.insert(Ty);
1341 SPIRVTypeInst SpirvType = createSPIRVType(Ty, MIRBuilder, AccessQual,
1342 ExplicitLayoutRequired, EmitIR);
1343 TypesInProcessing.erase(Ty);
1344 VRegToTypeMap[&MIRBuilder.getMF()][getSPIRVTypeID(SpirvType)] = SpirvType;
1345
1346 // TODO: We could end up with two SPIR-V types pointing to the same llvm type.
1347 // Is that a problem?
1348 SPIRVToLLVMType[SpirvType] = unifyPtrType(Ty);
1349
1350 if (SpirvType->getOpcode() == SPIRV::OpTypeForwardPointer ||
1351 SpirvType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR ||
1352 findMI(Ty, false, &MIRBuilder.getMF()) || isSpecialOpaqueType(Ty))
1353 return SpirvType;
1354
1355 if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);
1356 ExtTy && isTypedPointerWrapper(ExtTy))
1357 add(ExtTy->getTypeParameter(0), ExtTy->getIntParameter(0), SpirvType);
1358 else if (!isPointerTy(Ty))
1359 add(Ty, ExplicitLayoutRequired, SpirvType);
1360 else if (isTypedPointerTy(Ty))
1361 add(cast<TypedPointerType>(Ty)->getElementType(),
1362 getPointerAddressSpace(Ty), SpirvType);
1363 else
1365 getPointerAddressSpace(Ty), SpirvType);
1366 return SpirvType;
1367}
1368
1371 const MachineFunction *MF) const {
1372 auto t = VRegToTypeMap.find(MF ? MF : CurMF);
1373 if (t != VRegToTypeMap.end()) {
1374 auto tt = t->second.find(VReg);
1375 if (tt != t->second.end())
1376 return tt->second;
1377 }
1378 return nullptr;
1379}
1380
1382 MachineFunction *MF) {
1383 if (!MF)
1384 MF = CurMF;
1385 MachineInstr *Instr = getVRegDef(MF->getRegInfo(), VReg);
1386 return getSPIRVTypeForVReg(Instr->getOperand(1).getReg(), MF);
1387}
1388
1390 const Type *Ty, MachineIRBuilder &MIRBuilder,
1391 SPIRV::AccessQualifier::AccessQualifier AccessQual,
1392 bool ExplicitLayoutRequired, bool EmitIR) {
1393 // SPIR-V doesn't support single-element vectors. Treat <1 x T> as T if the
1394 // SPV_EXT_long_vector extension is not available.
1395 const auto &STI = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
1396 if (auto *FVT = dyn_cast<FixedVectorType>(Ty);
1397 FVT && FVT->getNumElements() == 1 &&
1398 !STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))
1399 return getOrCreateSPIRVType(FVT->getElementType(), MIRBuilder, AccessQual,
1400 ExplicitLayoutRequired, EmitIR);
1401 const MachineFunction *MF = &MIRBuilder.getMF();
1402 Register Reg;
1403 if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);
1404 ExtTy && isTypedPointerWrapper(ExtTy))
1405 Reg = find(ExtTy->getTypeParameter(0), ExtTy->getIntParameter(0), MF);
1406 else if (!isPointerTy(Ty))
1407 Reg = find(Ty = adjustIntTypeByWidth(Ty), ExplicitLayoutRequired, MF);
1408 else if (isTypedPointerTy(Ty))
1409 Reg = find(cast<TypedPointerType>(Ty)->getElementType(),
1410 getPointerAddressSpace(Ty), MF);
1411 else
1412 Reg = find(Type::getInt8Ty(MIRBuilder.getMF().getFunction().getContext()),
1413 getPointerAddressSpace(Ty), MF);
1414 if (Reg.isValid() && !isSpecialOpaqueType(Ty))
1415 return getSPIRVTypeForVReg(Reg);
1416
1417 TypesInProcessing.clear();
1418 SPIRVTypeInst STy = restOfCreateSPIRVType(Ty, MIRBuilder, AccessQual,
1419 ExplicitLayoutRequired, EmitIR);
1420 // Create normal pointer types for the corresponding OpTypeForwardPointers.
1421 for (auto &CU : ForwardPointerTypes) {
1422 // Pointer type themselves do not require an explicit layout. The types
1423 // they pointer to might, but that is taken care of when creating the type.
1424 bool PtrNeedsLayout = false;
1425 const Type *Ty2 = CU.first;
1426 SPIRVTypeInst STy2 = CU.second;
1427 if ((Reg = find(Ty2, PtrNeedsLayout, MF)).isValid())
1428 STy2 = getSPIRVTypeForVReg(Reg);
1429 else
1430 STy2 = restOfCreateSPIRVType(Ty2, MIRBuilder, AccessQual, PtrNeedsLayout,
1431 EmitIR);
1432 if (Ty == Ty2)
1433 STy = STy2;
1434 }
1435 ForwardPointerTypes.clear();
1436 return STy;
1437}
1438
1440 unsigned TypeOpcode) const {
1442 assert(Type && "isScalarOfType VReg has no type assigned");
1443 return Type->getOpcode() == TypeOpcode;
1444}
1445
1447 unsigned TypeOpcode) const {
1449 assert(Type && "isScalarOrVectorOfType VReg has no type assigned");
1450 if (Type->getOpcode() == TypeOpcode)
1451 return true;
1452 if (!isVectorType(Type))
1453 return false;
1454 Register ScalarTypeVReg = Type->getOperand(1).getReg();
1455 SPIRVTypeInst ScalarType = getSPIRVTypeForVReg(ScalarTypeVReg);
1456 return ScalarType->getOpcode() == TypeOpcode;
1457}
1458
1460 switch (Type->getOpcode()) {
1461 case SPIRV::OpTypeImage:
1462 case SPIRV::OpTypeSampler:
1463 case SPIRV::OpTypeSampledImage:
1464 return true;
1465 case SPIRV::OpTypeStruct:
1466 return BlockDecoratedTypes.contains(Type);
1467 default:
1468 return false;
1469 }
1470 return false;
1471}
1472unsigned
1476
1477unsigned
1479 if (!Type)
1480 return 0;
1481 if (isVectorType(Type))
1482 return (Type->getOpcode() == SPIRV::OpTypeVector)
1483 ? static_cast<unsigned>(Type->getOperand(2).getImm())
1484 : foldImm(Type->getOperand(2), &CurMF->getRegInfo());
1485 return 1;
1486}
1487
1490 if (!Type)
1491 return nullptr;
1492 Register ScalarReg = (isVectorType(Type)) ? Type->getOperand(1).getReg()
1493 : Type->getOperand(0).getReg();
1494 SPIRVTypeInst ScalarType = getSPIRVTypeForVReg(ScalarReg);
1495 assert(isScalarOrVectorOfType(Type->getOperand(0).getReg(),
1496 ScalarType->getOpcode()));
1497 return ScalarType;
1498}
1499
1500unsigned
1502 assert(Type && "Invalid Type pointer");
1504 if (ScalarType->getOpcode() == SPIRV::OpTypeInt ||
1505 ScalarType->getOpcode() == SPIRV::OpTypeFloat)
1506 return ScalarType->getOperand(1).getImm();
1507 if (ScalarType->getOpcode() == SPIRV::OpTypeBool)
1508 return 1;
1509 if (ScalarType->getOpcode() == SPIRV::OpTypePointer)
1510 return getPointerSize(); // TODO: does not work for different per AS sizes.
1512 "Attempting to get bit width of non-integer/float/pointer type.");
1513}
1514
1516 SPIRVTypeInst Type) const {
1517 assert(Type && "Invalid Type pointer");
1518 unsigned NumElements = getScalarOrVectorComponentCount(Type);
1520 return ScalarType->getOpcode() == SPIRV::OpTypeInt ||
1521 ScalarType->getOpcode() == SPIRV::OpTypeFloat
1522 ? NumElements * ScalarType->getOperand(1).getImm()
1523 : 0;
1524}
1525
1527 // A function pointer has to keep its function type, which an untyped pointer
1528 // cannot express.
1529 if (ElemType && ElemType->getOpcode() == SPIRV::OpTypeFunction)
1530 return true;
1531 auto It = SPIRVToLLVMType.find(ElemType);
1532 return It != SPIRVToLLVMType.end() && It->second &&
1533 isSpecialOpaqueType(It->second);
1534}
1535
1537 SPIRVTypeInst ElemType, const SPIRVSubtarget &ST) const {
1538 // Shaders keep typed pointers, as this implementation targets compute.
1539 return ST.canUseExtension(SPIRV::Extension::SPV_KHR_untyped_pointers) &&
1540 !ST.isShader() && !shouldKeepTypedPtrType(ElemType);
1541}
1542
1546 return ScalarType && ScalarType->getOpcode() == SPIRV::OpTypeInt ? ScalarType
1547 : nullptr;
1548}
1549
1552 return IntType && IntType->getOperand(2).getImm() != 0;
1553}
1554
1556 return PtrType && PtrType->getOpcode() == SPIRV::OpTypePointer
1557 ? getSPIRVTypeForVReg(PtrType->getOperand(2).getReg())
1558 : nullptr;
1559}
1560
1562 SPIRVTypeInst Type2) const {
1563 if (!Type1 || !Type2)
1564 return false;
1565 // Ignore difference between <1.5 and >=1.5 protocol versions:
1566 // it's valid if either Result Type or Operand is a pointer, and the other
1567 // is a pointer, an integer scalar, or an integer vector.
1568 if (Type1.isPointer() &&
1569 (Type2.isPointer() || retrieveScalarOrVectorIntType(Type2)))
1570 return true;
1571 if (Type2.isPointer() &&
1572 (Type1.isPointer() || retrieveScalarOrVectorIntType(Type1)))
1573 return true;
1574 unsigned Bits1 = getNumScalarOrVectorTotalBitWidth(Type1),
1575 Bits2 = getNumScalarOrVectorTotalBitWidth(Type2);
1576 return Bits1 > 0 && Bits1 == Bits2;
1577}
1578
1579SPIRV::StorageClass::StorageClass
1582 assert(Type && Type.isPointer() && Type->getOperand(1).isImm() &&
1583 "Pointer type is expected");
1585}
1586
1587SPIRV::StorageClass::StorageClass
1589 return static_cast<SPIRV::StorageClass::StorageClass>(
1590 Type->getOperand(1).getImm());
1591}
1592
1594 MachineIRBuilder &MIRBuilder, Type *ElemType,
1595 SPIRV::StorageClass::StorageClass SC, bool IsWritable, bool EmitIr) {
1596 auto Key = SPIRV::irhandle_vkbuffer(ElemType, SC, IsWritable);
1597 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1598 return MI;
1599
1600 bool ExplicitLayoutRequired = storageClassRequiresExplictLayout(SC);
1601 // We need to get the SPIR-V type for the element here, so we can add the
1602 // decoration to it.
1603 auto *T = StructType::create(ElemType);
1604 SPIRVTypeInst BlockType =
1605 getOrCreateSPIRVType(T, MIRBuilder, SPIRV::AccessQualifier::None,
1606 ExplicitLayoutRequired, EmitIr);
1607
1608 buildOpDecorate(BlockType->defs().begin()->getReg(), MIRBuilder,
1609 SPIRV::Decoration::Block, {});
1610 BlockDecoratedTypes.insert(BlockType);
1611
1612 if (!IsWritable) {
1613 buildOpMemberDecorate(BlockType->defs().begin()->getReg(), MIRBuilder,
1614 SPIRV::Decoration::NonWritable, 0, {});
1615 }
1616
1617 SPIRVTypeInst R =
1618 getOrCreateSPIRVPointerTypeInternal(BlockType, MIRBuilder, SC);
1619 add(Key, R);
1620 return R;
1621}
1622
1626 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1627 return MI;
1628 auto *T = Type::getInt8Ty(MIRBuilder.getContext());
1629 SPIRVTypeInst R = getOrCreateSPIRVIntegerType(8, MIRBuilder);
1630 finishCreatingSPIRVType(T, R);
1631 add(Key, R);
1632 return R;
1633}
1634
1636 MachineIRBuilder &MIRBuilder, Type *T) {
1637 const auto SC = SPIRV::StorageClass::PushConstant;
1638
1639 auto Key = SPIRV::irhandle_vkbuffer(T, SC, /* IsWritable= */ false);
1640 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1641 return MI;
1642
1643 // We need to get the SPIR-V type for the element here, so we can add the
1644 // decoration to it.
1646 T, MIRBuilder, SPIRV::AccessQualifier::None,
1647 /* ExplicitLayoutRequired= */ true, /* EmitIr= */ false);
1648
1649 buildOpDecorate(BlockType->defs().begin()->getReg(), MIRBuilder,
1650 SPIRV::Decoration::Block, {});
1651 BlockDecoratedTypes.insert(BlockType);
1652 SPIRVTypeInst R = BlockType;
1653 add(Key, R);
1654 return R;
1655}
1656
1658 MachineIRBuilder &MIRBuilder, const TargetExtType *T, bool EmitIr) {
1659 auto Key = SPIRV::handle(T);
1660 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1661 return MI;
1662
1663 StructType *ST = cast<StructType>(T->getTypeParameter(0));
1664 ArrayRef<uint32_t> Offsets = T->int_params().slice(1);
1665 assert(ST->getNumElements() == Offsets.size());
1666
1667 StructOffsetDecorator Decorator = [&MIRBuilder, &Offsets](Register Reg) {
1668 for (uint32_t I = 0; I < Offsets.size(); ++I) {
1669 buildOpMemberDecorate(Reg, MIRBuilder, SPIRV::Decoration::Offset, I,
1670 {Offsets[I]});
1671 }
1672 };
1673
1674 // We need a new OpTypeStruct instruction because decorations will be
1675 // different from a struct with an explicit layout created from a different
1676 // entry point.
1677 SPIRVTypeInst SPIRVStructType =
1678 getOpTypeStruct(ST, MIRBuilder, SPIRV::AccessQualifier::None,
1679 std::move(Decorator), EmitIr);
1680 add(Key, SPIRVStructType);
1681 return SPIRVStructType;
1682}
1683
1685 const TargetExtType *ExtensionType,
1686 const SPIRV::AccessQualifier::AccessQualifier Qualifier,
1687 MachineIRBuilder &MIRBuilder) {
1688 assert(ExtensionType->getNumTypeParameters() == 1 &&
1689 "SPIR-V image builtin type must have sampled type parameter!");
1690 const SPIRVTypeInst SampledType =
1691 getOrCreateSPIRVType(ExtensionType->getTypeParameter(0), MIRBuilder,
1692 SPIRV::AccessQualifier::ReadWrite, true);
1693 assert((ExtensionType->getNumIntParameters() == 7 ||
1694 ExtensionType->getNumIntParameters() == 6) &&
1695 "Invalid number of parameters for SPIR-V image builtin!");
1696
1697 SPIRV::AccessQualifier::AccessQualifier accessQualifier =
1698 SPIRV::AccessQualifier::None;
1699 if (ExtensionType->getNumIntParameters() == 7) {
1700 accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly
1701 ? SPIRV::AccessQualifier::WriteOnly
1702 : SPIRV::AccessQualifier::AccessQualifier(
1703 ExtensionType->getIntParameter(6));
1704 }
1705
1706 // Create or get an existing type from GlobalRegistry.
1707 SPIRVTypeInst R = getOrCreateOpTypeImage(
1708 MIRBuilder, SampledType,
1709 SPIRV::Dim::Dim(ExtensionType->getIntParameter(0)),
1710 ExtensionType->getIntParameter(1), ExtensionType->getIntParameter(2),
1711 ExtensionType->getIntParameter(3), ExtensionType->getIntParameter(4),
1712 SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(5)),
1713 accessQualifier);
1714 SPIRVToLLVMType[R] = ExtensionType;
1715 return R;
1716}
1717
1718SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypeImage(
1719 MachineIRBuilder &MIRBuilder, SPIRVTypeInst SampledType,
1720 SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed,
1721 uint32_t Multisampled, uint32_t Sampled,
1722 SPIRV::ImageFormat::ImageFormat ImageFormat,
1723 SPIRV::AccessQualifier::AccessQualifier AccessQual) {
1724 auto Key = SPIRV::irhandle_image(SPIRVToLLVMType.lookup(SampledType), Dim,
1725 Depth, Arrayed, Multisampled, Sampled,
1726 ImageFormat, AccessQual);
1727 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1728 return MI;
1729 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1730 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1731 auto MIB =
1732 MIRBuilder.buildInstr(SPIRV::OpTypeImage)
1733 .addDef(createTypeVReg(MIRBuilder))
1734 .addUse(getSPIRVTypeID(SampledType))
1735 .addImm(Dim)
1736 .addImm(Depth) // Depth (whether or not it is a Depth image).
1737 .addImm(Arrayed) // Arrayed.
1738 .addImm(Multisampled) // Multisampled (0 = only single-sample).
1739 .addImm(Sampled) // Sampled (0 = usage known at runtime).
1740 .addImm(ImageFormat);
1741 if (AccessQual != SPIRV::AccessQualifier::None)
1742 MIB.addImm(AccessQual);
1743 return MIB;
1744 });
1745 add(Key, NewMI);
1746 return NewMI;
1747}
1748
1752 const MachineFunction *MF = &MIRBuilder.getMF();
1753 if (const MachineInstr *MI = findMI(Key, MF))
1754 return MI;
1755 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1756 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1757 return MIRBuilder.buildInstr(SPIRV::OpTypeSampler)
1758 .addDef(createTypeVReg(MIRBuilder));
1759 });
1760 add(Key, NewMI);
1761 return NewMI;
1762}
1763
1765 MachineIRBuilder &MIRBuilder,
1766 SPIRV::AccessQualifier::AccessQualifier AccessQual) {
1767 auto Key = SPIRV::irhandle_pipe(AccessQual);
1768 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1769 return MI;
1770 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1771 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1772 return MIRBuilder.buildInstr(SPIRV::OpTypePipe)
1773 .addDef(createTypeVReg(MIRBuilder))
1774 .addImm(AccessQual);
1775 });
1776 add(Key, NewMI);
1777 return NewMI;
1778}
1779
1781 MachineIRBuilder &MIRBuilder) {
1782 auto Key = SPIRV::irhandle_event();
1783 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1784 return MI;
1785 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1786 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1787 return MIRBuilder.buildInstr(SPIRV::OpTypeDeviceEvent)
1788 .addDef(createTypeVReg(MIRBuilder));
1789 });
1790 add(Key, NewMI);
1791 return NewMI;
1792}
1793
1795 SPIRVTypeInst ImageType, MachineIRBuilder &MIRBuilder) {
1797 SPIRVToLLVMType.lookup(MIRBuilder.getMF().getRegInfo().getVRegDef(
1798 ImageType->getOperand(1).getReg())),
1799 ImageType);
1800 if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))
1801 return MI;
1802 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1803 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1804 return MIRBuilder.buildInstr(SPIRV::OpTypeSampledImage)
1805 .addDef(createTypeVReg(MIRBuilder))
1806 .addUse(getSPIRVTypeID(ImageType));
1807 });
1808 add(Key, NewMI);
1809 return NewMI;
1810}
1811
1813 MachineIRBuilder &MIRBuilder, const TargetExtType *ExtensionType,
1814 SPIRVTypeInst ElemType, uint32_t Scope, uint32_t Rows, uint32_t Columns,
1815 uint32_t Use, bool EmitIR) {
1816 if (const MachineInstr *MI =
1817 findMI(ExtensionType, false, &MIRBuilder.getMF()))
1818 return MI;
1819 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1820 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1821 SPIRVTypeInst SpvTypeInt32 =
1822 getOrCreateSPIRVIntegerType(32, MIRBuilder);
1823 const Type *ET = getTypeForSPIRVType(ElemType);
1824 if (ET->isIntegerTy() && ET->getIntegerBitWidth() == 4 &&
1826 .canUseExtension(SPIRV::Extension::SPV_INTEL_int4)) {
1827 MIRBuilder.buildInstr(SPIRV::OpCapability)
1828 .addImm(SPIRV::Capability::Int4CooperativeMatrixINTEL);
1829 }
1830 return MIRBuilder.buildInstr(SPIRV::OpTypeCooperativeMatrixKHR)
1831 .addDef(createTypeVReg(MIRBuilder))
1832 .addUse(getSPIRVTypeID(ElemType))
1833 .addUse(buildConstantInt(Scope, MIRBuilder, SpvTypeInt32, EmitIR))
1834 .addUse(buildConstantInt(Rows, MIRBuilder, SpvTypeInt32, EmitIR))
1835 .addUse(buildConstantInt(Columns, MIRBuilder, SpvTypeInt32, EmitIR))
1836 .addUse(buildConstantInt(Use, MIRBuilder, SpvTypeInt32, EmitIR));
1837 });
1838 add(ExtensionType, false, NewMI);
1839 return NewMI;
1840}
1841
1843 const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode) {
1844 if (const MachineInstr *MI = findMI(Ty, false, &MIRBuilder.getMF()))
1845 return MI;
1846 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1847 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1848 return MIRBuilder.buildInstr(Opcode).addDef(createTypeVReg(MIRBuilder));
1849 });
1850 add(Ty, false, NewMI);
1851 return NewMI;
1852}
1853
1855 const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode,
1857 if (const MachineInstr *MI = findMI(Ty, false, &MIRBuilder.getMF()))
1858 return MI;
1859 Register ResVReg = createTypeVReg(MIRBuilder);
1860 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1861 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1862 MachineInstrBuilder MIB = MIRBuilder.buildInstr(SPIRV::UNKNOWN_type)
1863 .addDef(ResVReg)
1864 .addImm(Opcode);
1865 for (const MachineOperand &Operand : Operands) {
1866 assert((Operand.isReg() || Operand.isImm()) &&
1867 "Expected register or immediate operand");
1868 MIB.add(Operand);
1869 }
1870 return MIB;
1871 });
1872 add(Ty, false, NewMI);
1873 return NewMI;
1874}
1875
1876// Returns nullptr if unable to recognize SPIRV type name
1878 StringRef TypeStr, MachineIRBuilder &MIRBuilder, bool EmitIR,
1879 SPIRV::StorageClass::StorageClass SC,
1880 SPIRV::AccessQualifier::AccessQualifier AQ) {
1881 unsigned VecElts = 0;
1882 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
1883
1884 // Parse strings representing either a SPIR-V or OpenCL builtin type.
1885 if (hasBuiltinTypePrefix(TypeStr))
1887 TypeStr.str(), MIRBuilder.getContext()),
1888 MIRBuilder, AQ, false, true);
1889
1890 // Parse type name in either "typeN" or "type vector[N]" format, where
1891 // N is the number of elements of the vector.
1892 Type *Ty;
1893
1894 Ty = parseBasicTypeName(TypeStr, Ctx);
1895 if (!Ty)
1896 // Unable to recognize SPIRV type name
1897 return nullptr;
1898
1899 SPIRVTypeInst SpirvTy = getOrCreateSPIRVType(Ty, MIRBuilder, AQ, false, true);
1900
1901 // Handle "type*" or "type* vector[N]".
1902 if (TypeStr.consume_front("*"))
1903 SpirvTy = getOrCreateSPIRVPointerType(Ty, MIRBuilder, SC);
1904
1905 // Handle "typeN*" or "type vector[N]*".
1906 bool IsPtrToVec = TypeStr.consume_back("*");
1907
1908 if (TypeStr.consume_front(" vector[")) {
1909 TypeStr = TypeStr.substr(0, TypeStr.find(']'));
1910 }
1911 TypeStr.getAsInteger(10, VecElts);
1912 if (VecElts > 0)
1913 SpirvTy = getOrCreateSPIRVVectorType(SpirvTy, VecElts, MIRBuilder, EmitIR);
1914
1915 if (IsPtrToVec)
1916 SpirvTy = getOrCreateSPIRVPointerType(SpirvTy, MIRBuilder, SC);
1917
1918 return SpirvTy;
1919}
1920
1923 MachineIRBuilder &MIRBuilder) {
1924 return getOrCreateSPIRVType(
1926 MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false, true);
1927}
1928
1930SPIRVGlobalRegistry::finishCreatingSPIRVType(const Type *LLVMTy,
1931 SPIRVTypeInst SpirvType) {
1932 assert(CurMF == SpirvType->getMF());
1933 VRegToTypeMap[CurMF][getSPIRVTypeID(SpirvType)] = SpirvType;
1934 SPIRVToLLVMType[SpirvType] = unifyPtrType(LLVMTy);
1935 return SpirvType;
1936}
1937
1940 const SPIRVInstrInfo &TII,
1941 unsigned SPIRVOPcode, Type *Ty) {
1942 if (const MachineInstr *MI = findMI(Ty, false, CurMF))
1943 return MI;
1944 MachineBasicBlock &DepMBB = I.getMF()->front();
1945 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
1946 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1947 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
1948 auto NewTypeMI = BuildMI(MIRBuilder.getMBB(), *MIRBuilder.getInsertPt(),
1949 MIRBuilder.getDL(), TII.get(SPIRVOPcode))
1950 .addDef(createTypeVReg(CurMF->getRegInfo()))
1951 .addImm(BitWidth);
1952 // Don't add Encoding to FP type
1953 if (!Ty->isFloatTy()) {
1954 return NewTypeMI.addImm(0);
1955 } else {
1956 return NewTypeMI;
1957 }
1958 });
1959 add(Ty, false, NewMI);
1960 return finishCreatingSPIRVType(Ty, NewMI);
1961}
1962
1964 unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII) {
1965 // Maybe adjust bit width to keep DuplicateTracker consistent. Without
1966 // such an adjustment SPIRVGlobalRegistry::getOpTypeInt() could create, for
1967 // example, the same "OpTypeInt 8" type for a series of LLVM integer types
1968 // with number of bits less than 8, causing duplicate type definitions.
1969 if (BitWidth > 1)
1970 BitWidth = adjustOpTypeIntWidth(BitWidth);
1971 Type *LLVMTy = IntegerType::get(CurMF->getFunction().getContext(), BitWidth);
1972 return getOrCreateSPIRVType(BitWidth, I, TII, SPIRV::OpTypeInt, LLVMTy);
1973}
1974
1976 unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII) {
1977 LLVMContext &Ctx = CurMF->getFunction().getContext();
1978 Type *LLVMTy;
1979 switch (BitWidth) {
1980 case 16:
1981 LLVMTy = Type::getHalfTy(Ctx);
1982 break;
1983 case 32:
1984 LLVMTy = Type::getFloatTy(Ctx);
1985 break;
1986 case 64:
1987 LLVMTy = Type::getDoubleTy(Ctx);
1988 break;
1989 default:
1990 llvm_unreachable("Bit width is of unexpected size.");
1991 }
1992 return getOrCreateSPIRVType(BitWidth, I, TII, SPIRV::OpTypeFloat, LLVMTy);
1993}
1994
1997 bool EmitIR) {
1998 return getOrCreateSPIRVType(
1999 IntegerType::get(MIRBuilder.getMF().getFunction().getContext(), 1),
2000 MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false, EmitIR);
2001}
2002
2005 const SPIRVInstrInfo &TII) {
2006 Type *Ty = IntegerType::get(CurMF->getFunction().getContext(), 1);
2007 if (const MachineInstr *MI = findMI(Ty, false, CurMF))
2008 return MI;
2009 MachineBasicBlock &DepMBB = I.getMF()->front();
2010 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
2011 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2012 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
2013 return BuildMI(MIRBuilder.getMBB(), *MIRBuilder.getInsertPt(),
2014 MIRBuilder.getDL(), TII.get(SPIRV::OpTypeBool))
2015 .addDef(createTypeVReg(CurMF->getRegInfo()));
2016 });
2017 add(Ty, false, NewMI);
2018 return finishCreatingSPIRVType(Ty, NewMI);
2019}
2020
2022 SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder,
2023 bool EmitIR) {
2024 return getOrCreateSPIRVType(
2026 NumElements),
2027 MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false, EmitIR);
2028}
2029
2031 SPIRVTypeInst BaseType, unsigned NumElements, MachineInstr &I,
2032 const SPIRVInstrInfo &TII) {
2033 const auto &STI = I.getMF()->getSubtarget<SPIRVSubtarget>();
2034 if (!STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))
2035 // At this point of time all 1-element vectors are resolved. Add assertion
2036 // to fire if anything changes.
2037 assert(NumElements >= 2 &&
2038 "SPIR-V vectors must have at least 2 components");
2040 const_cast<Type *>(getTypeForSPIRVType(BaseType)), NumElements);
2041 if (const MachineInstr *MI = findMI(Ty, false, CurMF))
2042 return MI;
2043 MachineInstr *DepMI =
2044 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(BaseType));
2045 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
2046 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2047 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
2048 // TODO: consider adding non-const accessors to SPIRVTypeInst, which
2049 // would remove the need for the gash casting here.
2050 return const_cast<MachineInstr *>(
2051 static_cast<const MachineInstr *>(getOpTypeVectorImpl(
2052 NumElements, BaseType, MIRBuilder, isLongVectorEXT(Ty))));
2053 });
2054 add(Ty, false, NewMI);
2055 return finishCreatingSPIRVType(Ty, NewMI);
2056}
2057
2059 const Type *BaseType, MachineInstr &I, SPIRV::StorageClass::StorageClass SC,
2060 bool ForceTyped) {
2061 MachineIRBuilder MIRBuilder(I);
2062 return getOrCreateSPIRVPointerType(BaseType, MIRBuilder, SC, ForceTyped);
2063}
2064
2066 const Type *BaseType, MachineIRBuilder &MIRBuilder,
2067 SPIRV::StorageClass::StorageClass SC, bool ForceTyped) {
2068 if (BaseType->isFunctionTy() &&
2069 !cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget())
2070 .canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)) {
2071 const Function &F = MIRBuilder.getMF().getFunction();
2072 F.getContext().diagnose(
2074 "Function used as a data pointer requires "
2075 "SPV_INTEL_function_pointers extension",
2076 DebugLoc(), DS_Error));
2077 }
2078 // TODO: Need to check if EmitIr should always be true.
2079 SPIRVTypeInst SpirvBaseType = getOrCreateSPIRVType(
2080 BaseType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
2082 assert(SpirvBaseType);
2083 return getOrCreateSPIRVPointerTypeInternal(SpirvBaseType, MIRBuilder, SC,
2084 ForceTyped);
2085}
2086
2088 SPIRVTypeInst PtrType, SPIRV::StorageClass::StorageClass SC,
2089 MachineInstr &I) {
2090 [[maybe_unused]] SPIRV::StorageClass::StorageClass OldSC =
2091 getPointerStorageClass(PtrType);
2094
2095 SPIRVTypeInst PointeeType = getPointeeType(PtrType);
2096 MachineIRBuilder MIRBuilder(I);
2097 return getOrCreateSPIRVPointerTypeInternal(PointeeType, MIRBuilder, SC);
2098}
2099
2102 SPIRV::StorageClass::StorageClass SC) {
2103 const Type *LLVMType = getTypeForSPIRVType(BaseType);
2105 SPIRVTypeInst R = getOrCreateSPIRVPointerType(LLVMType, MIRBuilder, SC);
2106 assert(
2107 (R->getOpcode() == SPIRV::OpTypeUntypedPointerKHR ||
2108 getPointeeType(R) == BaseType) &&
2109 "The base type was not correctly laid out for the given storage class.");
2110 return R;
2111}
2112
2113SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVPointerTypeInternal(
2115 SPIRV::StorageClass::StorageClass SC, bool ForceTyped) {
2116 // Check if we should use untyped pointers.
2117 const SPIRVSubtarget &ST =
2118 cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());
2119 if (!ForceTyped && shouldUseUntypedPointer(BaseType, ST))
2120 return getOrCreateSPIRVUntypedPointerType(SC, MIRBuilder);
2121
2122 const Type *PointerElementType = getTypeForSPIRVType(BaseType);
2124 if (const MachineInstr *MI = findMI(PointerElementType, AddressSpace, CurMF))
2125 return MI;
2126 Type *Ty = TypedPointerType::get(const_cast<Type *>(PointerElementType),
2127 AddressSpace);
2128 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2129 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
2130 return BuildMI(MIRBuilder.getMBB(), MIRBuilder.getInsertPt(),
2131 MIRBuilder.getDebugLoc(),
2132 MIRBuilder.getTII().get(SPIRV::OpTypePointer))
2134 .addImm(static_cast<uint32_t>(SC))
2136 });
2137 add(PointerElementType, AddressSpace, NewMI);
2138 return finishCreatingSPIRVType(Ty, NewMI);
2139}
2140
2142 SPIRV::StorageClass::StorageClass SC, MachineIRBuilder &MIRBuilder) {
2143 [[maybe_unused]] const SPIRVSubtarget &ST =
2144 cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());
2145 assert(ST.canUseExtension(SPIRV::Extension::SPV_KHR_untyped_pointers) &&
2146 !ST.isShader() && "Untyped pointers are not available");
2148 // Use STK_UntypedPointer handle keyed by address space only.
2150 if (const MachineInstr *MI = findMI(Handle, CurMF))
2151 return MI;
2152
2153 Type *Ty = PointerType::get(MIRBuilder.getMF().getFunction().getContext(),
2154 AddressSpace);
2155 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2156 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
2157 return BuildMI(MIRBuilder.getMBB(), MIRBuilder.getInsertPt(),
2158 MIRBuilder.getDebugLoc(),
2159 MIRBuilder.getTII().get(SPIRV::OpTypeUntypedPointerKHR))
2160 .addDef(createTypeVReg(CurMF->getRegInfo()))
2161 .addImm(static_cast<uint32_t>(SC));
2162 });
2163 add(Handle, NewMI);
2164 return finishCreatingSPIRVType(Ty, NewMI);
2165}
2166
2168 SPIRVTypeInst SpvType,
2169 const SPIRVInstrInfo &TII) {
2170 UndefValue *UV =
2171 UndefValue::get(const_cast<Type *>(getTypeForSPIRVType(SpvType)));
2172 Register Res = find(UV, CurMF);
2173 if (Res.isValid())
2174 return Res;
2175
2176 LLT LLTy = LLT::scalar(64);
2177 Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);
2178 CurMF->getRegInfo().setRegClass(Res, &SPIRV::iIDRegClass);
2179 assignSPIRVTypeToVReg(SpvType, Res, *CurMF);
2180
2181 MachineInstr *DepMI =
2182 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
2183 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
2184 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2185 MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
2186 auto MIB = BuildMI(MIRBuilder.getMBB(), *MIRBuilder.getInsertPt(),
2187 MIRBuilder.getDL(), TII.get(SPIRV::OpUndef))
2188 .addDef(Res)
2189 .addUse(getSPIRVTypeID(SpvType));
2190 constrainSelectedInstRegOperands(MIB);
2191 return MIB;
2192 });
2193 add(UV, NewMI);
2194 return Res;
2195}
2196
2197const TargetRegisterClass *
2199 unsigned Opcode = SpvType->getOpcode();
2200 switch (Opcode) {
2201 case SPIRV::OpTypeFloat:
2202 return &SPIRV::fIDRegClass;
2203 case SPIRV::OpTypePointer:
2204 case SPIRV::OpTypeUntypedPointerKHR:
2205 return &SPIRV::pIDRegClass;
2206 case SPIRV::OpTypeVector:
2207 case SPIRV::OpTypeVectorIdEXT: {
2209 unsigned ElemOpcode = ElemType ? ElemType->getOpcode() : 0;
2210 if (ElemOpcode == SPIRV::OpTypeFloat)
2211 return &SPIRV::vfIDRegClass;
2212 if (ElemType && ElemType.isPointer())
2213 return &SPIRV::vpIDRegClass;
2214 return &SPIRV::viIDRegClass;
2215 }
2216 }
2217 return &SPIRV::iIDRegClass;
2218}
2219
2220inline unsigned getAS(SPIRVTypeInst SpvType) {
2222 static_cast<SPIRV::StorageClass::StorageClass>(
2223 SpvType->getOperand(1).getImm()));
2224}
2225
2227 if (FloatType->getNumOperands() == 3 &&
2228 FloatType->getOperand(2).getImm() == SPIRV::FPEncoding::BFloat16KHR)
2229 return LLT::bfloat16();
2230 return LLT::floatIEEE(FloatType->getOperand(1).getImm());
2231}
2232
2234 unsigned Opcode = SpvType ? SpvType->getOpcode() : 0;
2235 switch (Opcode) {
2236 case SPIRV::OpTypeFloat:
2237 return getFloatRegType(SpvType);
2238 case SPIRV::OpTypeInt:
2239 case SPIRV::OpTypeBool:
2240 return LLT::scalar(getScalarOrVectorBitWidth(SpvType));
2241 case SPIRV::OpTypePointer:
2242 case SPIRV::OpTypeUntypedPointerKHR:
2243 return LLT::pointer(getAS(SpvType), getPointerSize());
2244 case SPIRV::OpTypeVector:
2245 case SPIRV::OpTypeVectorIdEXT: {
2247 LLT ET;
2248 switch (ElemType ? ElemType->getOpcode() : 0) {
2249 case SPIRV::OpTypePointer:
2250 case SPIRV::OpTypeUntypedPointerKHR:
2251 ET = LLT::pointer(getAS(ElemType), getPointerSize());
2252 break;
2253 case SPIRV::OpTypeFloat:
2254 ET = getFloatRegType(ElemType);
2255 break;
2256 case SPIRV::OpTypeInt:
2257 case SPIRV::OpTypeBool:
2258 ET = LLT::scalar(getScalarOrVectorBitWidth(ElemType));
2259 break;
2260 default:
2261 ET = LLT::scalar(64);
2262 }
2264 return LLT::scalarOrVector(EC, ET);
2265 }
2266 }
2267 return LLT::scalar(64);
2268}
2269
2270// Aliasing list MD contains several scope MD nodes whithin it. Each scope MD
2271// has a selfreference and an extra MD node for aliasing domain and also it
2272// can contain an optional string operand. Domain MD contains a self-reference
2273// with an optional string operand. Here we unfold the list, creating SPIR-V
2274// aliasing instructions.
2275// TODO: add support for an optional string operand.
2277 MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD) {
2278 if (AliasingListMD->getNumOperands() == 0)
2279 return nullptr;
2280 const MachineFunction *MF = &MIRBuilder.getMF();
2281 if (auto L = AliasInstMDMap.find({MF, AliasingListMD});
2282 L != AliasInstMDMap.end())
2283 return L->second;
2284
2286 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2287 for (const MDOperand &MDListOp : AliasingListMD->operands()) {
2288 if (MDNode *ScopeMD = dyn_cast<MDNode>(MDListOp)) {
2289 if (ScopeMD->getNumOperands() < 2)
2290 return nullptr;
2291 MDNode *DomainMD = dyn_cast<MDNode>(ScopeMD->getOperand(1));
2292 if (!DomainMD)
2293 return nullptr;
2294 auto *Domain = [&] {
2295 auto D = AliasInstMDMap.find({MF, DomainMD});
2296 if (D != AliasInstMDMap.end())
2297 return D->second;
2298 const Register Ret = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2299 auto MIB =
2300 MIRBuilder.buildInstr(SPIRV::OpAliasDomainDeclINTEL).addDef(Ret);
2301 return MIB.getInstr();
2302 }();
2303 AliasInstMDMap.insert(
2304 std::make_pair(std::make_pair(MF, DomainMD), Domain));
2305 auto *Scope = [&] {
2306 auto S = AliasInstMDMap.find({MF, ScopeMD});
2307 if (S != AliasInstMDMap.end())
2308 return S->second;
2309 const Register Ret = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2310 auto MIB = MIRBuilder.buildInstr(SPIRV::OpAliasScopeDeclINTEL)
2311 .addDef(Ret)
2312 .addUse(Domain->getOperand(0).getReg());
2313 return MIB.getInstr();
2314 }();
2315 AliasInstMDMap.insert(std::make_pair(std::make_pair(MF, ScopeMD), Scope));
2316 ScopeList.push_back(Scope);
2317 }
2318 }
2319
2320 const Register Ret = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2321 auto MIB =
2322 MIRBuilder.buildInstr(SPIRV::OpAliasScopeListDeclINTEL).addDef(Ret);
2323 for (auto *Scope : ScopeList)
2324 MIB.addUse(Scope->getOperand(0).getReg());
2325 auto List = MIB.getInstr();
2326 AliasInstMDMap.insert(
2327 std::make_pair(std::make_pair(MF, AliasingListMD), List));
2328 return List;
2329}
2330
2332 Register Reg, MachineIRBuilder &MIRBuilder, uint32_t Dec,
2333 const MDNode *AliasingListMD) {
2334 MachineInstr *AliasList =
2335 getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD);
2336 if (!AliasList)
2337 return;
2338 MIRBuilder.buildInstr(SPIRV::OpDecorateId)
2339 .addUse(Reg)
2340 .addImm(Dec)
2341 .addUse(AliasList->getOperand(0).getReg());
2342}
2344 bool DeleteOld) {
2345 Old->replaceAllUsesWith(New);
2346 updateIfExistDeducedElementType(Old, New, DeleteOld);
2347 updateIfExistAssignPtrTypeInstr(Old, New, DeleteOld);
2348}
2349
2351 bool CanUseAnyVectorRank) {
2352 Value *OfType = getNormalizedPoisonValue(Ty, CanUseAnyVectorRank);
2353 CallInst *AssignCI = nullptr;
2354 if (Arg->getType()->isAggregateType() && Ty->isAggregateType() &&
2355 allowEmitFakeUse(Arg)) {
2356 LLVMContext &Ctx = Arg->getContext();
2359 MDString::get(Ctx, Arg->getName())};
2360 B.CreateIntrinsic(Intrinsic::spv_value_md,
2361 {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))});
2362 AssignCI = B.CreateIntrinsicWithoutFolding(Intrinsic::fake_use, {Arg});
2363 } else {
2364 AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()},
2365 OfType, Arg, {}, B);
2366 }
2367 addAssignPtrTypeInstr(Arg, AssignCI);
2368}
2369
2371 Value *Arg) {
2372 Value *OfType = PoisonValue::get(ElemTy);
2373 CallInst *AssignPtrTyCI = findAssignPtrTypeInstr(Arg);
2374 Function *CurrF =
2375 B.GetInsertBlock() ? B.GetInsertBlock()->getParent() : nullptr;
2376 if (AssignPtrTyCI == nullptr ||
2377 AssignPtrTyCI->getParent()->getParent() != CurrF) {
2378 AssignPtrTyCI = buildIntrWithMD(
2379 Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg,
2380 {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
2381 addDeducedElementType(AssignPtrTyCI, ElemTy);
2383 addAssignPtrTypeInstr(Arg, AssignPtrTyCI);
2384 } else {
2385 updateAssignType(AssignPtrTyCI, Arg, OfType);
2386 }
2387}
2388
2390 Value *OfType) {
2391 AssignCI->setArgOperand(1, buildMD(OfType));
2392 if (cast<IntrinsicInst>(AssignCI)->getIntrinsicID() !=
2393 Intrinsic::spv_assign_ptr_type)
2394 return;
2395
2396 // update association with the pointee type
2397 Type *ElemTy = OfType->getType();
2398 addDeducedElementType(AssignCI, ElemTy);
2400}
2401
2402void SPIRVGlobalRegistry::addStructOffsetDecorations(
2403 Register Reg, StructType *Ty, MachineIRBuilder &MIRBuilder) {
2404 ArrayRef<TypeSize> Offsets = DL.getStructLayout(Ty)->getMemberOffsets();
2405 for (uint32_t I = 0; I < Ty->getNumElements(); ++I) {
2406 buildOpMemberDecorate(Reg, MIRBuilder, SPIRV::Decoration::Offset, I,
2407 {static_cast<uint32_t>(Offsets[I])});
2408 }
2409}
2410
2411void SPIRVGlobalRegistry::addArrayStrideDecorations(
2412 Register Reg, Type *ElementType, MachineIRBuilder &MIRBuilder) {
2413 uint32_t SizeInBytes = DL.getTypeAllocSize(ElementType);
2414 buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::ArrayStride,
2415 {SizeInBytes});
2416}
static unsigned getIntrinsicID(const SDNode *N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static constexpr Value * getValue(Ty &ValueOrUse)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
MachineRegisterInfo & MRI
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI Fold Operands
static bool storageClassRequiresExplictLayout(SPIRV::StorageClass::StorageClass SC)
static Register createTypeVReg(MachineRegisterInfo &MRI)
static bool allowEmitFakeUse(const Value *Arg)
static unsigned typeToAddressSpace(const Type *Ty)
static LLT getFloatRegType(SPIRVTypeInst FloatType)
unsigned getAS(SPIRVTypeInst SpvType)
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
APInt bitcastToAPInt() const
Definition APFloat.h:1475
bool isPosZero() const
Definition APFloat.h:1594
Class for arbitrary precision integers.
Definition APInt.h:78
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
Class to represent array types.
uint64_t getNumElements() const
Type * getElementType() const
void setArgOperand(unsigned i, Value *v)
This class represents a function call, abstracting a target machine's calling convention.
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
const APFloat & getValue() const
Definition Constants.h:464
const APFloat & getValueAPF() const
Definition Constants.h:463
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
Return an anonymous struct that has the specified elements.
Definition Constants.h:643
static LLVM_ABI ConstantTargetNone * get(TargetExtType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
Definition Constant.h:44
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition Constant.h:65
LLVM_ABI const APInt & getUniqueInteger() const
If C is a constant integer then return its value, otherwise C must be a vector of constant integers,...
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
A debug info location.
Definition DebugLoc.h:126
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
Class to represent fixed width SIMD vectors.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
Class to represent function types.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this GlobalObject.
Module * getParent()
Get the module that this global value is contained inside of...
@ ExternalLinkage
Externally visible function.
Definition GlobalValue.h:53
Type * getValueType() const
MaybeAlign getAlign() const
Returns the alignment of the given variable.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
static constexpr LLT bfloat16()
static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy)
static LLT floatIEEE(unsigned SizeInBits)
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition MCInstrInfo.h:89
Metadata node.
Definition Metadata.h:1087
ArrayRef< MDOperand > operands() const
Definition Metadata.h:1441
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1588
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1449
Tracking metadata reference owned by Metadata.
Definition Metadata.h:908
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:595
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1534
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
MachineInstrBundleIterator< MachineInstr > iterator
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
LLVMContext & getContext() const
const TargetInstrInfo & getTII()
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
const DebugLoc & getDL()
Getter for DebugLoc.
MachineFunction & getMF()
Getter for the function we currently build.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineIRBuilderState & getState()
Getter for the State.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
mop_range defs()
Returns all explicit operands that are register definitions.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI void insert(mop_iterator InsertBefore, ArrayRef< MachineOperand > Ops)
Inserts Ops BEFORE It. Can untie/retie tied operands.
mop_range uses()
Returns all operands which may be register uses.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:105
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg, bool CanUseAnyVectorRank)
SPIRVTypeInst getImageType(const TargetExtType *ExtensionType, const SPIRV::AccessQualifier::AccessQualifier Qualifier, MachineIRBuilder &MIRBuilder)
bool isScalarOrVectorSigned(SPIRVTypeInst Type) const
void addAssignPtrTypeInstr(Value *Val, CallInst *AssignPtrTyCI)
SPIRVTypeInst getOrCreateOpTypeSampledImage(SPIRVTypeInst ImageType, MachineIRBuilder &MIRBuilder)
unsigned getNumScalarOrVectorTotalBitWidth(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateUnknownType(const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode, const ArrayRef< MachineOperand > Operands)
void assignSPIRVTypeToVReg(SPIRVTypeInst Type, Register VReg, const MachineFunction &MF)
SPIRVTypeInst getOrCreateOpTypeFunctionWithArgs(const Type *Ty, SPIRVTypeInst RetType, const SmallVectorImpl< SPIRVTypeInst > &ArgTypes, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg)
const TargetRegisterClass * getRegClass(SPIRVTypeInst SpvType) const
MachineInstr * getOrAddMemAliasingINTELInst(MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD)
unsigned getScalarOrVectorBitWidth(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
SPIRVTypeInst getOrCreateSPIRVTypeByName(StringRef TypeStr, MachineIRBuilder &MIRBuilder, bool EmitIR, SPIRV::StorageClass::StorageClass SC=SPIRV::StorageClass::Function, SPIRV::AccessQualifier::AccessQualifier AQ=SPIRV::AccessQualifier::ReadWrite)
Register buildGlobalVariable(Register Reg, SPIRVTypeInst BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)
SPIRVTypeInst assignIntTypeToVReg(unsigned BitWidth, Register VReg, MachineInstr &I, const SPIRVInstrInfo &TII)
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
void replaceAllUsesWith(Value *Old, Value *New, bool DeleteOld=true)
SPIRVTypeInst getOrCreateOpTypeByOpcode(const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode)
unsigned getScalarOrVectorComponentCount(Register VReg) const
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
bool isBitcastCompatible(SPIRVTypeInst Type1, SPIRVTypeInst Type2) const
void addDeducedElementType(Value *Val, Type *Ty)
bool shouldKeepTypedPtrType(SPIRVTypeInst ElemType) const
SPIRVTypeInst getOrCreatePaddingType(MachineIRBuilder &MIRBuilder)
Register getOrCreateConstFP(APFloat Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
LLT getRegType(SPIRVTypeInst SpvType) const
SPIRVTypeInst getOpTypeVoid(MachineIRBuilder &MIRBuilder)
void invalidateMachineInstr(MachineInstr *MI)
bool isResourceType(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder, bool EmitIR)
void updateIfExistDeducedElementType(Value *OldVal, Value *NewVal, bool DeleteOld)
bool isScalarOfType(Register VReg, unsigned TypeOpcode) const
Register getSPIRVTypeID(SPIRVTypeInst SpirvType) const
Register getOrCreateConstInt(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
SPIRVTypeInst retrieveScalarOrVectorIntType(SPIRVTypeInst Type) const
Register getOrCreateGlobalVariableWithBinding(SPIRVTypeInst VarType, uint32_t Set, uint32_t Binding, StringRef Name, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst getOrCreateOpTypeCoopMatr(MachineIRBuilder &MIRBuilder, const TargetExtType *ExtensionType, SPIRVTypeInst ElemType, uint32_t Scope, uint32_t Rows, uint32_t Columns, uint32_t Use, bool EmitIR)
SPIRVTypeInst changePointerStorageClass(SPIRVTypeInst PtrType, SPIRV::StorageClass::StorageClass SC, MachineInstr &I)
Register getOrCreateConstVector(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Register buildConstantFP(APFloat Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType=nullptr)
SPIRVTypeInst getOrCreateOpTypePipe(MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccQual)
void addGlobalObject(const Value *V, const MachineFunction *MF, Register R)
SPIRVTypeInst getScalarOrVectorComponentType(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateSPIRVFloatType(unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII)
SPIRVTypeInst getOrCreateVulkanBufferType(MachineIRBuilder &MIRBuilder, Type *ElemType, SPIRV::StorageClass::StorageClass SC, bool IsWritable, bool EmitIr=false)
SPIRVTypeInst getPointeeType(SPIRVTypeInst PtrType)
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
Register getOrCreateConsIntVector(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType, bool EmitIR)
void updateIfExistAssignPtrTypeInstr(Value *OldVal, Value *NewVal, bool DeleteOld)
SPIRVTypeInst assignTypeToVReg(const Type *Type, Register VReg, MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
SPIRVTypeInst getOrCreateLayoutType(MachineIRBuilder &MIRBuilder, const TargetExtType *T, bool EmitIr=false)
Register createConstInt(const ConstantInt *CI, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull)
Register getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType)
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
SPIRVTypeInst getOrCreateSPIRVUntypedPointerType(SPIRV::StorageClass::StorageClass SC, MachineIRBuilder &MIRBuilder)
Register getOrCreateUndef(MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
SPIRVTypeInst getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder)
void buildMemAliasingOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, uint32_t Dec, const MDNode *GVarMD)
SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const
bool shouldUseUntypedPointer(SPIRVTypeInst ElemType, const SPIRVSubtarget &ST) const
Register buildConstantSampler(Register Res, unsigned AddrMode, unsigned Param, unsigned FilerMode, MachineIRBuilder &MIRBuilder)
void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType)
CallInst * findAssignPtrTypeInstr(const Value *Val)
Register buildConstantInt(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType, bool EmitIR, bool ZeroAsNull=true)
SPIRVTypeInst getOrCreateVulkanPushConstantType(MachineIRBuilder &MIRBuilder, Type *ElemType)
Register createConstFP(const ConstantFP *CF, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull)
SPIRVTypeInst getOrCreateOpTypeDeviceEvent(MachineIRBuilder &MIRBuilder)
const MachineInstr * findMI(SPIRV::IRHandle Handle, const MachineFunction *MF)
bool erase(const MachineInstr *MI)
bool add(SPIRV::IRHandle Handle, const MachineInstr *MI)
Register find(SPIRV::IRHandle Handle, const MachineFunction *MF)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
Definition StringRef.h:691
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition StringRef.h:490
std::string str() const
Get the contents as an std::string.
Definition StringRef.h:222
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition StringRef.h:661
Class to represent struct types.
ArrayRef< Type * > elements() const
static LLVM_ABI StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition Type.cpp:657
bool isPacked() const
unsigned getNumElements() const
Random access to the elements.
bool hasName() const
Return true if this is a named struct that has a non-empty name.
LLVM_ABI StringRef getName() const
Return the name for this struct type if it has an identity.
Definition Type.cpp:731
Class to represent target extensions types, which are generally unintrospectable from target-independ...
unsigned getNumIntParameters() const
Type * getTypeParameter(unsigned i) const
unsigned getNumTypeParameters() const
unsigned getIntParameter(unsigned i) const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:295
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isArrayTy() const
True if this is an instance of ArrayType.
Definition Type.h:274
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
Type * getArrayElementType() const
Definition Type.h:427
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
LLVM_ABI uint64_t getArrayNumElements() const
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
Definition Type.h:167
bool isFP128Ty() const
Return true if this is 'fp128'.
Definition Type.h:164
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:292
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition Type.h:314
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
Definition Type.cpp:272
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:271
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:269
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
static LLVM_ABI TypedPointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
'undef' values are things that do not have specified contents.
Definition Constants.h:1657
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
static ConstantAsMetadata * getConstant(Value *C)
Definition Metadata.h:493
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
bool hasName() const
Definition Value.h:263
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
Type * getElementType() const
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
IteratorT begin() const
#define UINT64_MAX
Definition DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
IRHandle handle(const Type *Ty)
IRHandle irhandle_sampled_image(const Type *SampledTy, const MachineInstr *ImageTy)
IRHandle irhandle_padding()
IRHandle irhandle_vkbuffer(const Type *ElementType, StorageClass::StorageClass SC, bool IsWriteable)
IRHandle irhandle_untyped_pointer(unsigned AddressSpace)
IRHandle irhandle_sampler()
TargetExtType * parseBuiltinTypeNameToTargetExtType(std::string TypeName, LLVMContext &Context)
Translates a string representing a SPIR-V or OpenCL builtin type to a TargetExtType that can be furth...
IRHandle irhandle_event()
SPIRVTypeInst lowerBuiltinType(const Type *OpaqueType, SPIRV::AccessQualifier::AccessQualifier AccessQual, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
IRHandle irhandle_pipe(uint8_t AQ)
IRHandle irhandle_image(const Type *SampledTy, unsigned Dim, unsigned Depth, unsigned Arrayed, unsigned MS, unsigned Sampled, unsigned ImageFormat, unsigned AQ=0)
This is an optimization pass for GlobalISel generic memory operations.
void addStringImm(StringRef Str, MCInst &Inst)
bool isTypedPointerWrapper(const TargetExtType *ExtTy)
Definition SPIRVUtils.h:414
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
unsigned getPointerAddressSpace(const Type *T)
Definition SPIRVUtils.h:385
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
CallInst * buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef< Type * > Types, Value *Arg, Value *Arg2, ArrayRef< Constant * > Imms, IRBuilder<> &B)
bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType, uint64_t &TotalSize)
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, ArrayRef< uint32_t > DecArgs, StringRef StrImm)
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
Definition SPIRVUtils.h:238
bool isVectorType(SPIRVTypeInst SPVTy)
bool getSpirvBuiltInIdByName(llvm::StringRef Name, SPIRV::BuiltIn::BuiltIn &BI)
MetadataAsValue * buildMD(Value *Arg)
Definition SPIRVUtils.h:546
bool isTypedPointerTy(const Type *T)
Definition SPIRVUtils.h:363
void buildOpName(Register Target, StringRef Name, MachineIRBuilder &MIRBuilder)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Type * getTypedPointerWrapper(Type *ElemTy, unsigned AS)
Definition SPIRVUtils.h:409
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
Type * toTypedPointer(Type *Ty)
Definition SPIRVUtils.h:469
bool isSpecialOpaqueType(const Type *Ty)
bool isPointerTy(const Type *T)
Definition SPIRVUtils.h:373
MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB)
const Type * unifyPtrType(const Type *Ty)
Definition SPIRVUtils.h:496
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
std::function< void(Register)> StructOffsetDecorator
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)
void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder, const MDNode *GVarMD, const SPIRVSubtarget &ST)
int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
bool isLongVectorEXT(const Type *Ty, bool CanUseVector16=true)
Definition SPIRVUtils.h:519
Type * parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx)
DWARFExpression::Operation Op
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
bool hasBuiltinTypePrefix(StringRef Name)
void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, uint32_t Member, ArrayRef< uint32_t > DecArgs, StringRef StrImm)
bool isPointerTyOrWrapper(const Type *Ty)
Definition SPIRVUtils.h:421
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
PoisonValue * getNormalizedPoisonValue(Type *Ty, bool CanUseAnyVectorRank)
Definition SPIRVUtils.h:541
MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition Alignment.h:135