LLVM 24.0.0git
SPIRVModuleAnalysis.cpp
Go to the documentation of this file.
1//===- SPIRVModuleAnalysis.cpp - analysis of global instrs & regs - C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The analysis collects instructions that should be output at the module level
10// and performs the global register numbering.
11//
12// The results of this analysis are used in AsmPrinter to rename registers
13// globally and to output required instructions at the module level.
14//
15//===----------------------------------------------------------------------===//
16
17// TODO: Per LLVM best practices, the report_fatal_error (deprecated) /
18// ReportFatalUsageError calls in this file should be replaced with the
19// Diagnostic infrastructure (e.g. the reportUnsupported function below).
20
21#include "SPIRVModuleAnalysis.h"
24#include "SPIRV.h"
25#include "SPIRVSubtarget.h"
26#include "SPIRVTargetMachine.h"
27#include "SPIRVUtils.h"
28#include "llvm/ADT/STLExtras.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "spirv-module-analysis"
37
39
41 "SPIRV module analysis", true, true)
42
44 const Function &Func = MI.getMF()->getFunction();
45 Func.getContext().diagnose(
46 DiagnosticInfoUnsupported(Func, Msg, MI.getDebugLoc()));
47}
48
49// Retrieve an unsigned from an MDNode with a list of them as operands.
50static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex,
51 unsigned DefaultVal = 0) {
52 if (MdNode && OpIndex < MdNode->getNumOperands()) {
53 const auto &Op = MdNode->getOperand(OpIndex);
54 return mdconst::extract<ConstantInt>(Op)->getZExtValue();
55 }
56 return DefaultVal;
57}
58
59// Return true if Cap should be avoided when another option enables the feature.
61 SPIRV::Capability::Capability Cap) {
62 return Cap == (ST.isShader() ? SPIRV::Capability::Kernel
63 : SPIRV::Capability::Shader) ||
64 is_contained(ST.getCLOpts().avoid_capabilities, Cap);
65}
66
68getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
69 unsigned i, const SPIRVSubtarget &ST,
71 VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, i);
72 VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, i);
73 VersionTuple SPIRVVersion = ST.getSPIRVVersion();
74 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;
75 bool MaxVerOK =
76 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;
77 bool VerOK = MinVerOK && MaxVerOK;
79 ExtensionList ReqExts = getSymbolicOperandExtensions(Category, i);
80 // A met minimum version makes the extension unnecessary.
81 if (VerOK && !ReqMinVer.empty())
82 ReqExts.clear();
83 if (ReqCaps.empty()) {
84 if (ReqExts.empty()) {
85 if (VerOK)
86 return {true, {}, {}, ReqMinVer, ReqMaxVer};
87 return {false, {}, {}, VersionTuple(), VersionTuple()};
88 }
89 } else if (VerOK) {
90 if (ReqCaps.size() == 1) {
91 auto Cap = ReqCaps[0];
92 if (Reqs.isCapabilityAvailable(Cap)) {
94 SPIRV::OperandCategory::CapabilityOperand, Cap));
95 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
96 }
97 } else {
98 // By SPIR-V specification: "If an instruction, enumerant, or other
99 // feature specifies multiple enabling capabilities, only one such
100 // capability needs to be declared to use the feature." However, one
101 // capability may be preferred over another. We use command line
102 // argument(s) and AvoidCapabilities to avoid selection of certain
103 // capabilities if there are other options.
104 CapabilityList UseCaps;
105 for (auto Cap : ReqCaps)
106 if (Reqs.isCapabilityAvailable(Cap))
107 UseCaps.push_back(Cap);
108 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
109 auto Cap = UseCaps[i];
110 if (i == Sz - 1 || !shouldAvoidCapability(ST, Cap)) {
112 SPIRV::OperandCategory::CapabilityOperand, Cap));
113 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
114 }
115 }
116 }
117 }
118 // If there are no capabilities, or we can't satisfy the version or
119 // capability requirements, use the list of extensions (if the subtarget
120 // can handle them all). Without an extension the version must hold. An
121 // extension stands in for an unmet version.
122 if ((!ReqExts.empty() || VerOK) &&
123 llvm::all_of(ReqExts, [&ST](const SPIRV::Extension::Extension &Ext) {
124 return ST.canUseExtension(Ext);
125 })) {
126 if (VerOK)
127 return {true, {}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
128 return {true, {}, std::move(ReqExts), VersionTuple(), VersionTuple()};
129 }
130 return {false, {}, {}, VersionTuple(), VersionTuple()};
131}
132
133void SPIRVModuleAnalysisImpl::setBaseInfo(const Module &M) {
134 MAI.MaxID = 0;
135 for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)
136 MAI.MS[i].clear();
137 MAI.RegisterAliasTable.clear();
138 MAI.InstrsToDelete.clear();
139 MAI.MBBNames.clear();
140 MAI.GlobalObjMap.clear();
141 MAI.GlobalVarList.clear();
142 MAI.ExtInstSetMap.clear();
143 MAI.InstrAuxDataRecords.clear();
144 MAI.DebugValueMIs.clear();
145 MAI.Reqs.clear();
146 MAI.Reqs.initAvailableCapabilities(*ST);
147
148 // TODO: determine memory model and source language from the configuratoin.
149 if (auto MemModel = M.getNamedMetadata("spirv.MemoryModel")) {
150 auto MemMD = MemModel->getOperand(0);
151 MAI.Addr = static_cast<SPIRV::AddressingModel::AddressingModel>(
152 getMetadataUInt(MemMD, 0));
153 MAI.Mem =
154 static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MemMD, 1));
155 } else {
156 // TODO: Add support for VulkanMemoryModel.
157 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
158 : SPIRV::MemoryModel::OpenCL;
159 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
160 unsigned PtrSize = ST->getPointerSize();
161 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32
162 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64
163 : SPIRV::AddressingModel::Logical;
164 } else {
165 // TODO: Add support for PhysicalStorageBufferAddress.
166 MAI.Addr = SPIRV::AddressingModel::Logical;
167 }
168 }
169 // Get the OpenCL version number from metadata.
170 // TODO: support other source languages.
171 if (auto VerNode = M.getNamedMetadata("opencl.ocl.version")) {
172 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;
173 // Construct version literal in accordance with SPIRV-LLVM-Translator.
174 // TODO: support multiple OCL version metadata.
175 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");
176 auto VersionMD = VerNode->getOperand(0);
177 unsigned MajorNum = getMetadataUInt(VersionMD, 0, 2);
178 unsigned MinorNum = getMetadataUInt(VersionMD, 1);
179 unsigned RevNum = getMetadataUInt(VersionMD, 2);
180 // Prevent Major part of OpenCL version to be 0
181 MAI.SrcLangVersion =
182 (std::max(1U, MajorNum) * 100 + MinorNum) * 1000 + RevNum;
183 // When opencl.cxx.version is also present, validate compatibility
184 // and use C++ for OpenCL as source language with the C++ version.
185 if (auto *CxxVerNode = M.getNamedMetadata("opencl.cxx.version")) {
186 assert(CxxVerNode->getNumOperands() > 0 && "Invalid SPIR");
187 auto *CxxMD = CxxVerNode->getOperand(0);
188 unsigned CxxVer =
189 (getMetadataUInt(CxxMD, 0) * 100 + getMetadataUInt(CxxMD, 1)) * 1000 +
190 getMetadataUInt(CxxMD, 2);
191 if ((MAI.SrcLangVersion == 200000 && CxxVer == 100000) ||
192 (MAI.SrcLangVersion == 300000 && CxxVer == 202100000)) {
193 MAI.SrcLang = SPIRV::SourceLanguage::CPP_for_OpenCL;
194 MAI.SrcLangVersion = CxxVer;
195 } else {
197 "opencl cxx version is not compatible with opencl c version!");
198 }
199 }
200 } else {
201 // If there is no information about OpenCL version we are forced to generate
202 // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling
203 // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV
204 // Translator avoids potential issues with run-times in a similar manner.
205 if (!ST->isShader()) {
206 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
207 MAI.SrcLangVersion = 100000;
208 } else {
209 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;
210 MAI.SrcLangVersion = 0;
211 }
212 }
213
214 if (auto ExtNode = M.getNamedMetadata("opencl.used.extensions")) {
215 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {
216 MDNode *MD = ExtNode->getOperand(I);
217 if (!MD || MD->getNumOperands() == 0)
218 continue;
219 for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J)
220 MAI.SrcExt.insert(cast<MDString>(MD->getOperand(J))->getString());
221 }
222 }
223
224 // Update required capabilities for this memory model, addressing model and
225 // source language.
226 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::MemoryModelOperand,
227 MAI.Mem, *ST);
228 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::SourceLanguageOperand,
229 MAI.SrcLang, *ST);
230 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand,
231 MAI.Addr, *ST);
232
233 if (MAI.Mem == SPIRV::MemoryModel::VulkanKHR)
234 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_vulkan_memory_model);
235
236 if (!ST->isShader()) {
237 // TODO: check if it's required by default.
238 MAI.ExtInstSetMap[static_cast<unsigned>(
239 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
240 }
241}
242
243// Appends the signature of the decoration instructions that decorate R to
244// Signature.
246 InstrSignature &Signature) {
247 for (MachineInstr &UseMI : MRI.use_instructions(R)) {
248 // We don't handle OpDecorateId because getting the register alias for the
249 // ID can cause problems, and we do not need it for now.
250 if (UseMI.getOpcode() != SPIRV::OpDecorate &&
251 UseMI.getOpcode() != SPIRV::OpMemberDecorate)
252 continue;
253
254 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {
255 const MachineOperand &MO = UseMI.getOperand(I);
256 if (MO.isReg())
257 continue;
258 Signature.push_back(hash_value(MO));
259 }
260 }
261}
262
263// Returns a representation of an instruction as a vector of MachineOperand
264// hash values, see llvm::hash_value(const MachineOperand &MO) for details.
265// This creates a signature of the instruction with the same content
266// that MachineOperand::isIdenticalTo uses for comparison.
269 bool UseDefReg) {
270 Register DefReg;
271 InstrSignature Signature{MI.getOpcode()};
272 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
273 // The only decorations that can be applied more than once to a given <id>
274 // or structure member are FuncParamAttr (38), UserSemantic (5635),
275 // CacheControlLoadINTEL (6442), and CacheControlStoreINTEL (6443). For all
276 // the rest of decorations, we will only add to the signature the Opcode,
277 // the id to which it applies, and the decoration id, disregarding any
278 // decoration flags. This will ensure that any subsequent decoration with
279 // the same id will be deemed as a duplicate. Then, at the call site, we
280 // will be able to handle duplicates in the best way.
281 unsigned Opcode = MI.getOpcode();
282 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {
283 unsigned DecorationID = MI.getOperand(1).getImm();
284 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&
285 DecorationID != SPIRV::Decoration::UserSemantic &&
286 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&
287 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)
288 continue;
289 }
290 const MachineOperand &MO = MI.getOperand(i);
291 size_t h;
292 if (MO.isReg()) {
293 if (!UseDefReg && MO.isDef()) {
294 assert(!DefReg.isValid() && "Multiple def registers.");
295 DefReg = MO.getReg();
296 continue;
297 }
298 Register RegAlias = MAI.getRegisterAlias(MI.getMF(), MO.getReg());
299 if (!RegAlias.isValid()) {
300 LLVM_DEBUG({
301 dbgs() << "Unexpectedly, no global id found for the operand ";
302 MO.print(dbgs());
303 dbgs() << "\nInstruction: ";
304 MI.print(dbgs());
305 dbgs() << "\n";
306 });
307 report_fatal_error("All v-regs must have been mapped to global id's");
308 }
309 // mimic llvm::hash_value(const MachineOperand &MO)
310 h = hash_combine(MO.getType(), (unsigned)RegAlias, MO.getSubReg(),
311 MO.isDef());
312 } else {
313 h = hash_value(MO);
314 }
315 Signature.push_back(h);
316 }
317
318 if (DefReg.isValid()) {
319 // Decorations change the semantics of the current instruction. So two
320 // identical instruction with different decorations cannot be merged. That
321 // is why we add the decorations to the signature.
322 appendDecorationsForReg(MI.getMF()->getRegInfo(), DefReg, Signature);
323 }
324 return Signature;
325}
326
327// Operand index of Invoke in device enqueue instructions, 0 if none.
328static unsigned getInvokeOperandIdx(unsigned Opcode) {
329 switch (Opcode) {
330 case SPIRV::OpEnqueueKernel:
331 return 8;
332 case SPIRV::OpGetKernelNDrangeSubGroupCount:
333 case SPIRV::OpGetKernelNDrangeMaxSubGroupSize:
334 return 3;
335 case SPIRV::OpGetKernelWorkGroupSize:
336 case SPIRV::OpGetKernelPreferredWorkGroupSizeMultiple:
337 return 2;
338 default:
339 return 0;
340 }
341}
342
343bool SPIRVModuleAnalysisImpl::isDeclSection(const MachineRegisterInfo &MRI,
344 const MachineInstr &MI) {
345 unsigned Opcode = MI.getOpcode();
346 switch (Opcode) {
347 case SPIRV::OpTypeForwardPointer:
348 // omit now, collect later
349 return false;
350 case SPIRV::OpVariable:
351 case SPIRV::OpUntypedVariableKHR:
352 return static_cast<SPIRV::StorageClass::StorageClass>(
353 MI.getOperand(2).getImm()) != SPIRV::StorageClass::Function;
354 case SPIRV::OpFunction:
355 case SPIRV::OpFunctionParameter:
356 return true;
357 }
358 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {
359 // The OpUndef may be a placeholder for a function reference recorded by
360 // selectGlobalValue. Skip emitting it if any user consumes it as a
361 // function-pointer-like operand (OpConstantFunctionPointerINTEL operand 2,
362 // or the Invoke operand of a device enqueue instruction). The rewrite
363 // happens in visitFunPtrUse, which aliases the OpUndef's vreg to the
364 // function's global <id>.
365 Register DefReg = MI.getOperand(0).getReg();
366 if (GR->getFunctionDefinitionByUse(&MI.getOperand(0))) {
367 for (MachineInstr &UseMI : MRI.use_instructions(DefReg)) {
368 unsigned UseOp = UseMI.getOpcode();
369 if (UseOp == SPIRV::OpConstantFunctionPointerINTEL ||
370 getInvokeOperandIdx(UseOp)) {
371 MAI.setSkipEmission(&MI);
372 return false;
373 }
374 }
375 }
376 for (MachineInstr &UseMI : MRI.use_instructions(DefReg)) {
377 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)
378 continue;
379 // it's a dummy definition, FP constant refers to a function,
380 // and this is resolved in another way; let's skip this definition
381 assert(UseMI.getOperand(2).isReg() &&
382 UseMI.getOperand(2).getReg() == DefReg);
383 MAI.setSkipEmission(&MI);
384 return false;
385 }
386 }
387 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
388 TII->isInlineAsmDefInstr(MI);
389}
390
391// This is a special case of a function pointer referring to a possibly
392// forward function declaration. The operand is a dummy OpUndef that
393// requires a special treatment.
394// FunPtrOp is the MachineOperand previously recorded via
395// SPIRVGlobalRegistry::recordFunctionPointer, identifying which Function
396// this placeholder refers to.
397void SPIRVModuleAnalysisImpl::visitFunPtrUse(
398 Register OpReg, const MachineOperand *FunPtrOp,
399 InstrGRegsMap &SignatureToGReg,
400 std::map<const Value *, unsigned> &GlobalToGReg,
401 const MachineFunction *MF) {
402 const MachineOperand *OpFunDef = GR->getFunctionDefinitionByUse(FunPtrOp);
403 assert(OpFunDef && OpFunDef->isReg());
404 // find the actual function definition and number it globally in advance
405 const MachineInstr *OpDefMI = OpFunDef->getParent();
406 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);
407 const MachineFunction *FunDefMF = OpDefMI->getParent()->getParent();
408 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();
409 do {
410 visitDecl(FunDefMRI, SignatureToGReg, GlobalToGReg, FunDefMF, *OpDefMI);
411 OpDefMI = OpDefMI->getNextNode();
412 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||
413 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));
414 // associate the function pointer with the newly assigned global number
415 MCRegister GlobalFunDefReg =
416 MAI.getRegisterAlias(FunDefMF, OpFunDef->getReg());
417 assert(GlobalFunDefReg.isValid() &&
418 "Function definition must refer to a global register");
419 MAI.setRegisterAlias(MF, OpReg, GlobalFunDefReg);
420}
421
422// Depth first recursive traversal of dependencies. Repeated visits are guarded
423// by MAI.hasRegisterAlias().
424void SPIRVModuleAnalysisImpl::visitDecl(
425 const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,
426 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
427 const MachineInstr &MI) {
428 unsigned Opcode = MI.getOpcode();
429
430 // Process each operand of the instruction to resolve dependencies
431 for (const MachineOperand &MO : MI.operands()) {
432 if (!MO.isReg() || MO.isDef())
433 continue;
434 Register OpReg = MO.getReg();
435 // Handle function pointers special case
436 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&
437 MRI.getRegClass(OpReg) == &SPIRV::pIDRegClass) {
438 visitFunPtrUse(OpReg, &MI.getOperand(2), SignatureToGReg, GlobalToGReg,
439 MF);
440 continue;
441 }
442 // Skip already processed instructions
443 if (MAI.hasRegisterAlias(MF, MO.getReg()))
444 continue;
445 // Recursively visit dependencies
446 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(OpReg)) {
447 if (isDeclSection(MRI, *OpDefMI))
448 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, *OpDefMI);
449 continue;
450 }
451 // Handle the unexpected case of no unique definition for the SPIR-V
452 // instruction
453 LLVM_DEBUG({
454 dbgs() << "Unexpectedly, no unique definition for the operand ";
455 MO.print(dbgs());
456 dbgs() << "\nInstruction: ";
457 MI.print(dbgs());
458 dbgs() << "\n";
459 });
461 "No unique definition is found for the virtual register");
462 }
463
464 MCRegister GReg;
465 bool IsFunDef = false;
466 if (TII->isSpecConstantInstr(MI)) {
467 GReg = MAI.getNextIDRegister();
468 MAI.MS[SPIRV::MB_TypeConstVars].push_back(&MI);
469 } else if (Opcode == SPIRV::OpFunction ||
470 Opcode == SPIRV::OpFunctionParameter) {
471 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);
472 } else if (Opcode == SPIRV::OpTypeStruct ||
473 Opcode == SPIRV::OpConstantComposite) {
474 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
475 const MachineInstr *NextInstr = MI.getNextNode();
476 while (NextInstr &&
477 ((Opcode == SPIRV::OpTypeStruct &&
478 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||
479 (Opcode == SPIRV::OpConstantComposite &&
480 NextInstr->getOpcode() ==
481 SPIRV::OpConstantCompositeContinuedINTEL))) {
482 MCRegister Tmp = handleTypeDeclOrConstant(*NextInstr, SignatureToGReg);
483 MAI.setRegisterAlias(MF, NextInstr->getOperand(0).getReg(), Tmp);
484 MAI.setSkipEmission(NextInstr);
485 NextInstr = NextInstr->getNextNode();
486 }
487 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
488 TII->isInlineAsmDefInstr(MI)) {
489 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
490 } else if (Opcode == SPIRV::OpVariable ||
491 Opcode == SPIRV::OpUntypedVariableKHR) {
492 GReg = handleVariable(MF, MI, GlobalToGReg);
493 } else {
494 LLVM_DEBUG({
495 dbgs() << "\nInstruction: ";
496 MI.print(dbgs());
497 dbgs() << "\n";
498 });
499 llvm_unreachable("Unexpected instruction is visited");
500 }
501 MAI.setRegisterAlias(MF, MI.getOperand(0).getReg(), GReg);
502 if (!IsFunDef)
503 MAI.setSkipEmission(&MI);
504}
505
506MCRegister SPIRVModuleAnalysisImpl::handleFunctionOrParameter(
507 const MachineFunction *MF, const MachineInstr &MI,
508 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
509 const Value *GObj = GR->getGlobalObject(MF, MI.getOperand(0).getReg());
510 assert(GObj && "Unregistered global definition");
511 const Function *F = dyn_cast<Function>(GObj);
512 if (!F)
513 F = dyn_cast<Argument>(GObj)->getParent();
514 assert(F && "Expected a reference to a function or an argument");
515 IsFunDef = !F->isDeclaration();
516 auto [It, Inserted] = GlobalToGReg.try_emplace(GObj);
517 if (!Inserted)
518 return It->second;
519 MCRegister GReg = MAI.getNextIDRegister();
520 It->second = GReg;
521 if (!IsFunDef)
522 MAI.MS[SPIRV::MB_ExtFuncDecls].push_back(&MI);
523 return GReg;
524}
525
526MCRegister SPIRVModuleAnalysisImpl::handleTypeDeclOrConstant(
527 const MachineInstr &MI, InstrGRegsMap &SignatureToGReg) {
528 InstrSignature MISign = instrToSignature(MI, MAI, false);
529 auto [It, Inserted] = SignatureToGReg.try_emplace(MISign);
530 if (!Inserted)
531 return It->second;
532 MCRegister GReg = MAI.getNextIDRegister();
533 It->second = GReg;
534 MAI.MS[SPIRV::MB_TypeConstVars].push_back(&MI);
535 return GReg;
536}
537
538MCRegister SPIRVModuleAnalysisImpl::handleVariable(
539 const MachineFunction *MF, const MachineInstr &MI,
540 std::map<const Value *, unsigned> &GlobalToGReg) {
541 MAI.GlobalVarList.push_back(&MI);
542 const Value *GObj = GR->getGlobalObject(MF, MI.getOperand(0).getReg());
543 assert(GObj && "Unregistered global definition");
544 auto [It, Inserted] = GlobalToGReg.try_emplace(GObj);
545 if (!Inserted)
546 return It->second;
547 MCRegister GReg = MAI.getNextIDRegister();
548 It->second = GReg;
549 MAI.MS[SPIRV::MB_TypeConstVars].push_back(&MI);
550 if (const auto *GV = dyn_cast<GlobalVariable>(GObj))
551 MAI.GlobalObjMap[GV] = GReg;
552 return GReg;
553}
554
555void SPIRVModuleAnalysisImpl::collectDeclarations(const Module &M) {
556 InstrGRegsMap SignatureToGReg;
557 std::map<const Value *, unsigned> GlobalToGReg;
558 for (const Function &F : M) {
559 MachineFunction *MF = GetMF(F);
560 if (!MF)
561 continue;
562 const MachineRegisterInfo &MRI = MF->getRegInfo();
563 unsigned PastHeader = 0;
564 for (MachineBasicBlock &MBB : *MF) {
565 for (MachineInstr &MI : MBB) {
566 if (MI.getNumOperands() == 0)
567 continue;
568 unsigned Opcode = MI.getOpcode();
569 if (Opcode == SPIRV::OpFunction) {
570 if (PastHeader == 0) {
571 PastHeader = 1;
572 continue;
573 }
574 } else if (Opcode == SPIRV::OpFunctionParameter) {
575 if (PastHeader < 2)
576 continue;
577 } else if (PastHeader > 0) {
578 PastHeader = 2;
579 }
580
581 const MachineOperand &DefMO = MI.getOperand(0);
582 switch (Opcode) {
583 case SPIRV::OpExtension:
584 MAI.Reqs.addExtension(SPIRV::Extension::Extension(DefMO.getImm()));
585 MAI.setSkipEmission(&MI);
586 break;
587 case SPIRV::OpCapability:
588 MAI.Reqs.addCapability(SPIRV::Capability::Capability(DefMO.getImm()));
589 MAI.setSkipEmission(&MI);
590 if (PastHeader > 0)
591 PastHeader = 2;
592 break;
593 default:
594 if (DefMO.isReg() && isDeclSection(MRI, MI) &&
595 !MAI.hasRegisterAlias(MF, DefMO.getReg()))
596 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);
597 // Device enqueue instructions are not decls, but their Invoke
598 // operand may be a function-pointer placeholder OpUndef. Resolve it
599 // to the OpFunction's global <id> via visitFunPtrUse.
600 if (unsigned InvokeIdx = getInvokeOperandIdx(Opcode)) {
601 const MachineOperand &InvokeMO = MI.getOperand(InvokeIdx);
602 if (InvokeMO.isReg()) {
603 Register InvokeReg = InvokeMO.getReg();
604 if (!MAI.hasRegisterAlias(MF, InvokeReg)) {
605 if (const MachineInstr *DefMI =
606 MRI.getUniqueVRegDef(InvokeReg)) {
607 if (DefMI->getOpcode() == SPIRV::OpUndef) {
608 const MachineOperand *FunPtrOp = &DefMI->getOperand(0);
609 if (GR->getFunctionDefinitionByUse(FunPtrOp))
610 visitFunPtrUse(InvokeReg, FunPtrOp, SignatureToGReg,
611 GlobalToGReg, MF);
612 }
613 }
614 }
615 }
616 }
617 }
618 }
619 }
620 }
621}
622
623// Look for IDs declared with Import linkage, and map the corresponding function
624// to the register defining that variable (which will usually be the result of
625// an OpFunction). This lets us call externally imported functions using
626// the correct ID registers.
627void SPIRVModuleAnalysisImpl::collectFuncNames(MachineInstr &MI,
628 const Function *F) {
629 if (MI.getOpcode() == SPIRV::OpDecorate) {
630 // If it's got Import linkage.
631 auto Dec = MI.getOperand(1).getImm();
632 if (Dec == SPIRV::Decoration::LinkageAttributes) {
633 auto Lnk = MI.getOperand(MI.getNumOperands() - 1).getImm();
634 if (Lnk == SPIRV::LinkageType::Import) {
635 // Map imported function name to function ID register.
636 const Function *ImportedFunc =
637 F->getParent()->getFunction(getStringImm(MI, 2));
638 Register Target = MI.getOperand(0).getReg();
639 MAI.GlobalObjMap[ImportedFunc] =
640 MAI.getRegisterAlias(MI.getMF(), Target);
641 }
642 }
643 } else if (MI.getOpcode() == SPIRV::OpFunction) {
644 // Record all internal OpFunction declarations.
645 Register Reg = MI.defs().begin()->getReg();
646 MCRegister GlobalReg = MAI.getRegisterAlias(MI.getMF(), Reg);
647 assert(GlobalReg.isValid());
648 MAI.GlobalObjMap[F] = GlobalReg;
649 }
650}
651
652// Collect the given instruction in the specified MS. We assume global register
653// numbering has already occurred by this point. We can directly compare reg
654// arguments when detecting duplicates.
657 bool Append = true) {
658 MAI.setSkipEmission(&MI);
659 InstrSignature MISign = instrToSignature(MI, MAI, true);
660 auto FoundMI = IS.insert(std::move(MISign));
661 if (!FoundMI.second) {
662 if (MI.getOpcode() == SPIRV::OpDecorate) {
663 assert(MI.getNumOperands() >= 2 &&
664 "Decoration instructions must have at least 2 operands");
665 assert(MSType == SPIRV::MB_Annotations &&
666 "Only OpDecorate instructions can be duplicates");
667 // For FPFastMathMode decoration, we need to merge the flags of the
668 // duplicate decoration with the original one, so we need to find the
669 // original instruction that has the same signature. For the rest of
670 // instructions, we will simply skip the duplicate.
671 if (MI.getOperand(1).getImm() != SPIRV::Decoration::FPFastMathMode)
672 return; // Skip duplicates of other decorations.
673
674 const SPIRV::InstrList &Decorations = MAI.MS[MSType];
675 for (const MachineInstr *OrigMI : Decorations) {
676 if (instrToSignature(*OrigMI, MAI, true) == MISign) {
677 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&
678 "Original instruction must have the same number of operands");
679 assert(
680 OrigMI->getNumOperands() == 3 &&
681 "FPFastMathMode decoration must have 3 operands for OpDecorate");
682 unsigned OrigFlags = OrigMI->getOperand(2).getImm();
683 unsigned NewFlags = MI.getOperand(2).getImm();
684 if (OrigFlags == NewFlags)
685 return; // No need to merge, the flags are the same.
686
687 // Emit warning about possible conflict between flags.
688 unsigned FinalFlags = OrigFlags | NewFlags;
689 llvm::errs()
690 << "Warning: Conflicting FPFastMathMode decoration flags "
691 "in instruction: "
692 << *OrigMI << "Original flags: " << OrigFlags
693 << ", new flags: " << NewFlags
694 << ". They will be merged on a best effort basis, but not "
695 "validated. Final flags: "
696 << FinalFlags << "\n";
697 MachineInstr *OrigMINonConst = const_cast<MachineInstr *>(OrigMI);
698 MachineOperand &OrigFlagsOp = OrigMINonConst->getOperand(2);
699 OrigFlagsOp = MachineOperand::CreateImm(FinalFlags);
700 return; // Merge done, so we found a duplicate; don't add it to MAI.MS
701 }
702 }
703 assert(false && "No original instruction found for the duplicate "
704 "OpDecorate, but we found one in IS.");
705 }
706 return; // insert failed, so we found a duplicate; don't add it to MAI.MS
707 }
708 // No duplicates, so add it.
709 if (Append)
710 MAI.MS[MSType].push_back(&MI);
711 else
712 MAI.MS[MSType].insert(MAI.MS[MSType].begin(), &MI);
713}
714
715// Some global instructions make reference to function-local ID regs, so cannot
716// be correctly collected until these registers are globally numbered.
717void SPIRVModuleAnalysisImpl::processOtherInstrs(const Module &M) {
719 for (const Function &F : M) {
720 if (F.isDeclaration())
721 continue;
722 MachineFunction *MF = GetMF(F);
723 assert(MF);
724
725 for (MachineBasicBlock &MBB : *MF)
726 for (MachineInstr &MI : MBB) {
727 if (MI.isDebugValueLike()) {
728 MAI.DebugValueMIs.push_back(&MI);
729 continue;
730 }
731 if (MAI.getSkipEmission(&MI))
732 continue;
733 const unsigned OpCode = MI.getOpcode();
734 if (OpCode == SPIRV::OpString) {
736 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(2).isImm() &&
737 MI.getOperand(2).getImm() ==
738 SPIRV::InstructionSet::
739 NonSemantic_Shader_DebugInfo_100) {
740 // TODO: This branch is dead. SPIRVNonSemanticDebugHandler emits NSDI
741 // instructions directly as MCInsts at print time; no
742 // MachineInstructions with the NSDI ext set are created anymore.
743 // Remove this block and
744 // MB_NonSemanticGlobalDI once per-function NSDI emission is confirmed
745 // not to need MIR routing.
746 MachineOperand Ins = MI.getOperand(3);
747 namespace NS = SPIRV::NonSemanticExtInst;
748 static constexpr int64_t GlobalNonSemanticDITy[] = {
749 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
750 NS::DebugTypeBasic, NS::DebugTypePointer};
751 bool IsGlobalDI = false;
752 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
753 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
754 if (IsGlobalDI)
756 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {
758 } else if (OpCode == SPIRV::OpEntryPoint) {
760 } else if (TII->isAliasingInstr(MI)) {
762 } else if (TII->isDecorationInstr(MI)) {
763 if (MI.getOpcode() == SPIRV::OpDecorate &&
764 MI.getOperand(1).getImm() ==
765 static_cast<unsigned>(
766 SPIRV::Decoration::AuxDataInstructionMetadata)) {
767 MAI.setSkipEmission(&MI);
768 std::string Str = getStringImm(MI, 2);
770 auto MaybeKind =
771 StringSwitch<std::optional<AMDMD>>(Str)
772 .Case("amdgpu.no.fine.grained.memory",
773 AMDMD::NoFineGrainedMemory)
774 .Case("amdgpu.no.remote.memory", AMDMD::NoRemoteMemory)
775 .Case("atomic.ignore.denormal.mode",
776 AMDMD::IgnoreDenormalMode)
777 .Default(std::nullopt);
778 if (MaybeKind)
779 MAI.InstrAuxDataRecords.push_back(
780 {MF, MI.getOperand(0).getReg(), *MaybeKind});
781 continue;
782 }
784 collectFuncNames(MI, &F);
785 } else if (TII->isConstantInstr(MI)) {
786 // Now OpSpecConstant*s are not in DT,
787 // but they need to be collected anyway.
789 } else if (OpCode == SPIRV::OpFunction) {
790 collectFuncNames(MI, &F);
791 } else if (OpCode == SPIRV::OpTypeForwardPointer) {
793 }
794 }
795 }
796 // Selection order can place a scope/list ahead of a domain/scope it
797 // references. The dependency meanwhile is domain -> scope -> list, so sort
798 // the def before its uses.
799 auto AliasingTier = [](const MachineInstr *MI) {
800 switch (MI->getOpcode()) {
801 case SPIRV::OpAliasDomainDeclINTEL:
802 return 0;
803 case SPIRV::OpAliasScopeDeclINTEL:
804 return 1;
805 case SPIRV::OpAliasScopeListDeclINTEL:
806 return 2;
807 default:
808 llvm_unreachable("unexpected aliasing instruction");
809 }
810 };
812 [&](const MachineInstr *LHS, const MachineInstr *RHS) {
813 return AliasingTier(LHS) < AliasingTier(RHS);
814 });
815}
816
817// Number registers in all functions globally from 0 onwards and store
818// the result in global register alias table. Some registers are already
819// numbered.
820void SPIRVModuleAnalysisImpl::numberRegistersGlobally(const Module &M) {
821 for (const Function &F : M) {
822 if (F.isDeclaration())
823 continue;
824 MachineFunction *MF = GetMF(F);
825 assert(MF);
826 for (MachineBasicBlock &MBB : *MF) {
827 for (MachineInstr &MI : MBB) {
828 for (MachineOperand &Op : MI.operands()) {
829 if (!Op.isReg())
830 continue;
831 Register Reg = Op.getReg();
832 if (MAI.hasRegisterAlias(MF, Reg))
833 continue;
834 MCRegister NewReg = MAI.getNextIDRegister();
835 MAI.setRegisterAlias(MF, Reg, NewReg);
836 }
837 if (MI.getOpcode() != SPIRV::OpExtInst)
838 continue;
839 auto Set = MI.getOperand(2).getImm();
840 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Set);
841 if (Inserted)
842 It->second = MAI.getNextIDRegister();
843 }
844 }
845 }
846}
847
849 const SPIRVSubtarget &ST,
850 StringRef Feature) {
851 if (!ST.isAtLeastSPIRVVer(VersionTuple(1, 3)))
852 report_fatal_error(Twine(Feature) + " requires SPIR-V 1.3", false);
853 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_quad_control))
854 report_fatal_error(Twine(Feature) + " requires the following SPIR-V "
855 "extension: SPV_KHR_quad_control",
856 false);
857 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_maximal_reconvergence))
858 report_fatal_error("SPV_KHR_quad_control requires the following SPIR-V "
859 "extension: SPV_KHR_maximal_reconvergence",
860 false);
861 Reqs.addExtension(SPIRV::Extension::SPV_KHR_quad_control);
862 Reqs.addExtension(SPIRV::Extension::SPV_KHR_maximal_reconvergence);
863}
864
865// RequirementHandler implementations.
867 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,
868 const SPIRVSubtarget &ST) {
869 if (Category == SPIRV::OperandCategory::ExecutionModeOperand &&
870 (i == SPIRV::ExecutionMode::QuadDerivativesKHR ||
871 i == SPIRV::ExecutionMode::RequireFullQuadsKHR))
873 getSymbolicOperandMnemonic(Category, i));
874 if (Category == SPIRV::OperandCategory::ExecutionModeOperand &&
875 (i == SPIRV::ExecutionMode::DerivativeGroupQuadsKHR ||
876 i == SPIRV::ExecutionMode::DerivativeGroupLinearKHR) &&
877 !ST.canUseExtension(SPIRV::Extension::SPV_KHR_compute_shader_derivatives))
879 " requires the following SPIR-V extension: "
880 "SPV_KHR_compute_shader_derivatives",
881 false);
882 addRequirements(getSymbolicOperandRequirements(Category, i, ST, *this));
883}
884
885void SPIRV::RequirementHandler::recursiveAddCapabilities(
886 const CapabilityList &ToPrune) {
887 for (const auto &Cap : ToPrune) {
888 AllCaps.insert(Cap);
889 CapabilityList ImplicitDecls =
890 getSymbolicOperandCapabilities(OperandCategory::CapabilityOperand, Cap);
891 recursiveAddCapabilities(ImplicitDecls);
892 }
893}
894
896 for (const auto &Cap : ToAdd) {
897 bool IsNewlyInserted = AllCaps.insert(Cap).second;
898 if (!IsNewlyInserted) // Don't re-add if it's already been declared.
899 continue;
900 CapabilityList ImplicitDecls =
901 getSymbolicOperandCapabilities(OperandCategory::CapabilityOperand, Cap);
902 recursiveAddCapabilities(ImplicitDecls);
903 MinimalCaps.push_back(Cap);
904 }
905}
906
908 const SPIRV::Requirements &Req) {
909 if (!Req.IsSatisfiable)
910 report_fatal_error("Adding SPIR-V requirements this target can't satisfy.");
911
912 if (Req.Cap.has_value())
913 addCapabilities({Req.Cap.value()});
914
915 addExtensions(Req.Exts);
916
917 if (!Req.MinVer.empty()) {
918 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {
919 LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer
920 << " and <= " << MaxVersion << "\n");
921 report_fatal_error("Adding SPIR-V requirements that can't be satisfied.");
922 }
923
924 if (MinVersion.empty() || Req.MinVer > MinVersion)
925 MinVersion = Req.MinVer;
926 }
927
928 if (!Req.MaxVer.empty()) {
929 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {
930 LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer
931 << " and >= " << MinVersion << "\n");
932 report_fatal_error("Adding SPIR-V requirements that can't be satisfied.");
933 }
934
935 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)
936 MaxVersion = Req.MaxVer;
937 }
938}
939
941 const SPIRVSubtarget &ST) const {
942 // Report as many errors as possible before aborting the compilation.
943 bool IsSatisfiable = true;
944 auto TargetVer = ST.getSPIRVVersion();
945
946 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {
948 dbgs() << "Target SPIR-V version too high for required features\n"
949 << "Required max version: " << MaxVersion << " target version "
950 << TargetVer << "\n");
951 IsSatisfiable = false;
952 }
953
954 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {
955 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"
956 << "Required min version: " << MinVersion
957 << " target version " << TargetVer << "\n");
958 IsSatisfiable = false;
959 }
960
961 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {
963 dbgs()
964 << "Version is too low for some features and too high for others.\n"
965 << "Required SPIR-V min version: " << MinVersion
966 << " required SPIR-V max version " << MaxVersion << "\n");
967 IsSatisfiable = false;
968 }
969
970 for (auto Cap : MinimalCaps) {
971 if (AvailableCaps.contains(Cap) && !shouldAvoidCapability(ST, Cap))
972 continue;
973 LLVM_DEBUG(dbgs() << "Capability not supported: "
975 OperandCategory::CapabilityOperand, Cap)
976 << "\n");
977 IsSatisfiable = false;
978 }
979
980 for (auto Ext : AllExtensions) {
981 if (ST.canUseExtension(Ext))
982 continue;
983 LLVM_DEBUG(dbgs() << "Extension not supported: "
985 OperandCategory::ExtensionOperand, Ext)
986 << "\n");
987 IsSatisfiable = false;
988 }
989
990 if (!IsSatisfiable)
991 report_fatal_error("Unable to meet SPIR-V requirements for this target.");
992}
993
994// Add the given capabilities and all their implicitly defined capabilities too.
996 for (const auto Cap : ToAdd)
997 if (AvailableCaps.insert(Cap).second)
999 SPIRV::OperandCategory::CapabilityOperand, Cap));
1000}
1001
1003 const Capability::Capability ToRemove,
1004 const Capability::Capability IfPresent) {
1005 if (AllCaps.contains(IfPresent)) {
1006 AllCaps.erase(ToRemove);
1007 llvm::erase(MinimalCaps, ToRemove);
1008 }
1009}
1010
1011namespace llvm {
1012namespace SPIRV {
1014 // Provided by both all supported Vulkan versions and OpenCl.
1015 addAvailableCaps({Capability::Shader, Capability::Linkage, Capability::Int8,
1016 Capability::Int16});
1017
1018 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 3)))
1019 addAvailableCaps({Capability::GroupNonUniform,
1020 Capability::GroupNonUniformVote,
1021 Capability::GroupNonUniformArithmetic,
1022 Capability::GroupNonUniformBallot,
1023 Capability::GroupNonUniformClustered,
1024 Capability::GroupNonUniformShuffle,
1025 Capability::GroupNonUniformShuffleRelative,
1026 Capability::GroupNonUniformQuad});
1027
1028 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 6)))
1029 addAvailableCaps({Capability::DotProduct, Capability::DotProductInputAll,
1030 Capability::DotProductInput4x8Bit,
1031 Capability::DotProductInput4x8BitPacked,
1032 Capability::DemoteToHelperInvocation});
1033
1034 // Add capabilities enabled by extensions.
1035 for (auto Extension : ST.getAllAvailableExtensions()) {
1036 CapabilityList EnabledCapabilities =
1038 addAvailableCaps(EnabledCapabilities);
1039 }
1040
1041 // EXT and NV use the same capability.
1042 if (ST.canUseExtension(Extension::SPV_EXT_shader_subgroup_partitioned))
1043 addAvailableCaps({Capability::GroupNonUniformPartitionedNV});
1044
1045 if (!ST.isShader()) {
1046 initAvailableCapabilitiesForOpenCL(ST);
1047 return;
1048 }
1049
1050 if (ST.isShader()) {
1051 initAvailableCapabilitiesForVulkan(ST);
1052 return;
1053 }
1054
1055 report_fatal_error("Unimplemented environment for SPIR-V generation.");
1056}
1057
1058void RequirementHandler::initAvailableCapabilitiesForOpenCL(
1059 const SPIRVSubtarget &ST) {
1060 // Add the min requirements for different OpenCL and SPIR-V versions.
1061 addAvailableCaps({Capability::Addresses, Capability::Float16Buffer,
1062 Capability::Kernel, Capability::Vector16,
1063 Capability::Groups, Capability::GenericPointer,
1064 Capability::StorageImageWriteWithoutFormat,
1065 Capability::StorageImageReadWithoutFormat});
1066 if (ST.hasOpenCLFullProfile())
1067 addAvailableCaps({Capability::Int64, Capability::Int64Atomics});
1068 if (ST.hasOpenCLImageSupport()) {
1069 addAvailableCaps({Capability::ImageBasic, Capability::LiteralSampler,
1070 Capability::Image1D, Capability::SampledBuffer,
1071 Capability::ImageBuffer});
1072 if (ST.isAtLeastOpenCLVer(VersionTuple(2, 0)))
1073 addAvailableCaps({Capability::ImageReadWrite});
1074 }
1075 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 1)) &&
1076 ST.isAtLeastOpenCLVer(VersionTuple(2, 2)))
1077 addAvailableCaps({Capability::SubgroupDispatch, Capability::PipeStorage});
1078 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 4)))
1079 addAvailableCaps({Capability::DenormPreserve, Capability::DenormFlushToZero,
1080 Capability::SignedZeroInfNanPreserve,
1081 Capability::RoundingModeRTE,
1082 Capability::RoundingModeRTZ});
1083 // TODO: verify if this needs some checks.
1084 addAvailableCaps({Capability::Float16, Capability::Float64});
1085
1086 // TODO: add OpenCL extensions.
1087}
1088
1089void RequirementHandler::initAvailableCapabilitiesForVulkan(
1090 const SPIRVSubtarget &ST) {
1091
1092 // Core in Vulkan 1.1 and earlier.
1093 addAvailableCaps({Capability::Int64,
1094 Capability::Geometry,
1095 Capability::Float16,
1096 Capability::Float64,
1097 Capability::GroupNonUniform,
1098 Capability::Image1D,
1099 Capability::SampledBuffer,
1100 Capability::ImageBuffer,
1101 Capability::UniformBufferArrayDynamicIndexing,
1102 Capability::SampledImageArrayDynamicIndexing,
1103 Capability::StorageBufferArrayDynamicIndexing,
1104 Capability::StorageImageArrayDynamicIndexing,
1105 Capability::DerivativeControl,
1106 Capability::SampleRateShading,
1107 Capability::MinLod,
1108 Capability::ImageQuery,
1109 Capability::ImageGatherExtended,
1110 Capability::Addresses,
1111 Capability::VulkanMemoryModelKHR,
1112 Capability::StorageImageExtendedFormats,
1113 Capability::StorageImageMultisample,
1114 Capability::ImageMSArray});
1115
1116 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 3)) ||
1117 ST.canUseExtension(Extension::SPV_KHR_variable_pointers))
1118 addAvailableCaps({Capability::VariablePointersStorageBuffer,
1119 Capability::VariablePointers});
1120
1121 // Became core in Vulkan 1.1.
1122 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 3)))
1123 addAvailableCaps({Capability::DrawParameters, Capability::MultiView});
1124
1125 // Became core in Vulkan 1.2
1126 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 5))) {
1128 {Capability::Int64Atomics, Capability::ShaderNonUniformEXT,
1129 Capability::RuntimeDescriptorArrayEXT,
1130 Capability::InputAttachmentArrayDynamicIndexingEXT,
1131 Capability::UniformTexelBufferArrayDynamicIndexingEXT,
1132 Capability::StorageTexelBufferArrayDynamicIndexingEXT,
1133 Capability::UniformBufferArrayNonUniformIndexingEXT,
1134 Capability::SampledImageArrayNonUniformIndexingEXT,
1135 Capability::StorageBufferArrayNonUniformIndexingEXT,
1136 Capability::StorageImageArrayNonUniformIndexingEXT,
1137 Capability::InputAttachmentArrayNonUniformIndexingEXT,
1138 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
1139 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
1140 }
1141
1142 // Became core in Vulkan 1.3
1143 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 6)))
1144 addAvailableCaps({Capability::StorageImageWriteWithoutFormat,
1145 Capability::StorageImageReadWithoutFormat});
1146}
1147
1148} // namespace SPIRV
1149} // namespace llvm
1150
1151// Add a capability that became core in SPIR-V 1.3. Earlier versions require
1152// the extension that introduced it.
1154 const SPIRVSubtarget &ST,
1155 SPIRV::Capability::Capability Cap,
1156 SPIRV::Extension::Extension Ext) {
1157 Reqs.addCapability(Cap);
1158 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 3)))
1159 return;
1160 if (!ST.canUseExtension(Ext))
1162 Twine("capability ") +
1164 SPIRV::OperandCategory::CapabilityOperand, Cap) +
1165 " requires SPIR-V 1.3 or the following SPIR-V extension: " +
1166 getSymbolicOperandMnemonic(SPIRV::OperandCategory::ExtensionOperand,
1167 Ext),
1168 false);
1169 Reqs.addExtension(Ext);
1170}
1171
1172// Add the required capabilities from a decoration instruction (including
1173// BuiltIns).
1174static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
1176 const SPIRVSubtarget &ST) {
1177 int64_t DecOp = MI.getOperand(DecIndex).getImm();
1178 auto Dec = static_cast<SPIRV::Decoration::Decoration>(DecOp);
1180 SPIRV::OperandCategory::DecorationOperand, Dec, ST, Reqs));
1181
1182 if (Dec == SPIRV::Decoration::BuiltIn) {
1183 int64_t BuiltInOp = MI.getOperand(DecIndex + 1).getImm();
1184 auto BuiltIn = static_cast<SPIRV::BuiltIn::BuiltIn>(BuiltInOp);
1185 if ((BuiltIn == SPIRV::BuiltIn::BaryCoordKHR ||
1186 BuiltIn == SPIRV::BuiltIn::BaryCoordNoPerspKHR) &&
1187 !ST.canUseExtension(
1188 SPIRV::Extension::SPV_KHR_fragment_shader_barycentric))
1190 "Barycentric built-ins require the following SPIR-V extension: "
1191 "SPV_KHR_fragment_shader_barycentric",
1192 false);
1193 switch (BuiltIn) {
1194 case SPIRV::BuiltIn::BaseVertex:
1195 case SPIRV::BuiltIn::BaseInstance:
1197 Reqs, ST, SPIRV::Capability::DrawParameters,
1198 SPIRV::Extension::SPV_KHR_shader_draw_parameters);
1199 break;
1200 case SPIRV::BuiltIn::ViewIndex:
1201 addCoreInSPIRV13Capability(Reqs, ST, SPIRV::Capability::MultiView,
1202 SPIRV::Extension::SPV_KHR_multiview);
1203 break;
1204 default:
1206 SPIRV::OperandCategory::BuiltInOperand, BuiltIn, ST, Reqs));
1207 break;
1208 }
1209 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {
1210 int64_t LinkageOp = MI.getOperand(MI.getNumOperands() - 1).getImm();
1211 SPIRV::LinkageType::LinkageType LnkType =
1212 static_cast<SPIRV::LinkageType::LinkageType>(LinkageOp);
1213 if (LnkType == SPIRV::LinkageType::LinkOnceODR)
1214 Reqs.addExtension(SPIRV::Extension::SPV_KHR_linkonce_odr);
1215 else if (LnkType == SPIRV::LinkageType::WeakAMD) {
1216 Reqs.addExtension(SPIRV::Extension::SPV_AMD_weak_linkage);
1217 Reqs.addCapability(SPIRV::Capability::WeakLinkageAMD);
1218 }
1219 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||
1220 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {
1221 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_cache_controls);
1222 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {
1223 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_global_variable_host_access);
1224 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||
1225 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
1226 Reqs.addExtension(
1227 SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
1228 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
1229 Reqs.addRequirements(SPIRV::Capability::ShaderNonUniformEXT);
1230 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {
1231 Reqs.addRequirements(SPIRV::Capability::FPMaxErrorINTEL);
1232 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_fp_max_error);
1233 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {
1234 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2)) {
1235 Reqs.addRequirements(SPIRV::Capability::FloatControls2);
1236 Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls2);
1237 }
1238 }
1239}
1240
1241// Add requirements for image handling.
1244 const SPIRVSubtarget &ST) {
1245 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");
1246 // The operand indices used here are based on the OpTypeImage layout, which
1247 // the MachineInstr follows as well.
1248 int64_t ImgFormatOp = MI.getOperand(7).getImm();
1249 auto ImgFormat = static_cast<SPIRV::ImageFormat::ImageFormat>(ImgFormatOp);
1250 Reqs.getAndAddRequirements(SPIRV::OperandCategory::ImageFormatOperand,
1251 ImgFormat, ST);
1252
1253 bool IsArrayed = MI.getOperand(4).getImm() == 1;
1254 bool IsMultisampled = MI.getOperand(5).getImm() == 1;
1255 bool NoSampler = MI.getOperand(6).getImm() == 2;
1256 // Add dimension requirements.
1257 assert(MI.getOperand(2).isImm());
1258 switch (MI.getOperand(2).getImm()) {
1259 case SPIRV::Dim::DIM_1D:
1260 Reqs.addRequirements(NoSampler ? SPIRV::Capability::Image1D
1261 : SPIRV::Capability::Sampled1D);
1262 break;
1263 case SPIRV::Dim::DIM_2D:
1264 if (IsMultisampled && NoSampler)
1265 Reqs.addRequirements(SPIRV::Capability::StorageImageMultisample);
1266 if (IsMultisampled && IsArrayed)
1267 Reqs.addRequirements(SPIRV::Capability::ImageMSArray);
1268 break;
1269 case SPIRV::Dim::DIM_3D:
1270 break;
1271 case SPIRV::Dim::DIM_Cube:
1272 Reqs.addRequirements(SPIRV::Capability::Shader);
1273 if (IsArrayed)
1274 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageCubeArray
1275 : SPIRV::Capability::SampledCubeArray);
1276 break;
1277 case SPIRV::Dim::DIM_Rect:
1278 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageRect
1279 : SPIRV::Capability::SampledRect);
1280 break;
1281 case SPIRV::Dim::DIM_Buffer:
1282 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageBuffer
1283 : SPIRV::Capability::SampledBuffer);
1284 break;
1285 case SPIRV::Dim::DIM_SubpassData:
1286 Reqs.addRequirements(SPIRV::Capability::InputAttachment);
1287 break;
1288 }
1289
1290 // Has optional access qualifier.
1291 if (!ST.isShader()) {
1292 if (MI.getNumOperands() > 8 &&
1293 MI.getOperand(8).getImm() == SPIRV::AccessQualifier::ReadWrite)
1294 Reqs.addRequirements(SPIRV::Capability::ImageReadWrite);
1295 else
1296 Reqs.addRequirements(SPIRV::Capability::ImageBasic);
1297 }
1298}
1299
1300static bool isBFloat16Type(SPIRVTypeInst TypeDef) {
1301 return TypeDef && TypeDef->getNumOperands() == 3 &&
1302 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&
1303 TypeDef->getOperand(1).getImm() == 16 &&
1304 TypeDef->getOperand(2).getImm() == SPIRV::FPEncoding::BFloat16KHR;
1305}
1306
1307// Registers created after isel (e.g. by MachineCSE) may lack registry entries.
1308// Fall back to the defining instruction's result type.
1310 const SPIRVGlobalRegistry *GR) {
1311 if (SPIRVTypeInst Ty = GR->getSPIRVTypeForVReg(Reg, &MF))
1312 return Ty;
1313 const MachineRegisterInfo &MRI = MF.getRegInfo();
1314 return MRI.getVRegDef(MRI.getVRegDef(Reg)->getOperand(1).getReg());
1315}
1316
1317// Add requirements for handling atomic float instructions
1318#define ATOM_FLT_REQ_EXT_MSG(ExtName) \
1319 "The atomic float instruction requires the following SPIR-V " \
1320 "extension: SPV_EXT_shader_atomic_float" ExtName
1323 const SPIRVSubtarget &ST) {
1324 SPIRVTypeInst VecTypeDef =
1325 MI.getMF()->getRegInfo().getVRegDef(MI.getOperand(1).getReg());
1326
1327 const unsigned Rank = VecTypeDef->getOperand(2).getImm();
1328 if (Rank != 2 && Rank != 4)
1329 reportFatalUsageError("Result type of an atomic vector float instruction "
1330 "must be a 2-component or 4 component vector");
1331
1332 SPIRVTypeInst EltTypeDef =
1333 MI.getMF()->getRegInfo().getVRegDef(VecTypeDef->getOperand(1).getReg());
1334
1335 if (EltTypeDef->getOpcode() != SPIRV::OpTypeFloat ||
1336 EltTypeDef->getOperand(1).getImm() != 16)
1338 "The element type for the result type of an atomic vector float "
1339 "instruction must be a 16-bit floating-point scalar");
1340
1341 // The extension is defined for fp16, but the AMD target lets a bf16 vector
1342 // use the same instruction so it can lower to a packed bf16 atomic.
1343 if (isBFloat16Type(EltTypeDef) &&
1344 ST.getTargetTriple().getVendor() != Triple::AMD)
1346 "The element type for the result type of an atomic vector float "
1347 "instruction cannot be a bfloat16 scalar");
1348 if (!ST.canUseExtension(SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector))
1350 "The atomic float16 vector instruction requires the following SPIR-V "
1351 "extension: SPV_NV_shader_atomic_fp16_vector");
1352
1353 Reqs.addExtension(SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector);
1354 Reqs.addCapability(SPIRV::Capability::AtomicFloat16VectorNV);
1355}
1356
1359 const SPIRVSubtarget &ST) {
1360 assert(MI.getOperand(1).isReg() &&
1361 "Expect register operand in atomic float instruction");
1362 Register TypeReg = MI.getOperand(1).getReg();
1363 SPIRVTypeInst TypeDef = MI.getMF()->getRegInfo().getVRegDef(TypeReg);
1364
1365 if (isVectorType(TypeDef))
1366 return AddAtomicVectorFloatRequirements(MI, Reqs, ST);
1367
1368 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)
1369 report_fatal_error("Result type of an atomic float instruction must be a "
1370 "floating-point type scalar");
1371
1372 unsigned BitWidth = TypeDef->getOperand(1).getImm();
1373 unsigned Op = MI.getOpcode();
1374 if (Op == SPIRV::OpAtomicFAddEXT) {
1375 if (!ST.canUseExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_add))
1377 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_add);
1378 switch (BitWidth) {
1379 case 16:
1380 if (isBFloat16Type(TypeDef)) {
1381 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))
1383 "The atomic bfloat16 instruction requires the following SPIR-V "
1384 "extension: SPV_INTEL_16bit_atomics",
1385 false);
1386 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);
1387 Reqs.addCapability(SPIRV::Capability::AtomicBFloat16AddINTEL);
1388 } else {
1389 if (!ST.canUseExtension(
1390 SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))
1391 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("16_add"), false);
1392 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);
1393 Reqs.addCapability(SPIRV::Capability::AtomicFloat16AddEXT);
1394 }
1395 break;
1396 case 32:
1397 Reqs.addCapability(SPIRV::Capability::AtomicFloat32AddEXT);
1398 break;
1399 case 64:
1400 Reqs.addCapability(SPIRV::Capability::AtomicFloat64AddEXT);
1401 break;
1402 default:
1404 "Unexpected floating-point type width in atomic float instruction");
1405 }
1406 } else {
1407 if (!ST.canUseExtension(
1408 SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))
1409 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_min_max"), false);
1410 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);
1411 switch (BitWidth) {
1412 case 16:
1413 if (isBFloat16Type(TypeDef)) {
1414 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))
1416 "The atomic bfloat16 instruction requires the following SPIR-V "
1417 "extension: SPV_INTEL_16bit_atomics",
1418 false);
1419 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);
1420 Reqs.addCapability(SPIRV::Capability::AtomicBFloat16MinMaxINTEL);
1421 } else {
1422 Reqs.addCapability(SPIRV::Capability::AtomicFloat16MinMaxEXT);
1423 }
1424 break;
1425 case 32:
1426 Reqs.addCapability(SPIRV::Capability::AtomicFloat32MinMaxEXT);
1427 break;
1428 case 64:
1429 Reqs.addCapability(SPIRV::Capability::AtomicFloat64MinMaxEXT);
1430 break;
1431 default:
1433 "Unexpected floating-point type width in atomic float instruction");
1434 }
1435 }
1436}
1437
1439 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1440 return false;
1441 uint32_t Dim = ImageInst->getOperand(2).getImm();
1442 uint32_t Sampled = ImageInst->getOperand(6).getImm();
1443 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
1444}
1445
1447 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1448 return false;
1449 uint32_t Dim = ImageInst->getOperand(2).getImm();
1450 uint32_t Sampled = ImageInst->getOperand(6).getImm();
1451 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
1452}
1453
1455 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1456 return false;
1457 uint32_t Dim = ImageInst->getOperand(2).getImm();
1458 uint32_t Sampled = ImageInst->getOperand(6).getImm();
1459 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
1460}
1461
1463 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1464 return false;
1465 uint32_t Dim = ImageInst->getOperand(2).getImm();
1466 uint32_t Sampled = ImageInst->getOperand(6).getImm();
1467 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
1468}
1469
1471 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1472 return false;
1473 uint32_t Dim = ImageInst->getOperand(2).getImm();
1474 uint32_t Sampled = ImageInst->getOperand(6).getImm();
1475 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
1476}
1477
1478bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
1479 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
1480 return false;
1481
1482 const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();
1483 Register ImageReg = SampledImageInst->getOperand(1).getReg();
1484 auto *ImageInst = MRI.getUniqueVRegDef(ImageReg);
1485 return isSampledImage(ImageInst);
1486}
1487
1489 for (const auto &MI : MRI.reg_instructions(Reg)) {
1490 if (MI.getOpcode() != SPIRV::OpDecorate)
1491 continue;
1492
1493 uint32_t Dec = MI.getOperand(1).getImm();
1494 if (Dec == SPIRV::Decoration::NonUniformEXT)
1495 return true;
1496 }
1497 return false;
1498}
1499
1502 const SPIRVSubtarget &Subtarget) {
1503 const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();
1504 // Get the result type. If it is an image type, then the shader uses
1505 // descriptor indexing. The appropriate capabilities will be added based
1506 // on the specifics of the image.
1507 Register ResTypeReg = Instr.getOperand(1).getReg();
1508 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(ResTypeReg);
1509
1510 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
1511 uint32_t StorageClass = ResTypeInst->getOperand(1).getImm();
1512 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
1513 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
1514 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
1515 return;
1516 }
1517
1518 bool IsNonUniform =
1519 hasNonUniformDecoration(Instr.getOperand(0).getReg(), MRI);
1520
1521 auto FirstIndexReg = Instr.getOperand(3).getReg();
1522 bool FirstIndexIsConstant =
1523 Subtarget.getInstrInfo()->isConstantInstr(*MRI.getVRegDef(FirstIndexReg));
1524
1525 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {
1526 if (IsNonUniform)
1527 Handler.addRequirements(
1528 SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);
1529 else if (!FirstIndexIsConstant)
1530 Handler.addRequirements(
1531 SPIRV::Capability::StorageBufferArrayDynamicIndexing);
1532 return;
1533 }
1534
1535 Register PointeeTypeReg = ResTypeInst->getOperand(2).getReg();
1536 MachineInstr *PointeeType = MRI.getUniqueVRegDef(PointeeTypeReg);
1537 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
1538 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
1539 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
1540 return;
1541 }
1542
1543 if (isUniformTexelBuffer(PointeeType)) {
1544 if (IsNonUniform)
1545 Handler.addRequirements(
1546 SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1547 else if (!FirstIndexIsConstant)
1548 Handler.addRequirements(
1549 SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1550 } else if (isInputAttachment(PointeeType)) {
1551 if (IsNonUniform)
1552 Handler.addRequirements(
1553 SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1554 else if (!FirstIndexIsConstant)
1555 Handler.addRequirements(
1556 SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
1557 } else if (isStorageTexelBuffer(PointeeType)) {
1558 if (IsNonUniform)
1559 Handler.addRequirements(
1560 SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1561 else if (!FirstIndexIsConstant)
1562 Handler.addRequirements(
1563 SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1564 } else if (isSampledImage(PointeeType) ||
1565 isCombinedImageSampler(PointeeType) ||
1566 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
1567 if (IsNonUniform)
1568 Handler.addRequirements(
1569 SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
1570 else if (!FirstIndexIsConstant)
1571 Handler.addRequirements(
1572 SPIRV::Capability::SampledImageArrayDynamicIndexing);
1573 } else if (isStorageImage(PointeeType)) {
1574 if (IsNonUniform)
1575 Handler.addRequirements(
1576 SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
1577 else if (!FirstIndexIsConstant)
1578 Handler.addRequirements(
1579 SPIRV::Capability::StorageImageArrayDynamicIndexing);
1580 }
1581}
1582
1584 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)
1585 return false;
1586 assert(TypeInst->getOperand(7).isImm() && "The image format must be an imm.");
1587 return TypeInst->getOperand(7).getImm() == 0;
1588}
1589
1592 const SPIRVSubtarget &ST) {
1593 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product))
1594 Reqs.addExtension(SPIRV::Extension::SPV_KHR_integer_dot_product);
1595 Reqs.addCapability(SPIRV::Capability::DotProduct);
1596
1597 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1598 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");
1599 SPIRVTypeInst TypeDef = getRegType(MI.getOperand(2).getReg(), *MI.getMF(),
1600 ST.getSPIRVGlobalRegistry());
1601 assert(TypeDef && "Unexpected type instruction for dot input");
1602 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1603 assert(TypeDef->getOperand(1).getImm() == 32);
1604 Reqs.addCapability(SPIRV::Capability::DotProductInput4x8BitPacked);
1605 } else if (isVectorType(TypeDef)) {
1606 SPIRVTypeInst ScalarTypeDef =
1607 MRI.getVRegDef(TypeDef->getOperand(1).getReg());
1608 assert(ScalarTypeDef->getOpcode() == SPIRV::OpTypeInt);
1609 if (ScalarTypeDef->getOperand(1).getImm() == 8) {
1610 assert(TypeDef->getOperand(2).getImm() == 4 &&
1611 "Dot operand of 8-bit integer type requires 4 components");
1612 Reqs.addCapability(SPIRV::Capability::DotProductInput4x8Bit);
1613 } else {
1614 Reqs.addCapability(SPIRV::Capability::DotProductInputAll);
1615 }
1616 }
1617}
1618
1621 const SPIRVSubtarget &ST) {
1622 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1623 SPIRVTypeInst PtrType =
1624 GR->getSPIRVTypeForVReg(MI.getOperand(4).getReg(), MI.getMF());
1625 if (PtrType) {
1626 MachineOperand ASOp = PtrType->getOperand(1);
1627 if (ASOp.isImm()) {
1628 unsigned AddrSpace = ASOp.getImm();
1629 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {
1630 if (!ST.canUseExtension(
1632 SPV_EXT_relaxed_printf_string_address_space)) {
1633 report_fatal_error("SPV_EXT_relaxed_printf_string_address_space is "
1634 "required because printf uses a format string not "
1635 "in constant address space.",
1636 false);
1637 }
1638 Reqs.addExtension(
1639 SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);
1640 }
1641 }
1642 }
1643}
1644
1647 const SPIRVSubtarget &ST, unsigned OpIdx) {
1648 if (MI.getNumOperands() <= OpIdx)
1649 return;
1650 uint32_t Mask = MI.getOperand(OpIdx).getImm();
1651 for (uint32_t I = 0; I < 32; ++I)
1652 if (Mask & (1U << I))
1653 Reqs.getAndAddRequirements(SPIRV::OperandCategory::ImageOperandOperand,
1654 1U << I, ST);
1655}
1656
1657static inline void maybeAddScatterGatherReq(const MachineInstr &MI,
1659 const SPIRVSubtarget &ST) {
1660 assert(MI.getOperand(1).isReg());
1661 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1662 SPIRVTypeInst ElemTypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
1663 if (ElemTypeDef->getOpcode() == SPIRV::OpTypePointer &&
1664 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
1665 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
1666 Reqs.addCapability(SPIRV::Capability::MaskedGatherScatterINTEL);
1667 }
1668}
1669
1672 const SPIRVSubtarget &ST) {
1673 SPIRV::RequirementHandler &Reqs = MAI.Reqs;
1674 unsigned Op = MI.getOpcode();
1675 switch (Op) {
1676 case SPIRV::OpMemoryModel: {
1677 int64_t Addr = MI.getOperand(0).getImm();
1678 Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand,
1679 Addr, ST);
1680 int64_t Mem = MI.getOperand(1).getImm();
1681 Reqs.getAndAddRequirements(SPIRV::OperandCategory::MemoryModelOperand, Mem,
1682 ST);
1683 break;
1684 }
1685 case SPIRV::OpEntryPoint: {
1686 int64_t Exe = MI.getOperand(0).getImm();
1687 Reqs.getAndAddRequirements(SPIRV::OperandCategory::ExecutionModelOperand,
1688 Exe, ST);
1689 break;
1690 }
1691 case SPIRV::OpExecutionMode:
1692 case SPIRV::OpExecutionModeId: {
1693 int64_t Exe = MI.getOperand(1).getImm();
1694 Reqs.getAndAddRequirements(SPIRV::OperandCategory::ExecutionModeOperand,
1695 Exe, ST);
1696 break;
1697 }
1698 case SPIRV::OpTypeMatrix:
1699 Reqs.addCapability(SPIRV::Capability::Matrix);
1700 break;
1701 case SPIRV::OpTypeInt: {
1702 unsigned BitWidth = MI.getOperand(1).getImm();
1703 if (BitWidth == 64)
1704 Reqs.addCapability(SPIRV::Capability::Int64);
1705 else if (BitWidth == 16)
1706 Reqs.addCapability(SPIRV::Capability::Int16);
1707 else if (BitWidth == 8)
1708 Reqs.addCapability(SPIRV::Capability::Int8);
1709 else if (BitWidth == 4 &&
1710 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4)) {
1711 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_int4);
1712 Reqs.addCapability(SPIRV::Capability::Int4TypeINTEL);
1713 } else if (BitWidth != 32) {
1714 if (!ST.canUseExtension(
1715 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers))
1717 "OpTypeInt type with a width other than 8, 16, 32 or 64 bits "
1718 "requires the following SPIR-V extension: "
1719 "SPV_ALTERA_arbitrary_precision_integers");
1720 Reqs.addExtension(
1721 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
1722 Reqs.addCapability(SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
1723 }
1724 break;
1725 }
1726 case SPIRV::OpDot: {
1727 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1728 SPIRVTypeInst TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
1729 if (isBFloat16Type(TypeDef))
1730 Reqs.addCapability(SPIRV::Capability::BFloat16DotProductKHR);
1731 break;
1732 }
1733 case SPIRV::OpTypeFloat: {
1734 unsigned BitWidth = MI.getOperand(1).getImm();
1735 if (BitWidth == 64)
1736 Reqs.addCapability(SPIRV::Capability::Float64);
1737 else if (BitWidth == 16) {
1738 if (isBFloat16Type(&MI)) {
1739 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_bfloat16))
1740 report_fatal_error("OpTypeFloat type with bfloat requires the "
1741 "following SPIR-V extension: SPV_KHR_bfloat16",
1742 false);
1743 Reqs.addExtension(SPIRV::Extension::SPV_KHR_bfloat16);
1744 Reqs.addCapability(SPIRV::Capability::BFloat16TypeKHR);
1745 } else {
1746 Reqs.addCapability(SPIRV::Capability::Float16);
1747 }
1748 }
1749 break;
1750 }
1751 case SPIRV::OpTypeVector: {
1752 unsigned NumComponents = MI.getOperand(2).getImm();
1753 if (NumComponents == 8 || NumComponents == 16)
1754 Reqs.addCapability(SPIRV::Capability::Vector16);
1755 else if (requiresLongVectorEXT(NumComponents))
1756 // Such widths are only expressible as OpTypeVectorIdEXT.
1758 "OpTypeVector with " + Twine(NumComponents) +
1759 " components requires the following SPIR-V extension: "
1760 "SPV_EXT_long_vector");
1761
1762 maybeAddScatterGatherReq(MI, Reqs, ST);
1763 break;
1764 }
1765 case SPIRV::OpTypeVectorIdEXT: {
1766 if (!ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))
1767 reportFatalUsageError("OpTypeVectorIdEXT requires the following SPIR-V "
1768 "extension: SPV_EXT_long_vector extension");
1769 Reqs.addExtension(SPIRV::Extension::SPV_EXT_long_vector);
1770 Reqs.addCapability(SPIRV::Capability::LongVectorEXT);
1771 maybeAddScatterGatherReq(MI, Reqs, ST);
1772 break;
1773 }
1774 case SPIRV::OpTypePointer: {
1775 auto SC = MI.getOperand(1).getImm();
1776 Reqs.getAndAddRequirements(SPIRV::OperandCategory::StorageClassOperand, SC,
1777 ST);
1778 // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer
1779 // capability.
1780 if (ST.isShader())
1781 break;
1782 assert(MI.getOperand(2).isReg());
1783 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1784 SPIRVTypeInst TypeDef = MRI.getVRegDef(MI.getOperand(2).getReg());
1785 if ((TypeDef->getNumOperands() == 2) &&
1786 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&
1787 (TypeDef->getOperand(1).getImm() == 16))
1788 Reqs.addCapability(SPIRV::Capability::Float16Buffer);
1789 break;
1790 }
1791 case SPIRV::OpExtInst: {
1792 if (MI.getOperand(2).getImm() ==
1793 static_cast<int64_t>(
1794 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {
1795 Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);
1796 break;
1797 }
1798 if (MI.getOperand(2).getImm() ==
1799 static_cast<int64_t>(SPIRV::InstructionSet::NonSemantic_DebugBreak)) {
1800 // Non-semantic instruction sets no longer require
1801 // SPV_KHR_non_semantic_info as of SPIR-V 1.6
1802 if (!ST.isAtLeastSPIRVVer(VersionTuple(1, 6)))
1803 Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);
1804 break;
1805 }
1806 if (MI.getOperand(3).getImm() ==
1807 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {
1808 addPrintfRequirements(MI, Reqs, ST);
1809 break;
1810 }
1811 if (MI.getOperand(2).getImm() ==
1812 static_cast<int64_t>(SPIRV::InstructionSet::OpenCL_std)) {
1813 const MachineFunction *MF = MI.getMF();
1814 const MachineRegisterInfo &MRI = MF->getRegInfo();
1815 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1816
1817 auto IsBFloat16 = [&](SPIRVTypeInst TypeDef) {
1818 if (TypeDef && TypeDef->getOpcode() == SPIRV::OpTypeVector)
1819 TypeDef = MRI.getVRegDef(TypeDef->getOperand(1).getReg());
1820 return isBFloat16Type(TypeDef);
1821 };
1822
1823 // Result type is operand 1; arguments start at operand 4.
1824 bool UsesBFloat16 = IsBFloat16(MRI.getVRegDef(MI.getOperand(1).getReg()));
1825 for (unsigned I = 4, E = MI.getNumOperands(); I < E && !UsesBFloat16;
1826 ++I) {
1827 const MachineOperand &MO = MI.getOperand(I);
1828 if (MO.isReg())
1829 UsesBFloat16 = IsBFloat16(GR->getResultType(
1830 MO.getReg(), const_cast<MachineFunction *>(MF)));
1831 }
1832
1833 if (UsesBFloat16) {
1834 if (!ST.canUseExtension(
1835 SPIRV::Extension::SPV_EXT_bfloat16_arithmetic)) {
1837 MI, "OpenCL Extended instructions with bfloat16 require the "
1838 "following SPIR-V extension: SPV_EXT_bfloat16_arithmetic");
1839 break;
1840 }
1841 Reqs.addExtension(SPIRV::Extension::SPV_EXT_bfloat16_arithmetic);
1842 Reqs.addCapability(SPIRV::Capability::BFloat16ArithmeticEXT);
1843 }
1844 }
1845 break;
1846 }
1847 case SPIRV::OpAliasDomainDeclINTEL:
1848 case SPIRV::OpAliasScopeDeclINTEL:
1849 case SPIRV::OpAliasScopeListDeclINTEL: {
1850 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing);
1851 Reqs.addCapability(SPIRV::Capability::MemoryAccessAliasingINTEL);
1852 break;
1853 }
1854 case SPIRV::OpBitReverse:
1855 case SPIRV::OpBitFieldInsert:
1856 case SPIRV::OpBitFieldSExtract:
1857 case SPIRV::OpBitFieldUExtract:
1858 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions)) {
1859 Reqs.addCapability(SPIRV::Capability::Shader);
1860 break;
1861 }
1862 Reqs.addExtension(SPIRV::Extension::SPV_KHR_bit_instructions);
1863 Reqs.addCapability(SPIRV::Capability::BitInstructions);
1864 break;
1865 case SPIRV::OpTypeRuntimeArray:
1866 Reqs.addCapability(SPIRV::Capability::Shader);
1867 break;
1868 case SPIRV::OpTypeOpaque:
1869 case SPIRV::OpTypeEvent:
1870 Reqs.addCapability(SPIRV::Capability::Kernel);
1871 break;
1872 case SPIRV::OpTypePipe:
1873 case SPIRV::OpTypeReserveId:
1874 Reqs.addCapability(SPIRV::Capability::Pipes);
1875 break;
1876 case SPIRV::OpTypePipeStorage:
1877 Reqs.addCapability(SPIRV::Capability::PipeStorage);
1878 break;
1879 case SPIRV::OpTypeDeviceEvent:
1880 case SPIRV::OpTypeQueue:
1881 case SPIRV::OpBuildNDRange:
1882 case SPIRV::OpEnqueueKernel:
1883 case SPIRV::OpGetKernelNDrangeSubGroupCount:
1884 case SPIRV::OpGetKernelNDrangeMaxSubGroupSize:
1885 case SPIRV::OpGetKernelWorkGroupSize:
1886 case SPIRV::OpGetKernelPreferredWorkGroupSizeMultiple:
1887 Reqs.addCapability(SPIRV::Capability::DeviceEnqueue);
1888 break;
1889 case SPIRV::OpDecorate:
1890 case SPIRV::OpDecorateId:
1891 case SPIRV::OpDecorateString:
1892 addOpDecorateReqs(MI, 1, Reqs, ST);
1893 break;
1894 case SPIRV::OpMemberDecorate:
1895 case SPIRV::OpMemberDecorateString:
1896 addOpDecorateReqs(MI, 2, Reqs, ST);
1897 break;
1898 case SPIRV::OpInBoundsPtrAccessChain:
1899 Reqs.addCapability(SPIRV::Capability::Addresses);
1900 break;
1901 case SPIRV::OpConstantSampler:
1902 Reqs.addCapability(SPIRV::Capability::LiteralSampler);
1903 break;
1904 case SPIRV::OpInBoundsAccessChain:
1905 case SPIRV::OpAccessChain:
1906 addOpAccessChainReqs(MI, Reqs, ST);
1907 break;
1908 case SPIRV::OpTypeImage:
1909 addOpTypeImageReqs(MI, Reqs, ST);
1910 break;
1911 case SPIRV::OpTypeSampler:
1912 if (!ST.isShader()) {
1913 Reqs.addCapability(SPIRV::Capability::ImageBasic);
1914 }
1915 break;
1916 case SPIRV::OpTypeForwardPointer:
1917 // TODO: check if it's OpenCL's kernel.
1918 Reqs.addCapability(SPIRV::Capability::Addresses);
1919 break;
1920 case SPIRV::OpAtomicFlagTestAndSet:
1921 case SPIRV::OpAtomicLoad:
1922 case SPIRV::OpAtomicStore:
1923 case SPIRV::OpAtomicExchange:
1924 case SPIRV::OpAtomicCompareExchange:
1925 case SPIRV::OpAtomicCompareExchangeWeak:
1926 case SPIRV::OpAtomicIIncrement:
1927 case SPIRV::OpAtomicIDecrement:
1928 case SPIRV::OpAtomicIAdd:
1929 case SPIRV::OpAtomicISub:
1930 case SPIRV::OpAtomicUMin:
1931 case SPIRV::OpAtomicUMax:
1932 case SPIRV::OpAtomicSMin:
1933 case SPIRV::OpAtomicSMax:
1934 case SPIRV::OpAtomicAnd:
1935 case SPIRV::OpAtomicOr:
1936 case SPIRV::OpAtomicXor: {
1937 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1938 SPIRVTypeInst TypeDef;
1939 if (Op == SPIRV::OpAtomicStore) {
1940 assert(MI.getOperand(3).isReg());
1941 TypeDef = getRegType(MI.getOperand(3).getReg(), *MI.getMF(),
1942 ST.getSPIRVGlobalRegistry());
1943 assert(TypeDef && "Unexpected type instruction for OpAtomicStore");
1944 } else {
1945 assert(MI.getOperand(1).isReg() && "Unexpected operand in atomic");
1946 TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
1947 }
1948
1949 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1950 unsigned BitWidth = TypeDef->getOperand(1).getImm();
1951 if (BitWidth == 64)
1952 Reqs.addCapability(SPIRV::Capability::Int64Atomics);
1953 else if (BitWidth == 16) {
1954 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))
1956 "16-bit integer atomic operations require the following SPIR-V "
1957 "extension: SPV_INTEL_16bit_atomics",
1958 false);
1959 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);
1960 switch (Op) {
1961 case SPIRV::OpAtomicLoad:
1962 case SPIRV::OpAtomicStore:
1963 case SPIRV::OpAtomicExchange:
1964 case SPIRV::OpAtomicCompareExchange:
1965 case SPIRV::OpAtomicCompareExchangeWeak:
1966 Reqs.addCapability(
1967 SPIRV::Capability::AtomicInt16CompareExchangeINTEL);
1968 break;
1969 default:
1970 Reqs.addCapability(SPIRV::Capability::Int16AtomicsINTEL);
1971 break;
1972 }
1973 }
1974 } else if (isBFloat16Type(TypeDef)) {
1975 if (is_contained({SPIRV::OpAtomicLoad, SPIRV::OpAtomicStore,
1976 SPIRV::OpAtomicExchange},
1977 Op)) {
1978 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))
1980 "The atomic bfloat16 instruction requires the following SPIR-V "
1981 "extension: SPV_INTEL_16bit_atomics",
1982 false);
1983 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);
1984 Reqs.addCapability(SPIRV::Capability::AtomicBFloat16LoadStoreINTEL);
1985 }
1986 }
1987 break;
1988 }
1989 case SPIRV::OpGroupNonUniformIAdd:
1990 case SPIRV::OpGroupNonUniformFAdd:
1991 case SPIRV::OpGroupNonUniformIMul:
1992 case SPIRV::OpGroupNonUniformFMul:
1993 case SPIRV::OpGroupNonUniformSMin:
1994 case SPIRV::OpGroupNonUniformUMin:
1995 case SPIRV::OpGroupNonUniformFMin:
1996 case SPIRV::OpGroupNonUniformSMax:
1997 case SPIRV::OpGroupNonUniformUMax:
1998 case SPIRV::OpGroupNonUniformFMax:
1999 case SPIRV::OpGroupNonUniformBitwiseAnd:
2000 case SPIRV::OpGroupNonUniformBitwiseOr:
2001 case SPIRV::OpGroupNonUniformBitwiseXor:
2002 case SPIRV::OpGroupNonUniformLogicalAnd:
2003 case SPIRV::OpGroupNonUniformLogicalOr:
2004 case SPIRV::OpGroupNonUniformLogicalXor: {
2005 assert(MI.getOperand(3).isImm());
2006 int64_t GroupOp = MI.getOperand(3).getImm();
2007 switch (GroupOp) {
2008 case SPIRV::GroupOperation::Reduce:
2009 case SPIRV::GroupOperation::InclusiveScan:
2010 case SPIRV::GroupOperation::ExclusiveScan:
2011 Reqs.addCapability(SPIRV::Capability::GroupNonUniformArithmetic);
2012 break;
2013 case SPIRV::GroupOperation::ClusteredReduce:
2014 Reqs.addCapability(SPIRV::Capability::GroupNonUniformClustered);
2015 break;
2016 case SPIRV::GroupOperation::PartitionedReduceNV:
2017 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:
2018 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:
2019 if (ST.canUseExtension(
2020 SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned))
2021 Reqs.addExtension(
2022 SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned);
2023 else
2024 Reqs.addExtension(SPIRV::Extension::SPV_NV_shader_subgroup_partitioned);
2025 Reqs.addCapability(SPIRV::Capability::GroupNonUniformPartitionedNV);
2026 break;
2027 }
2028 break;
2029 }
2030 case SPIRV::OpGroupNonUniformQuadSwap:
2031 Reqs.addCapability(SPIRV::Capability::GroupNonUniformQuad);
2032 break;
2033 case SPIRV::OpImageQueryLod:
2034 Reqs.addCapability(SPIRV::Capability::ImageQuery);
2035 break;
2036 case SPIRV::OpImageQuerySize:
2037 case SPIRV::OpImageQuerySizeLod:
2038 case SPIRV::OpImageQueryLevels:
2039 case SPIRV::OpImageQuerySamples:
2040 if (ST.isShader())
2041 Reqs.addCapability(SPIRV::Capability::ImageQuery);
2042 break;
2043 case SPIRV::OpImageQueryFormat: {
2044 Register ResultReg = MI.getOperand(0).getReg();
2045 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2046 static const unsigned CompareOps[] = {
2047 SPIRV::OpIEqual, SPIRV::OpINotEqual,
2048 SPIRV::OpUGreaterThan, SPIRV::OpUGreaterThanEqual,
2049 SPIRV::OpULessThan, SPIRV::OpULessThanEqual,
2050 SPIRV::OpSGreaterThan, SPIRV::OpSGreaterThanEqual,
2051 SPIRV::OpSLessThan, SPIRV::OpSLessThanEqual};
2052
2053 auto CheckAndAddExtension = [&](int64_t ImmVal) {
2054 if (ImmVal == 4323 || ImmVal == 4324) {
2055 if (ST.canUseExtension(SPIRV::Extension::SPV_EXT_image_raw10_raw12))
2056 Reqs.addExtension(SPIRV::Extension::SPV_EXT_image_raw10_raw12);
2057 else
2058 report_fatal_error("This requires the "
2059 "SPV_EXT_image_raw10_raw12 extension");
2060 }
2061 };
2062
2063 for (MachineInstr &UseInst : MRI.use_instructions(ResultReg)) {
2064 unsigned Opc = UseInst.getOpcode();
2065
2066 if (Opc == SPIRV::OpSwitch) {
2067 for (const MachineOperand &Op : UseInst.operands())
2068 if (Op.isImm())
2069 CheckAndAddExtension(Op.getImm());
2070 } else if (llvm::is_contained(CompareOps, Opc)) {
2071 for (unsigned i = 1; i < UseInst.getNumOperands(); ++i) {
2072 Register UseReg = UseInst.getOperand(i).getReg();
2073 MachineInstr *ConstInst = MRI.getVRegDef(UseReg);
2074 if (ConstInst && ConstInst->getOpcode() == SPIRV::OpConstantI) {
2075 int64_t ImmVal = ConstInst->getOperand(2).getImm();
2076 if (ImmVal)
2077 CheckAndAddExtension(ImmVal);
2078 }
2079 }
2080 }
2081 }
2082 break;
2083 }
2084
2085 case SPIRV::OpGroupNonUniformShuffle:
2086 case SPIRV::OpGroupNonUniformShuffleXor:
2087 Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffle);
2088 break;
2089 case SPIRV::OpGroupNonUniformShuffleUp:
2090 case SPIRV::OpGroupNonUniformShuffleDown:
2091 Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffleRelative);
2092 break;
2093 case SPIRV::OpGroupAll:
2094 case SPIRV::OpGroupAny:
2095 case SPIRV::OpGroupBroadcast:
2096 case SPIRV::OpGroupIAdd:
2097 case SPIRV::OpGroupFAdd:
2098 case SPIRV::OpGroupFMin:
2099 case SPIRV::OpGroupUMin:
2100 case SPIRV::OpGroupSMin:
2101 case SPIRV::OpGroupFMax:
2102 case SPIRV::OpGroupUMax:
2103 case SPIRV::OpGroupSMax:
2104 Reqs.addCapability(SPIRV::Capability::Groups);
2105 break;
2106 case SPIRV::OpGroupNonUniformElect:
2107 Reqs.addCapability(SPIRV::Capability::GroupNonUniform);
2108 break;
2109 case SPIRV::OpGroupNonUniformAll:
2110 case SPIRV::OpGroupNonUniformAny:
2111 case SPIRV::OpGroupNonUniformAllEqual:
2112 Reqs.addCapability(SPIRV::Capability::GroupNonUniformVote);
2113 break;
2114 case SPIRV::OpGroupNonUniformBroadcast:
2115 case SPIRV::OpGroupNonUniformBroadcastFirst:
2116 case SPIRV::OpGroupNonUniformBallot:
2117 case SPIRV::OpGroupNonUniformInverseBallot:
2118 case SPIRV::OpGroupNonUniformBallotBitExtract:
2119 case SPIRV::OpGroupNonUniformBallotBitCount:
2120 case SPIRV::OpGroupNonUniformBallotFindLSB:
2121 case SPIRV::OpGroupNonUniformBallotFindMSB:
2122 Reqs.addCapability(SPIRV::Capability::GroupNonUniformBallot);
2123 break;
2124 case SPIRV::OpSubgroupShuffleINTEL:
2125 case SPIRV::OpSubgroupShuffleDownINTEL:
2126 case SPIRV::OpSubgroupShuffleUpINTEL:
2127 case SPIRV::OpSubgroupShuffleXorINTEL:
2128 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
2129 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);
2130 Reqs.addCapability(SPIRV::Capability::SubgroupShuffleINTEL);
2131 }
2132 break;
2133 case SPIRV::OpSubgroupBlockReadINTEL:
2134 case SPIRV::OpSubgroupBlockWriteINTEL:
2135 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
2136 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);
2137 Reqs.addCapability(SPIRV::Capability::SubgroupBufferBlockIOINTEL);
2138 }
2139 break;
2140 case SPIRV::OpSubgroupImageBlockReadINTEL:
2141 case SPIRV::OpSubgroupImageBlockWriteINTEL:
2142 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
2143 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);
2144 Reqs.addCapability(SPIRV::Capability::SubgroupImageBlockIOINTEL);
2145 }
2146 break;
2147 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
2148 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
2149 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_media_block_io)) {
2150 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_media_block_io);
2151 Reqs.addCapability(SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
2152 }
2153 break;
2154 case SPIRV::OpAssumeTrueKHR:
2155 case SPIRV::OpExpectKHR:
2156 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
2157 Reqs.addExtension(SPIRV::Extension::SPV_KHR_expect_assume);
2158 Reqs.addCapability(SPIRV::Capability::ExpectAssumeKHR);
2159 }
2160 break;
2161 case SPIRV::OpFmaKHR:
2162 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_fma)) {
2163 Reqs.addExtension(SPIRV::Extension::SPV_KHR_fma);
2164 Reqs.addCapability(SPIRV::Capability::FmaKHR);
2165 }
2166 break;
2167 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:
2168 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:
2169 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {
2170 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes);
2171 Reqs.addCapability(SPIRV::Capability::USMStorageClassesINTEL);
2172 }
2173 break;
2174 case SPIRV::OpConstantFunctionPointerINTEL:
2175 case SPIRV::OpFunctionPointerCallINTEL:
2176 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)) {
2177 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_function_pointers);
2178 Reqs.addCapability(SPIRV::Capability::FunctionPointersINTEL);
2179 }
2180 break;
2181 case SPIRV::OpGroupNonUniformQuadAllKHR:
2182 addQuadControlRequirements(Reqs, ST, "OpGroupNonUniformQuadAllKHR");
2183 Reqs.addCapability(SPIRV::Capability::QuadControlKHR);
2184 break;
2185 case SPIRV::OpGroupNonUniformRotateKHR:
2186 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_subgroup_rotate))
2187 report_fatal_error("OpGroupNonUniformRotateKHR instruction requires the "
2188 "following SPIR-V extension: SPV_KHR_subgroup_rotate",
2189 false);
2190 Reqs.addExtension(SPIRV::Extension::SPV_KHR_subgroup_rotate);
2191 Reqs.addCapability(SPIRV::Capability::GroupNonUniformRotateKHR);
2192 Reqs.addCapability(SPIRV::Capability::GroupNonUniform);
2193 break;
2194 case SPIRV::OpFixedCosALTERA:
2195 case SPIRV::OpFixedSinALTERA:
2196 case SPIRV::OpFixedCosPiALTERA:
2197 case SPIRV::OpFixedSinPiALTERA:
2198 case SPIRV::OpFixedExpALTERA:
2199 case SPIRV::OpFixedLogALTERA:
2200 case SPIRV::OpFixedRecipALTERA:
2201 case SPIRV::OpFixedSqrtALTERA:
2202 case SPIRV::OpFixedSinCosALTERA:
2203 case SPIRV::OpFixedSinCosPiALTERA:
2204 case SPIRV::OpFixedRsqrtALTERA:
2205 if (!ST.canUseExtension(
2206 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))
2207 report_fatal_error("This instruction requires the "
2208 "following SPIR-V extension: "
2209 "SPV_ALTERA_arbitrary_precision_fixed_point",
2210 false);
2211 Reqs.addExtension(
2212 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);
2213 Reqs.addCapability(SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);
2214 break;
2215 case SPIRV::OpGroupIMulKHR:
2216 case SPIRV::OpGroupFMulKHR:
2217 case SPIRV::OpGroupBitwiseAndKHR:
2218 case SPIRV::OpGroupBitwiseOrKHR:
2219 case SPIRV::OpGroupBitwiseXorKHR:
2220 case SPIRV::OpGroupLogicalAndKHR:
2221 case SPIRV::OpGroupLogicalOrKHR:
2222 case SPIRV::OpGroupLogicalXorKHR:
2223 if (ST.canUseExtension(
2224 SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
2225 Reqs.addExtension(SPIRV::Extension::SPV_KHR_uniform_group_instructions);
2226 Reqs.addCapability(SPIRV::Capability::GroupUniformArithmeticKHR);
2227 }
2228 break;
2229 case SPIRV::OpReadClockKHR:
2230 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_shader_clock))
2231 report_fatal_error("OpReadClockKHR instruction requires the "
2232 "following SPIR-V extension: SPV_KHR_shader_clock",
2233 false);
2234 Reqs.addExtension(SPIRV::Extension::SPV_KHR_shader_clock);
2235 Reqs.addCapability(SPIRV::Capability::ShaderClockKHR);
2236 break;
2237 case SPIRV::OpAbortKHR:
2238 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_abort))
2239 report_fatal_error("OpAbortKHR instruction requires the "
2240 "following SPIR-V extension: SPV_KHR_abort",
2241 false);
2242 Reqs.addExtension(SPIRV::Extension::SPV_KHR_abort);
2243 Reqs.addCapability(SPIRV::Capability::AbortKHR);
2244 break;
2245 case SPIRV::OpPoisonKHR:
2246 case SPIRV::OpFreezeKHR:
2247 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_poison_freeze))
2248 report_fatal_error("OpPoisonKHR/OpFreezeKHR instruction requires the "
2249 "following SPIR-V extension: SPV_KHR_poison_freeze",
2250 false);
2251 Reqs.addExtension(SPIRV::Extension::SPV_KHR_poison_freeze);
2252 Reqs.addCapability(SPIRV::Capability::PoisonFreezeKHR);
2253 break;
2254 case SPIRV::OpAtomicFAddEXT:
2255 case SPIRV::OpAtomicFMinEXT:
2256 case SPIRV::OpAtomicFMaxEXT:
2257 AddAtomicFloatRequirements(MI, Reqs, ST);
2258 break;
2259 case SPIRV::OpConvertBF16ToFINTEL:
2260 case SPIRV::OpConvertFToBF16INTEL:
2261 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
2262 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bfloat16_conversion);
2263 Reqs.addCapability(SPIRV::Capability::BFloat16ConversionINTEL);
2264 }
2265 break;
2266 case SPIRV::OpRoundFToTF32INTEL:
2267 if (ST.canUseExtension(
2268 SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {
2269 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);
2270 Reqs.addCapability(SPIRV::Capability::TensorFloat32RoundingINTEL);
2271 }
2272 break;
2273 case SPIRV::OpVariableLengthArrayINTEL:
2274 case SPIRV::OpSaveMemoryINTEL:
2275 case SPIRV::OpRestoreMemoryINTEL:
2276 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2277 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_variable_length_array);
2278 Reqs.addCapability(SPIRV::Capability::VariableLengthArrayINTEL);
2279 }
2280 break;
2281 case SPIRV::OpUntypedVariableLengthArrayINTEL:
2282 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2283 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_variable_length_array);
2284 Reqs.addCapability(SPIRV::Capability::UntypedVariableLengthArrayINTEL);
2285 }
2286 break;
2287 case SPIRV::OpAsmTargetINTEL:
2288 case SPIRV::OpAsmINTEL:
2289 case SPIRV::OpAsmCallINTEL:
2290 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_inline_assembly)) {
2291 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_inline_assembly);
2292 Reqs.addCapability(SPIRV::Capability::AsmINTEL);
2293 }
2294 break;
2295 case SPIRV::OpTypeCooperativeMatrixKHR: {
2296 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
2298 "OpTypeCooperativeMatrixKHR type requires the "
2299 "following SPIR-V extension: SPV_KHR_cooperative_matrix",
2300 false);
2301 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
2302 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
2303 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2304 SPIRVTypeInst TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
2305 if (isBFloat16Type(TypeDef))
2306 Reqs.addCapability(SPIRV::Capability::BFloat16CooperativeMatrixKHR);
2307 break;
2308 }
2309 case SPIRV::OpArithmeticFenceEXT:
2310 if (!ST.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence))
2311 report_fatal_error("OpArithmeticFenceEXT requires the "
2312 "following SPIR-V extension: SPV_EXT_arithmetic_fence",
2313 false);
2314 Reqs.addExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence);
2315 Reqs.addCapability(SPIRV::Capability::ArithmeticFenceEXT);
2316 break;
2317 case SPIRV::OpControlBarrierArriveINTEL:
2318 case SPIRV::OpControlBarrierWaitINTEL:
2319 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_split_barrier)) {
2320 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_split_barrier);
2321 Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL);
2322 }
2323 break;
2324 case SPIRV::OpCooperativeMatrixMulAddKHR: {
2325 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
2326 report_fatal_error("Cooperative matrix instructions require the "
2327 "following SPIR-V extension: "
2328 "SPV_KHR_cooperative_matrix",
2329 false);
2330 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
2331 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
2332 constexpr unsigned MulAddMaxSize = 6;
2333 if (MI.getNumOperands() != MulAddMaxSize)
2334 break;
2335 const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();
2336 if (CoopOperands &
2337 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
2338 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
2339 report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation "
2340 "require the following SPIR-V extension: "
2341 "SPV_INTEL_joint_matrix",
2342 false);
2343 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
2344 Reqs.addCapability(
2345 SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
2346 }
2347 if (CoopOperands & SPIRV::CooperativeMatrixOperands::
2348 MatrixAAndBBFloat16ComponentsINTEL ||
2349 CoopOperands &
2350 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
2351 CoopOperands & SPIRV::CooperativeMatrixOperands::
2352 MatrixResultBFloat16ComponentsINTEL) {
2353 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
2354 report_fatal_error("***BF16ComponentsINTEL type interpretations "
2355 "require the following SPIR-V extension: "
2356 "SPV_INTEL_joint_matrix",
2357 false);
2358 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
2359 Reqs.addCapability(
2360 SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
2361 }
2362 break;
2363 }
2364 case SPIRV::OpCooperativeMatrixLoadKHR:
2365 case SPIRV::OpCooperativeMatrixStoreKHR:
2366 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2367 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2368 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
2369 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
2370 report_fatal_error("Cooperative matrix instructions require the "
2371 "following SPIR-V extension: "
2372 "SPV_KHR_cooperative_matrix",
2373 false);
2374 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
2375 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
2376
2377 // Check Layout operand in case if it's not a standard one and add the
2378 // appropriate capability.
2379 unsigned LayoutNum;
2380 switch (Op) {
2381 case SPIRV::OpCooperativeMatrixLoadKHR:
2382 LayoutNum = 3;
2383 break;
2384 case SPIRV::OpCooperativeMatrixStoreKHR:
2385 LayoutNum = 2;
2386 break;
2387 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2388 LayoutNum = 5;
2389 break;
2390 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2391 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2392 LayoutNum = 4;
2393 break;
2394 default:
2395 llvm_unreachable("unexpected cooperative matrix opcode");
2396 }
2397 Register RegLayout = MI.getOperand(LayoutNum).getReg();
2398 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2399 MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout);
2400 if (MILayout->getOpcode() == SPIRV::OpConstantI) {
2401 const unsigned LayoutVal = MILayout->getOperand(2).getImm();
2402 if (LayoutVal ==
2403 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
2404 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
2405 report_fatal_error("PackedINTEL layout require the following SPIR-V "
2406 "extension: SPV_INTEL_joint_matrix",
2407 false);
2408 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
2409 Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL);
2410 }
2411 }
2412
2413 // Nothing to do.
2414 if (Op == SPIRV::OpCooperativeMatrixLoadKHR ||
2415 Op == SPIRV::OpCooperativeMatrixStoreKHR)
2416 break;
2417
2418 std::string InstName;
2419 switch (Op) {
2420 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2421 InstName = "OpCooperativeMatrixPrefetchINTEL";
2422 break;
2423 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2424 InstName = "OpCooperativeMatrixLoadCheckedINTEL";
2425 break;
2426 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2427 InstName = "OpCooperativeMatrixStoreCheckedINTEL";
2428 break;
2429 }
2430
2431 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) {
2432 const std::string ErrorMsg =
2433 InstName + " instruction requires the "
2434 "following SPIR-V extension: SPV_INTEL_joint_matrix";
2435 report_fatal_error(ErrorMsg.c_str(), false);
2436 }
2437 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
2438 if (Op == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2439 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
2440 break;
2441 }
2442 Reqs.addCapability(
2443 SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2444 break;
2445 }
2446 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
2447 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
2448 report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL "
2449 "instructions require the following SPIR-V extension: "
2450 "SPV_INTEL_joint_matrix",
2451 false);
2452 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
2453 Reqs.addCapability(
2454 SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2455 break;
2456 case SPIRV::OpReadPipeBlockingALTERA:
2457 case SPIRV::OpWritePipeBlockingALTERA:
2458 if (ST.canUseExtension(SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {
2459 Reqs.addExtension(SPIRV::Extension::SPV_ALTERA_blocking_pipes);
2460 Reqs.addCapability(SPIRV::Capability::BlockingPipesALTERA);
2461 }
2462 break;
2463 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
2464 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
2465 report_fatal_error("OpCooperativeMatrixGetElementCoordINTEL requires the "
2466 "following SPIR-V extension: SPV_INTEL_joint_matrix",
2467 false);
2468 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
2469 Reqs.addCapability(
2470 SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
2471 break;
2472 case SPIRV::OpConvertHandleToImageINTEL:
2473 case SPIRV::OpConvertHandleToSamplerINTEL:
2474 case SPIRV::OpConvertHandleToSampledImageINTEL: {
2475 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bindless_images))
2476 report_fatal_error("OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "
2477 "instructions require the following SPIR-V extension: "
2478 "SPV_INTEL_bindless_images",
2479 false);
2480 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
2481 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;
2482 SPIRVTypeInst TyDef = GR->getSPIRVTypeForVReg(MI.getOperand(1).getReg());
2483 if (Op == SPIRV::OpConvertHandleToImageINTEL &&
2484 TyDef->getOpcode() != SPIRV::OpTypeImage) {
2485 report_fatal_error("Incorrect return type for the instruction "
2486 "OpConvertHandleToImageINTEL",
2487 false);
2488 } else if (Op == SPIRV::OpConvertHandleToSamplerINTEL &&
2489 TyDef->getOpcode() != SPIRV::OpTypeSampler) {
2490 report_fatal_error("Incorrect return type for the instruction "
2491 "OpConvertHandleToSamplerINTEL",
2492 false);
2493 } else if (Op == SPIRV::OpConvertHandleToSampledImageINTEL &&
2494 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {
2495 report_fatal_error("Incorrect return type for the instruction "
2496 "OpConvertHandleToSampledImageINTEL",
2497 false);
2498 }
2499 SPIRVTypeInst SpvTy =
2500 getRegType(MI.getOperand(2).getReg(), *MI.getMF(), GR);
2501 unsigned Bitwidth = GR->getScalarOrVectorBitWidth(SpvTy);
2502 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&
2503 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {
2505 "Parameter value must be a 32-bit scalar in case of "
2506 "Physical32 addressing model or a 64-bit scalar in case of "
2507 "Physical64 addressing model",
2508 false);
2509 }
2510 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bindless_images);
2511 Reqs.addCapability(SPIRV::Capability::BindlessImagesINTEL);
2512 break;
2513 }
2514 case SPIRV::OpSubgroup2DBlockLoadINTEL:
2515 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:
2516 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:
2517 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:
2518 case SPIRV::OpSubgroup2DBlockStoreINTEL: {
2519 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_2d_block_io))
2520 report_fatal_error("OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"
2521 "Prefetch/Store]INTEL instructions require the "
2522 "following SPIR-V extension: SPV_INTEL_2d_block_io",
2523 false);
2524 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_2d_block_io);
2525 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockIOINTEL);
2526
2527 if (Op == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {
2528 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransposeINTEL);
2529 break;
2530 }
2531 if (Op == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {
2532 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransformINTEL);
2533 break;
2534 }
2535 break;
2536 }
2537 case SPIRV::OpKill: {
2538 Reqs.addCapability(SPIRV::Capability::Shader);
2539 } break;
2540 case SPIRV::OpDemoteToHelperInvocation:
2541 Reqs.addCapability(SPIRV::Capability::DemoteToHelperInvocation);
2542
2543 if (ST.canUseExtension(
2544 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {
2545 if (!ST.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6)))
2546 Reqs.addExtension(
2547 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);
2548 }
2549 break;
2550 case SPIRV::OpSDot:
2551 case SPIRV::OpUDot:
2552 case SPIRV::OpSUDot:
2553 case SPIRV::OpSDotAccSat:
2554 case SPIRV::OpUDotAccSat:
2555 case SPIRV::OpSUDotAccSat:
2556 AddDotProductRequirements(MI, Reqs, ST);
2557 break;
2558 case SPIRV::OpImageSampleImplicitLod:
2559 case SPIRV::OpImageFetch:
2560 Reqs.addCapability(SPIRV::Capability::Shader);
2561 addImageOperandReqs(MI, Reqs, ST, 4);
2562 break;
2563 case SPIRV::OpImageSampleExplicitLod:
2564 addImageOperandReqs(MI, Reqs, ST, 4);
2565 break;
2566 case SPIRV::OpImageSampleDrefImplicitLod:
2567 case SPIRV::OpImageSampleDrefExplicitLod:
2568 case SPIRV::OpImageDrefGather:
2569 case SPIRV::OpImageGather:
2570 Reqs.addCapability(SPIRV::Capability::Shader);
2571 addImageOperandReqs(MI, Reqs, ST, 5);
2572 break;
2573 case SPIRV::OpImageRead: {
2574 Register ImageReg = MI.getOperand(2).getReg();
2575 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2576 ImageReg, const_cast<MachineFunction *>(MI.getMF()));
2577 // OpImageRead and OpImageWrite can use Unknown Image Formats
2578 // when the Kernel capability is declared. In the OpenCL environment we are
2579 // not allowed to produce
2580 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2581 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2582
2583 if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())
2584 Reqs.addCapability(SPIRV::Capability::StorageImageReadWithoutFormat);
2585 break;
2586 }
2587 case SPIRV::OpImageWrite: {
2588 Register ImageReg = MI.getOperand(0).getReg();
2589 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2590 ImageReg, const_cast<MachineFunction *>(MI.getMF()));
2591 // OpImageRead and OpImageWrite can use Unknown Image Formats
2592 // when the Kernel capability is declared. In the OpenCL environment we are
2593 // not allowed to produce
2594 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2595 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2596
2597 if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())
2598 Reqs.addCapability(SPIRV::Capability::StorageImageWriteWithoutFormat);
2599 break;
2600 }
2601 case SPIRV::OpTypeStructContinuedINTEL:
2602 case SPIRV::OpConstantCompositeContinuedINTEL:
2603 case SPIRV::OpSpecConstantCompositeContinuedINTEL:
2604 case SPIRV::OpCompositeConstructContinuedINTEL: {
2605 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_long_composites))
2607 "Continued instructions require the "
2608 "following SPIR-V extension: SPV_INTEL_long_composites",
2609 false);
2610 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_long_composites);
2611 Reqs.addCapability(SPIRV::Capability::LongCompositesINTEL);
2612 break;
2613 }
2614 case SPIRV::OpArbitraryFloatEQALTERA:
2615 case SPIRV::OpArbitraryFloatGEALTERA:
2616 case SPIRV::OpArbitraryFloatGTALTERA:
2617 case SPIRV::OpArbitraryFloatLEALTERA:
2618 case SPIRV::OpArbitraryFloatLTALTERA:
2619 case SPIRV::OpArbitraryFloatCbrtALTERA:
2620 case SPIRV::OpArbitraryFloatCosALTERA:
2621 case SPIRV::OpArbitraryFloatCosPiALTERA:
2622 case SPIRV::OpArbitraryFloatExp10ALTERA:
2623 case SPIRV::OpArbitraryFloatExp2ALTERA:
2624 case SPIRV::OpArbitraryFloatExpALTERA:
2625 case SPIRV::OpArbitraryFloatExpm1ALTERA:
2626 case SPIRV::OpArbitraryFloatHypotALTERA:
2627 case SPIRV::OpArbitraryFloatLog10ALTERA:
2628 case SPIRV::OpArbitraryFloatLog1pALTERA:
2629 case SPIRV::OpArbitraryFloatLog2ALTERA:
2630 case SPIRV::OpArbitraryFloatLogALTERA:
2631 case SPIRV::OpArbitraryFloatRecipALTERA:
2632 case SPIRV::OpArbitraryFloatSinCosALTERA:
2633 case SPIRV::OpArbitraryFloatSinCosPiALTERA:
2634 case SPIRV::OpArbitraryFloatSinALTERA:
2635 case SPIRV::OpArbitraryFloatSinPiALTERA:
2636 case SPIRV::OpArbitraryFloatSqrtALTERA:
2637 case SPIRV::OpArbitraryFloatACosALTERA:
2638 case SPIRV::OpArbitraryFloatACosPiALTERA:
2639 case SPIRV::OpArbitraryFloatAddALTERA:
2640 case SPIRV::OpArbitraryFloatASinALTERA:
2641 case SPIRV::OpArbitraryFloatASinPiALTERA:
2642 case SPIRV::OpArbitraryFloatATan2ALTERA:
2643 case SPIRV::OpArbitraryFloatATanALTERA:
2644 case SPIRV::OpArbitraryFloatATanPiALTERA:
2645 case SPIRV::OpArbitraryFloatCastFromIntALTERA:
2646 case SPIRV::OpArbitraryFloatCastALTERA:
2647 case SPIRV::OpArbitraryFloatCastToIntALTERA:
2648 case SPIRV::OpArbitraryFloatDivALTERA:
2649 case SPIRV::OpArbitraryFloatMulALTERA:
2650 case SPIRV::OpArbitraryFloatPowALTERA:
2651 case SPIRV::OpArbitraryFloatPowNALTERA:
2652 case SPIRV::OpArbitraryFloatPowRALTERA:
2653 case SPIRV::OpArbitraryFloatRSqrtALTERA:
2654 case SPIRV::OpArbitraryFloatSubALTERA: {
2655 if (!ST.canUseExtension(
2656 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point))
2658 "Floating point instructions can't be translated correctly without "
2659 "enabled SPV_ALTERA_arbitrary_precision_floating_point extension!",
2660 false);
2661 Reqs.addExtension(
2662 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point);
2663 Reqs.addCapability(
2664 SPIRV::Capability::ArbitraryPrecisionFloatingPointALTERA);
2665 break;
2666 }
2667 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {
2668 if (!ST.canUseExtension(
2669 SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))
2671 "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "
2672 "following SPIR-V "
2673 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",
2674 false);
2675 Reqs.addExtension(
2676 SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);
2677 Reqs.addCapability(
2678 SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);
2679 break;
2680 }
2681 case SPIRV::OpBitwiseFunctionINTEL: {
2682 if (!ST.canUseExtension(
2683 SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))
2685 "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "
2686 "extension: SPV_INTEL_ternary_bitwise_function",
2687 false);
2688 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);
2689 Reqs.addCapability(SPIRV::Capability::TernaryBitwiseFunctionINTEL);
2690 break;
2691 }
2692 case SPIRV::OpCopyMemorySized: {
2693 Reqs.addCapability(SPIRV::Capability::Addresses);
2694 // TODO: Add UntypedPointersKHR when implemented.
2695 break;
2696 }
2697 case SPIRV::OpTypeUntypedPointerKHR:
2698 Reqs.getAndAddRequirements(SPIRV::OperandCategory::StorageClassOperand,
2699 MI.getOperand(1).getImm(), ST);
2700 [[fallthrough]];
2701 case SPIRV::OpUntypedVariableKHR:
2702 case SPIRV::OpUntypedAccessChainKHR:
2703 case SPIRV::OpUntypedInBoundsAccessChainKHR:
2704 case SPIRV::OpUntypedPtrAccessChainKHR:
2705 case SPIRV::OpUntypedInBoundsPtrAccessChainKHR:
2706 case SPIRV::OpUntypedPrefetchKHR:
2707 case SPIRV::OpUntypedGroupAsyncCopyKHR: {
2708 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_untyped_pointers))
2709 report_fatal_error("Untyped pointer instructions require the following "
2710 "SPIR-V extension: SPV_KHR_untyped_pointers",
2711 false);
2712 Reqs.addExtension(SPIRV::Extension::SPV_KHR_untyped_pointers);
2713 Reqs.addCapability(SPIRV::Capability::UntypedPointersKHR);
2714 break;
2715 }
2716 case SPIRV::OpPredicatedLoadINTEL:
2717 case SPIRV::OpPredicatedStoreINTEL: {
2718 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_predicated_io))
2720 "OpPredicated[Load/Store]INTEL instructions require "
2721 "the following SPIR-V extension: SPV_INTEL_predicated_io",
2722 false);
2723 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_predicated_io);
2724 Reqs.addCapability(SPIRV::Capability::PredicatedIOINTEL);
2725 break;
2726 }
2727 case SPIRV::OpFAddS:
2728 case SPIRV::OpFSubS:
2729 case SPIRV::OpFMulS:
2730 case SPIRV::OpFDivS:
2731 case SPIRV::OpFRemS:
2732 case SPIRV::OpFMod:
2733 case SPIRV::OpFNegate:
2734 case SPIRV::OpFAddV:
2735 case SPIRV::OpFSubV:
2736 case SPIRV::OpFMulV:
2737 case SPIRV::OpFDivV:
2738 case SPIRV::OpFRemV:
2739 case SPIRV::OpFNegateV: {
2740 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2741 SPIRVTypeInst TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
2742 if (isVectorType(TypeDef))
2743 TypeDef = MRI.getVRegDef(TypeDef->getOperand(1).getReg());
2744 if (isBFloat16Type(TypeDef)) {
2745 if (!ST.canUseExtension(SPIRV::Extension::SPV_EXT_bfloat16_arithmetic))
2747 "Arithmetic instructions with bfloat16 arguments require the "
2748 "following SPIR-V extension: SPV_EXT_bfloat16_arithmetic",
2749 false);
2750 Reqs.addExtension(SPIRV::Extension::SPV_EXT_bfloat16_arithmetic);
2751 Reqs.addCapability(SPIRV::Capability::BFloat16ArithmeticEXT);
2752 }
2753 break;
2754 }
2755 case SPIRV::OpOrdered:
2756 case SPIRV::OpUnordered:
2757 case SPIRV::OpFOrdEqual:
2758 case SPIRV::OpFOrdNotEqual:
2759 case SPIRV::OpFOrdLessThan:
2760 case SPIRV::OpFOrdLessThanEqual:
2761 case SPIRV::OpFOrdGreaterThan:
2762 case SPIRV::OpFOrdGreaterThanEqual:
2763 case SPIRV::OpFUnordEqual:
2764 case SPIRV::OpFUnordNotEqual:
2765 case SPIRV::OpFUnordLessThan:
2766 case SPIRV::OpFUnordLessThanEqual:
2767 case SPIRV::OpFUnordGreaterThan:
2768 case SPIRV::OpFUnordGreaterThanEqual: {
2769 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2770 SPIRVTypeInst TypeDef = getRegType(MI.getOperand(2).getReg(), *MI.getMF(),
2771 ST.getSPIRVGlobalRegistry());
2772 assert(TypeDef && "Unexpected type instruction for comparison operand");
2773 if (isVectorType(TypeDef))
2774 TypeDef = MRI.getVRegDef(TypeDef->getOperand(1).getReg());
2775 if (isBFloat16Type(TypeDef)) {
2776 if (!ST.canUseExtension(SPIRV::Extension::SPV_EXT_bfloat16_arithmetic))
2778 "Relational instructions with bfloat16 arguments require the "
2779 "following SPIR-V extension: SPV_EXT_bfloat16_arithmetic",
2780 false);
2781 Reqs.addExtension(SPIRV::Extension::SPV_EXT_bfloat16_arithmetic);
2782 Reqs.addCapability(SPIRV::Capability::BFloat16ArithmeticEXT);
2783 }
2784 break;
2785 }
2786 case SPIRV::OpDPdxCoarse:
2787 case SPIRV::OpDPdyCoarse:
2788 case SPIRV::OpDPdxFine:
2789 case SPIRV::OpDPdyFine: {
2790 Reqs.addCapability(SPIRV::Capability::DerivativeControl);
2791 break;
2792 }
2793 case SPIRV::OpLoopControlINTEL: {
2794 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_unstructured_loop_controls);
2795 Reqs.addCapability(SPIRV::Capability::UnstructuredLoopControlsINTEL);
2796 break;
2797 }
2798
2799 default:
2800 break;
2801 }
2802
2803 // If we require capability Shader, then we can remove the requirement for
2804 // the BitInstructions capability, since Shader is a superset capability
2805 // of BitInstructions.
2806 Reqs.removeCapabilityIf(SPIRV::Capability::BitInstructions,
2807 SPIRV::Capability::Shader);
2808}
2809
2811 MachineFunctionGetter GetMF, const SPIRVSubtarget &ST) {
2812 // Collect requirements for existing instructions.
2813 for (const Function &F : M) {
2814 MachineFunction *MF = GetMF(F);
2815 if (!MF)
2816 continue;
2817 for (const MachineBasicBlock &MBB : *MF)
2818 for (const MachineInstr &MI : MBB)
2819 addInstrRequirements(MI, MAI, ST);
2820 }
2821 // Collect requirements for OpExecutionMode instructions.
2822 auto Node = M.getNamedMetadata("spirv.ExecutionMode");
2823 if (Node) {
2824 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,
2825 RequireKHRFloatControls2 = false,
2826 VerLower14 = !ST.isAtLeastSPIRVVer(VersionTuple(1, 4));
2827 bool HasIntelFloatControls2 =
2828 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_float_controls2);
2829 bool HasKHRFloatControls2 =
2830 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2831 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2832 MDNode *MDN = cast<MDNode>(Node->getOperand(i));
2833 const MDOperand &MDOp = MDN->getOperand(1);
2834 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(MDOp)) {
2835 Constant *C = CMeta->getValue();
2836 if (ConstantInt *Const = dyn_cast<ConstantInt>(C)) {
2837 auto EM = Const->getZExtValue();
2838 // SPV_KHR_float_controls is not available until v1.4:
2839 // add SPV_KHR_float_controls if the version is too low
2840 switch (EM) {
2841 case SPIRV::ExecutionMode::DenormPreserve:
2842 case SPIRV::ExecutionMode::DenormFlushToZero:
2843 case SPIRV::ExecutionMode::RoundingModeRTE:
2844 case SPIRV::ExecutionMode::RoundingModeRTZ:
2845 RequireFloatControls = VerLower14;
2847 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2848 break;
2849 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:
2850 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:
2851 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:
2852 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:
2853 if (HasIntelFloatControls2) {
2854 RequireIntelFloatControls2 = true;
2856 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2857 }
2858 break;
2859 case SPIRV::ExecutionMode::FPFastMathDefault: {
2860 if (HasKHRFloatControls2) {
2861 RequireKHRFloatControls2 = true;
2863 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2864 }
2865 break;
2866 }
2867 case SPIRV::ExecutionMode::ContractionOff:
2868 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:
2869 if (HasKHRFloatControls2) {
2870 RequireKHRFloatControls2 = true;
2872 SPIRV::OperandCategory::ExecutionModeOperand,
2873 SPIRV::ExecutionMode::FPFastMathDefault, ST);
2874 } else {
2876 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2877 }
2878 break;
2879 default:
2881 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2882 }
2883 }
2884 }
2885 }
2886 if (RequireFloatControls &&
2887 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls))
2888 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls);
2889 if (RequireIntelFloatControls2)
2890 MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_float_controls2);
2891 if (RequireKHRFloatControls2)
2892 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2893 }
2894 for (const Function &F : M) {
2895 if (F.isDeclaration())
2896 continue;
2897 if (F.getMetadata("reqd_work_group_size"))
2899 SPIRV::OperandCategory::ExecutionModeOperand,
2900 SPIRV::ExecutionMode::LocalSize, ST);
2901 if (F.getFnAttribute("hlsl.numthreads").isValid()) {
2903 SPIRV::OperandCategory::ExecutionModeOperand,
2904 SPIRV::ExecutionMode::LocalSize, ST);
2905 }
2906 if (F.getFnAttribute("enable-maximal-reconvergence").getValueAsBool()) {
2907 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_maximal_reconvergence);
2908 }
2909 if (F.getMetadata("work_group_size_hint"))
2911 SPIRV::OperandCategory::ExecutionModeOperand,
2912 SPIRV::ExecutionMode::LocalSizeHint, ST);
2913 if (F.getMetadata("intel_reqd_sub_group_size") ||
2914 F.getMetadata("reqd_sub_group_size"))
2916 SPIRV::OperandCategory::ExecutionModeOperand,
2917 SPIRV::ExecutionMode::SubgroupSize, ST);
2918 if (F.getMetadata("max_work_group_size"))
2920 SPIRV::OperandCategory::ExecutionModeOperand,
2921 SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);
2922 if (F.getMetadata("vec_type_hint"))
2924 SPIRV::OperandCategory::ExecutionModeOperand,
2925 SPIRV::ExecutionMode::VecTypeHint, ST);
2926
2927 if (F.hasOptNone()) {
2928 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_optnone)) {
2929 MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_optnone);
2930 MAI.Reqs.addCapability(SPIRV::Capability::OptNoneINTEL);
2931 } else if (ST.canUseExtension(SPIRV::Extension::SPV_EXT_optnone)) {
2932 MAI.Reqs.addExtension(SPIRV::Extension::SPV_EXT_optnone);
2933 MAI.Reqs.addCapability(SPIRV::Capability::OptNoneEXT);
2934 }
2935 }
2936 }
2937}
2938
2939static unsigned getFastMathFlags(const MachineInstr &I,
2940 const SPIRVSubtarget &ST) {
2941 unsigned Flags = SPIRV::FPFastMathMode::None;
2942 bool CanUseKHRFloatControls2 =
2943 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2944 if (I.getFlag(MachineInstr::MIFlag::FmNoNans))
2945 Flags |= SPIRV::FPFastMathMode::NotNaN;
2946 if (I.getFlag(MachineInstr::MIFlag::FmNoInfs))
2947 Flags |= SPIRV::FPFastMathMode::NotInf;
2948 if (I.getFlag(MachineInstr::MIFlag::FmNsz))
2949 Flags |= SPIRV::FPFastMathMode::NSZ;
2950 if (I.getFlag(MachineInstr::MIFlag::FmArcp))
2951 Flags |= SPIRV::FPFastMathMode::AllowRecip;
2952 if (I.getFlag(MachineInstr::MIFlag::FmContract) && CanUseKHRFloatControls2)
2953 Flags |= SPIRV::FPFastMathMode::AllowContract;
2954 if (I.getFlag(MachineInstr::MIFlag::FmReassoc)) {
2955 if (CanUseKHRFloatControls2)
2956 // LLVM reassoc maps to SPIRV transform, see
2957 // https://github.com/KhronosGroup/SPIRV-Registry/issues/326 for details.
2958 // Because we are enabling AllowTransform, we must enable AllowReassoc and
2959 // AllowContract too, as required by SPIRV spec. Also, we used to map
2960 // MIFlag::FmReassoc to FPFastMathMode::Fast, which now should instead by
2961 // replaced by turning all the other bits instead. Therefore, we're
2962 // enabling every bit here except None and Fast.
2963 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2964 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |
2965 SPIRV::FPFastMathMode::AllowTransform |
2966 SPIRV::FPFastMathMode::AllowReassoc |
2967 SPIRV::FPFastMathMode::AllowContract;
2968 else
2969 Flags |= SPIRV::FPFastMathMode::Fast;
2970 }
2971
2972 if (CanUseKHRFloatControls2) {
2973 // Error out if SPIRV::FPFastMathMode::Fast is enabled.
2974 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&
2975 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "
2976 "anymore.");
2977
2978 // Error out if AllowTransform is enabled without AllowReassoc and
2979 // AllowContract.
2980 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||
2981 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&
2982 Flags & SPIRV::FPFastMathMode::AllowContract))) &&
2983 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "
2984 "AllowContract flags to be enabled as well.");
2985 }
2986
2987 return Flags;
2988}
2989
2991 if (ST.isKernel())
2992 return true;
2993 if (ST.getSPIRVVersion() < VersionTuple(1, 2))
2994 return false;
2995 return ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2996}
2997
2999 MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII,
3001 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec) {
3002 bool Changed = false;
3003 // Insert after I so that the decorated register is defined before its use.
3004 MachineBasicBlock::iterator InsertPt = std::next(I.getIterator());
3005 auto Decorate = [&](SPIRV::Decoration::Decoration Dec,
3006 ArrayRef<uint32_t> DecArgs) {
3007 MachineIRBuilder MIRBuilder(*I.getParent(), InsertPt);
3008 MIRBuilder.setDebugLoc(I.getDebugLoc());
3009 buildOpDecorate(I.getOperand(0).getReg(), MIRBuilder, Dec, DecArgs);
3010 Changed = true;
3011 };
3012 if (TII.canUseIntegerWrapDecoration(I)) {
3013 if (I.getFlag(MachineInstr::MIFlag::NoSWrap) &&
3015 SPIRV::OperandCategory::DecorationOperand,
3016 SPIRV::Decoration::NoSignedWrap, ST, Reqs)
3017 .IsSatisfiable)
3018 Decorate(SPIRV::Decoration::NoSignedWrap, {});
3019 if (I.getFlag(MachineInstr::MIFlag::NoUWrap) &&
3021 SPIRV::OperandCategory::DecorationOperand,
3022 SPIRV::Decoration::NoUnsignedWrap, ST, Reqs)
3023 .IsSatisfiable)
3024 Decorate(SPIRV::Decoration::NoUnsignedWrap, {});
3025 }
3026 // In Kernel environments, FPFastMathMode on OpExtInst is valid per core
3027 // spec. For other instruction types, SPV_KHR_float_controls2 is required.
3028 bool CanUseFM =
3029 TII.canUseFastMathFlags(
3030 I, ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2)) ||
3031 (ST.isKernel() && I.getOpcode() == SPIRV::OpExtInst);
3032 if (!CanUseFM)
3033 return Changed;
3034
3035 unsigned FMFlags = getFastMathFlags(I, ST);
3036 if (FMFlags == SPIRV::FPFastMathMode::None) {
3037 // We also need to check if any FPFastMathDefault info was set for the
3038 // types used in this instruction.
3039 if (FPFastMathDefaultInfoVec.empty())
3040 return Changed;
3041
3042 // Use the first operand's FP type for arithmetic and comparisons,
3043 // and the result type for calls and extended instructions.
3044 SPIRVTypeInst FPType =
3045 I.getOpcode() == SPIRV::OpExtInst ||
3046 I.getOpcode() == SPIRV::OpFunctionCall
3047 ? I.getMF()->getRegInfo().getVRegDef(I.getOperand(1).getReg())
3048 : getRegType(I.getOperand(2).getReg(), *I.getMF(), GR);
3049 const Type *Ty = GR->getTypeForSPIRVType(FPType);
3050 Ty = Ty->isVectorTy() ? cast<VectorType>(Ty)->getElementType() : Ty;
3051
3052 // Match instruction type with the FPFastMathDefaultInfoVec.
3053 bool Emit = false;
3054 for (SPIRV::FPFastMathDefaultInfo &Elem : FPFastMathDefaultInfoVec) {
3055 if (Ty == Elem.Ty) {
3056 FMFlags = Elem.FastMathFlags;
3057 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||
3058 Elem.FPFastMathDefault;
3059 break;
3060 }
3061 }
3062
3063 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)
3064 return Changed;
3065 }
3067 Decorate(SPIRV::Decoration::FPFastMathMode, {FMFlags});
3068 return Changed;
3069}
3070
3071static void addMBBNames(const Module &M, MachineFunctionGetter GetMF,
3073 for (const Function &F : M) {
3074 MachineFunction *MF = GetMF(F);
3075 if (!MF)
3076 continue;
3077 if (MF->getFunction()
3079 .isValid())
3080 continue;
3081 for (const auto &MBB : *MF) {
3082 if (!MBB.hasName() || MBB.empty())
3083 continue;
3084 MAI.MBBNames.emplace_back(MAI.getOrCreateMBBRegister(MBB), MBB.getName());
3085 }
3086 }
3087}
3088
3090 const Module &M,
3091 SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap,
3092 const Function *F) {
3093 auto it = FPFastMathDefaultInfoMap.find(F);
3094 if (it != FPFastMathDefaultInfoMap.end())
3095 return it->second;
3096
3097 // If the map does not contain the entry, create a new one. Initialize it to
3098 // contain all 3 elements sorted by bit width of target type: {half, float,
3099 // double}.
3100 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3101 FPFastMathDefaultInfoVec.emplace_back(Type::getHalfTy(M.getContext()),
3102 SPIRV::FPFastMathMode::None);
3103 FPFastMathDefaultInfoVec.emplace_back(Type::getFloatTy(M.getContext()),
3104 SPIRV::FPFastMathMode::None);
3105 FPFastMathDefaultInfoVec.emplace_back(Type::getDoubleTy(M.getContext()),
3106 SPIRV::FPFastMathMode::None);
3107 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3108}
3109
3111 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3112 const Type *Ty) {
3113 size_t BitWidth = Ty->getScalarSizeInBits();
3114 int Index =
3116 BitWidth);
3117 assert(Index >= 0 && Index < 3 &&
3118 "Expected FPFastMathDefaultInfo for half, float, or double");
3119 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3120 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3121 return FPFastMathDefaultInfoVec[Index];
3122}
3123
3125 const Module &M,
3126 SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap,
3127 const SPIRVSubtarget &ST, const Function *OnlyF = nullptr) {
3128 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2))
3129 return;
3130
3131 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3132 // We need the entry point (function) as the key, and the target
3133 // type and flags as the value.
3134 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3135 // execution modes, as they are now deprecated and must be replaced
3136 // with FPFastMathDefaultInfo.
3137 auto Node = M.getNamedMetadata("spirv.ExecutionMode");
3138 if (!Node)
3139 return;
3140
3141 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3142 MDNode *MDN = cast<MDNode>(Node->getOperand(i));
3143 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3144 const Function *F = cast<Function>(
3145 cast<ConstantAsMetadata>(MDN->getOperand(0))->getValue());
3146 if (OnlyF && F != OnlyF)
3147 continue;
3148 const auto EM =
3150 cast<ConstantAsMetadata>(MDN->getOperand(1))->getValue())
3151 ->getZExtValue();
3152 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3153 assert(MDN->getNumOperands() == 4 &&
3154 "Expected 4 operands for FPFastMathDefault");
3155
3156 const Type *T = cast<ValueAsMetadata>(MDN->getOperand(2))->getType();
3157 unsigned Flags =
3159 cast<ConstantAsMetadata>(MDN->getOperand(3))->getValue())
3160 ->getZExtValue();
3161 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3162 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3164 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, T);
3165 Info.FastMathFlags = Flags;
3166 Info.FPFastMathDefault = true;
3167 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3168 assert(MDN->getNumOperands() == 2 &&
3169 "Expected no operands for ContractionOff");
3170
3171 // We need to save this info for every possible FP type, i.e. {half,
3172 // float, double, fp128}.
3173 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3174 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3175 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3176 Info.ContractionOff = true;
3177 }
3178 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3179 assert(MDN->getNumOperands() == 3 &&
3180 "Expected 1 operand for SignedZeroInfNanPreserve");
3181 unsigned TargetWidth =
3183 cast<ConstantAsMetadata>(MDN->getOperand(2))->getValue())
3184 ->getZExtValue();
3185 // We need to save this info only for the FP type with TargetWidth.
3186 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3187 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3190 assert(Index >= 0 && Index < 3 &&
3191 "Expected FPFastMathDefaultInfo for half, float, or double");
3192 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3193 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3194 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3195 }
3196 }
3197}
3198
3202 : ST(&ST), GR(ST.getSPIRVGlobalRegistry()), TII(ST.getInstrInfo()),
3203 MAI(MAI), GetMF(GetMF) {}
3204
3206 setBaseInfo(M);
3207
3208 addMBBNames(M, GetMF, MAI);
3209 collectFPFastMathDefaults(M, MAI.FPFastMathDefaultInfoMap, *ST);
3210
3211 collectReqs(M, MAI, GetMF, *ST);
3212
3213 // Process type/const/global var/func decl instructions, number their
3214 // destination registers from 0 to N, collect Extensions and Capabilities.
3215 collectDeclarations(M);
3216
3217 // Number rest of registers from N+1 onwards.
3218 numberRegistersGlobally(M);
3219
3220 // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions.
3221 processOtherInstrs(M);
3222
3223 // If there are no entry points, we need the Linkage capability.
3224 if (MAI.MS[SPIRV::MB_EntryPoints].empty())
3225 MAI.Reqs.addCapability(SPIRV::Capability::Linkage);
3226
3227 // Set maximum ID used.
3228 GR->setBound(MAI.MaxID);
3229}
3230
3235
3237 SPIRVTargetMachine &TM =
3241 *TM.getSubtargetImpl(), MAI,
3242 [&MMI](const Function &F) { return MMI.getMachineFunction(F); })
3243 .run(M);
3244 return false;
3245}
3246
3247AnalysisKey SPIRVModuleAnalysis::Key;
3248
3251 const auto &TM = static_cast<const SPIRVTargetMachine &>(
3252 MAM.getResult<MachineModuleAnalysis>(M).getMMI().getTarget());
3254 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
3255 Result MAI;
3256 SPIRVModuleAnalysisImpl(*TM.getSubtargetImpl(), MAI,
3257 [&FAM](const Function &F) -> MachineFunction * {
3258 MachineFunctionAnalysis::Result *MFA =
3259 FAM.getCachedResult<MachineFunctionAnalysis>(
3260 const_cast<Function &>(F));
3261 assert((MFA || F.isDeclaration()) &&
3262 "Missing MachineFunction for definition");
3263 return MFA ? &MFA->getMF() : nullptr;
3264 })
3265 .run(M);
3266 return MAI;
3267}
3268
3270 const auto &ST = MF.getSubtarget<SPIRVSubtarget>();
3271 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
3272 const SPIRVInstrInfo &TII = *ST.getInstrInfo();
3275 SPIRV::FPFastMathDefaultInfoMapTy FPFastMathDefaultInfoMap;
3276 const Function &F = MF.getFunction();
3277 collectFPFastMathDefaults(*F.getParent(), FPFastMathDefaultInfoMap, ST, &F);
3278 // Add decorations related to MI flags.
3279 bool Changed = false;
3280 for (auto &MBB : MF)
3281 for (auto &MI : make_early_inc_range(MBB))
3282 Changed |= handleMIFlagDecoration(MI, ST, TII, Reqs, GR,
3283 FPFastMathDefaultInfoMap[&F]);
3284 return Changed;
3285}
3286
3287namespace {
3288class SPIRVPrepareModuleAnalysisLegacy : public MachineFunctionPass {
3289public:
3290 static char ID;
3291 SPIRVPrepareModuleAnalysisLegacy() : MachineFunctionPass(ID) {}
3292 bool runOnMachineFunction(MachineFunction &MF) override {
3293 return prepareModuleAnalysis(MF);
3294 }
3295};
3296} // namespace
3297
3298char SPIRVPrepareModuleAnalysisLegacy::ID = 0;
3299
3300INITIALIZE_PASS(SPIRVPrepareModuleAnalysisLegacy,
3301 "spirv-prepare-module-analysis",
3302 "SPIRV prepare module analysis", false, false)
3303
3305 return new SPIRVPrepareModuleAnalysisLegacy();
3306}
3307
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
const AArch64InstrInfo MachineBasicBlock::iterator InsertPt
const AArch64InstrInfo & TII
ReachingDefInfo InstSet & ToRemove
MachineBasicBlock & MBB
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define DEBUG_TYPE
static Register UseReg(const MachineOperand &MO)
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
MachineRegisterInfo & MRI
#define ATOM_FLT_REQ_EXT_MSG(ExtName)
static bool isFastMathModeAvailable(const SPIRVSubtarget &ST)
static void addMBBNames(const Module &M, MachineFunctionGetter GetMF, SPIRV::ModuleAnalysisInfo &MAI)
static void maybeAddScatterGatherReq(const MachineInstr &MI, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST)
static void addImageOperandReqs(const MachineInstr &MI, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST, unsigned OpIdx)
bool isStorageImage(MachineInstr *ImageInst)
bool isInputAttachment(MachineInstr *ImageInst)
static bool isBFloat16Type(SPIRVTypeInst TypeDef)
static bool handleMIFlagDecoration(MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII, SPIRV::RequirementHandler &Reqs, const SPIRVGlobalRegistry *GR, SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec)
bool isSampledImage(MachineInstr *ImageInst)
static unsigned getInvokeOperandIdx(unsigned Opcode)
static SPIRV::FPFastMathDefaultInfo & getFPFastMathDefaultInfo(SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec, const Type *Ty)
static void collectFPFastMathDefaults(const Module &M, SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap, const SPIRVSubtarget &ST, const Function *OnlyF=nullptr)
static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI, SPIRV::ModuleSectionType MSType, InstrTraces &IS, bool Append=true)
void addPrintfRequirements(const MachineInstr &MI, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST)
static void addOpTypeImageReqs(const MachineInstr &MI, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST)
static void addQuadControlRequirements(SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST, StringRef Feature)
static bool isImageTypeWithUnknownFormat(SPIRVTypeInst TypeInst)
bool isUniformTexelBuffer(MachineInstr *ImageInst)
bool isStorageTexelBuffer(MachineInstr *ImageInst)
static void AddAtomicFloatRequirements(const MachineInstr &MI, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST)
bool isCombinedImageSampler(MachineInstr *SampledImageInst)
bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI)
const char * Msg
void addInstrRequirements(const MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI, const SPIRVSubtarget &ST)
static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI, MachineFunctionGetter GetMF, const SPIRVSubtarget &ST)
static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST)
static InstrSignature instrToSignature(const MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI, bool UseDefReg)
static void AddDotProductRequirements(const MachineInstr &MI, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST)
static SPIRV::Requirements getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category, unsigned i, const SPIRVSubtarget &ST, SPIRV::RequirementHandler &Reqs)
static SPIRVTypeInst getRegType(Register Reg, const MachineFunction &MF, const SPIRVGlobalRegistry *GR)
static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex, unsigned DefaultVal=0)
static void addCoreInSPIRV13Capability(SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST, SPIRV::Capability::Capability Cap, SPIRV::Extension::Extension Ext)
void addOpAccessChainReqs(const MachineInstr &Instr, SPIRV::RequirementHandler &Handler, const SPIRVSubtarget &Subtarget)
static bool prepareModuleAnalysis(MachineFunction &MF)
static void appendDecorationsForReg(const MachineRegisterInfo &MRI, Register R, InstrSignature &Signature)
static SPIRV::FPFastMathDefaultInfoVector & getOrCreateFPFastMathDefaultInfoVec(const Module &M, SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap, const Function *F)
static bool shouldAvoidCapability(const SPIRVSubtarget &ST, SPIRV::Capability::Capability Cap)
static void AddAtomicVectorFloatRequirements(const MachineInstr &MI, SPIRV::RequirementHandler &Reqs, const SPIRVSubtarget &ST)
static unsigned getFastMathFlags(const MachineInstr &I, const SPIRVSubtarget &ST)
#define SPIRV_BACKEND_SERVICE_FUN_NAME
Definition SPIRVUtils.h:558
This file contains some templates that are useful if you are working with the STL at all.
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
#define LLVM_DEBUG(...)
Definition Debug.h:119
Target-Independent Code Generator Pass Configuration Options pass.
static Function * getFunction(FunctionType *Ty, const Twine &Name, Module *M)
Value * RHS
Value * LHS
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
bool isValid() const
Return true if the attribute is any kind of attribute.
Definition Attributes.h:263
This is the shared class of boolean and integer constants.
Definition Constants.h:87
This is an important base class in LLVM.
Definition Constant.h:44
Diagnostic information for unsupported feature in backend.
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:766
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
constexpr bool isValid() const
Definition MCRegister.h:84
Metadata node.
Definition Metadata.h:1081
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
Register getReg(unsigned Idx) const
Get the register for the operand index.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const MachineOperand & getOperand(unsigned i) const
An analysis that produces MachineModuleInfo for a module.
This class contains meta information specific to a module.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
LLVM_ABI void print(raw_ostream &os, const TargetRegisterInfo *TRI=nullptr) const
Print the MachineOperand to os.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
LLVM_ABI LLVM_READONLY MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
iterator find(const KeyT &Key)
Definition MapVector.h:157
iterator end()
Definition MapVector.h:70
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
unsigned getScalarOrVectorBitWidth(SPIRVTypeInst Type) const
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
SPIRVModuleAnalysisImpl(const SPIRVSubtarget &ST, SPIRV::ModuleAnalysisInfo &MAI, MachineFunctionGetter GetMF)
SPIRV::ModuleAnalysisInfo Result
Result run(Module &M, ModuleAnalysisManager &MAM)
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
const SPIRVSubtarget * getSubtargetImpl() const
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Target-Independent Code Generator Pass Configuration Options.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
Definition Type.cpp:272
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:271
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:269
Represents a version number in the form major[.minor[.subminor[.build]]].
bool empty() const
Determine whether this version information is empty (e.g., all version components are zero).
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
MapVector< const Function *, FPFastMathDefaultInfoVector > FPFastMathDefaultInfoMapTy
SmallVector< const MachineInstr * > InstrList
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
Definition STLExtras.h:2132
std::string getStringImm(const MachineInstr &MI, unsigned StartIndex)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
hash_code hash_value(const FixedPointSemantics &Val)
ExtensionList getSymbolicOperandExtensions(SPIRV::OperandCategory::OperandCategory Category, uint32_t Value)
CapabilityList getSymbolicOperandCapabilities(SPIRV::OperandCategory::OperandCategory Category, uint32_t Value)
SmallVector< SPIRV::Extension::Extension, 8 > ExtensionList
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
SmallVector< size_t > InstrSignature
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
static void reportUnsupported(MachineIRBuilder &MIRBuilder, const Twine &Msg)
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, ArrayRef< uint32_t > DecArgs, StringRef StrImm)
bool isVectorType(SPIRVTypeInst SPVTy)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
VersionTuple getSymbolicOperandMaxVersion(SPIRV::OperandCategory::OperandCategory Category, uint32_t Value)
function_ref< MachineFunction *(const Function &)> MachineFunctionGetter
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition STLExtras.h:2216
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
CapabilityList getCapabilitiesEnabledByExtension(SPIRV::Extension::Extension Extension)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
FunctionPass * createSPIRVPrepareModuleAnalysisLegacyPass()
std::string getSymbolicOperandMnemonic(SPIRV::OperandCategory::OperandCategory Category, int32_t Value)
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool requiresLongVectorEXT(unsigned NumComponents, bool CanUseVector16=true)
Definition SPIRVUtils.h:513
DWARFExpression::Operation Op
VersionTuple getSymbolicOperandMinVersion(SPIRV::OperandCategory::OperandCategory Category, uint32_t Value)
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
SmallVector< SPIRV::Capability::Capability, 8 > CapabilityList
std::set< InstrSignature > InstrTraces
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
Definition Hashing.h:307
std::map< SmallVector< size_t >, unsigned > InstrGRegsMap
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
#define N
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
static size_t computeFPFastMathDefaultInfoVecIndex(size_t BitWidth)
Definition SPIRVUtils.h:149
void setSkipEmission(const MachineInstr *MI)
MCRegister getRegisterAlias(const MachineFunction *MF, Register Reg)
MCRegister getOrCreateMBBRegister(const MachineBasicBlock &MBB)
InstrList MS[NUM_MODULE_SECTIONS]
AddressingModel::AddressingModel Addr
SmallVector< std::pair< MCRegister, StringRef > > MBBNames
void checkSatisfiable(const SPIRVSubtarget &ST) const
void getAndAddRequirements(SPIRV::OperandCategory::OperandCategory Category, uint32_t i, const SPIRVSubtarget &ST)
void addRequirements(const Requirements &Req)
bool isCapabilityAvailable(Capability::Capability Cap) const
void removeCapabilityIf(const Capability::Capability ToRemove, const Capability::Capability IfPresent)
void addExtensions(const ExtensionList &ToAdd)
void addAvailableCaps(const CapabilityList &ToAdd)
void addExtension(Extension::Extension ToAdd)
void initAvailableCapabilities(const SPIRVSubtarget &ST)
void addCapability(Capability::Capability ToAdd)
void addCapabilities(const CapabilityList &ToAdd)
const std::optional< Capability::Capability > Cap