LLVM 24.0.0git
SPIRVLegalizerInfo.cpp
Go to the documentation of this file.
1//===- SPIRVLegalizerInfo.cpp --- SPIR-V Legalization Rules ------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the Machinelegalizer class for SPIR-V.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVLegalizerInfo.h"
14#include "SPIRV.h"
15#include "SPIRVGlobalRegistry.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/Support/Debug.h"
28
29using namespace llvm;
30using namespace llvm::LegalizeActions;
31using namespace llvm::LegalityPredicates;
32
33#define DEBUG_TYPE "spirv-legalizer"
34
35LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts) {
36 return [IsExtendedInts, TypeIdx](const LegalityQuery &Query) {
37 const LLT Ty = Query.Types[TypeIdx];
38 return IsExtendedInts && Ty.isValid() && Ty.isScalar();
39 };
40}
41
42LegalityPredicate typeOfLongVectors(unsigned TypeIdx, bool IsLongVecs) {
43 return [TypeIdx, IsLongVecs](const LegalityQuery &Query) {
44 const LLT Ty = Query.Types[TypeIdx];
45 return IsLongVecs && Ty.isValid() && Ty.isVector();
46 };
47}
48
49static bool needsVectorLegalization(const LLT &Ty, const SPIRVSubtarget &ST) {
50 if (!Ty.isVector() ||
51 ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))
52 return false;
53 unsigned NumElements = Ty.getNumElements();
54 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
55 return (NumElements > 4 && !isPowerOf2_32(NumElements)) ||
56 NumElements > MaxVectorSize;
57}
58
60 using namespace TargetOpcode;
61
62 this->ST = &ST;
63 GR = ST.getSPIRVGlobalRegistry();
64
65 const LLT s1 = LLT::scalar(1);
66 const LLT s8 = LLT::scalar(8);
67 const LLT s16 = LLT::scalar(16);
68 const LLT s32 = LLT::scalar(32);
69 const LLT s64 = LLT::scalar(64);
70 const LLT s128 = LLT::scalar(128);
71
72 const LLT v16s64 = LLT::fixed_vector(16, 64);
73 const LLT v16s32 = LLT::fixed_vector(16, 32);
74 const LLT v16s16 = LLT::fixed_vector(16, 16);
75 const LLT v16s8 = LLT::fixed_vector(16, 8);
76 const LLT v16s1 = LLT::fixed_vector(16, 1);
77
78 const LLT v8s64 = LLT::fixed_vector(8, 64);
79 const LLT v8s32 = LLT::fixed_vector(8, 32);
80 const LLT v8s16 = LLT::fixed_vector(8, 16);
81 const LLT v8s8 = LLT::fixed_vector(8, 8);
82 const LLT v8s1 = LLT::fixed_vector(8, 1);
83
84 const LLT v4s64 = LLT::fixed_vector(4, 64);
85 const LLT v4s32 = LLT::fixed_vector(4, 32);
86 const LLT v4s16 = LLT::fixed_vector(4, 16);
87 const LLT v4s8 = LLT::fixed_vector(4, 8);
88 const LLT v4s1 = LLT::fixed_vector(4, 1);
89
90 const LLT v3s64 = LLT::fixed_vector(3, 64);
91 const LLT v3s32 = LLT::fixed_vector(3, 32);
92 const LLT v3s16 = LLT::fixed_vector(3, 16);
93 const LLT v3s8 = LLT::fixed_vector(3, 8);
94 const LLT v3s1 = LLT::fixed_vector(3, 1);
95
96 const LLT v2s64 = LLT::fixed_vector(2, 64);
97 const LLT v2s32 = LLT::fixed_vector(2, 32);
98 const LLT v2s16 = LLT::fixed_vector(2, 16);
99 const LLT v2s8 = LLT::fixed_vector(2, 8);
100 const LLT v2s1 = LLT::fixed_vector(2, 1);
101
102 const unsigned PSize = ST.getPointerSize();
103 const LLT p0 = LLT::pointer(SPIRVAS::Function, PSize);
104 const LLT p1 = LLT::pointer(SPIRVAS::CrossWorkgroup, PSize);
105 const LLT p2 = LLT::pointer(SPIRVAS::UniformConstant, PSize);
106 const LLT p3 = LLT::pointer(SPIRVAS::Workgroup, PSize);
107 const LLT p4 = LLT::pointer(SPIRVAS::Generic, PSize);
108 const LLT p5 = LLT::pointer(SPIRVAS::DeviceOnlyINTEL, PSize);
109 const LLT p6 = LLT::pointer(SPIRVAS::HostOnlyINTEL, PSize);
110 const LLT p7 = LLT::pointer(SPIRVAS::Input, PSize);
111 const LLT p8 = LLT::pointer(SPIRVAS::Output, PSize);
112 const LLT p9 = LLT::pointer(SPIRVAS::CodeSectionINTEL, PSize);
113 const LLT p10 = LLT::pointer(SPIRVAS::Private, PSize);
114 const LLT p11 = LLT::pointer(SPIRVAS::StorageBuffer, PSize);
115 const LLT p12 = LLT::pointer(SPIRVAS::Uniform, PSize);
116 const LLT p13 = LLT::pointer(SPIRVAS::PushConstant, PSize);
117
118 // TODO: remove copy-pasting here by using concatenation in some way.
119 auto allPtrsScalarsAndVectors = {
120 p0, p1, p2, p3, p4, p5, p6, p7, p8,
121 p9, p10, p11, p12, p13, s1, s8, s16, s32,
122 s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
123 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64, v8s1,
124 v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
125
126 auto allVectors = {v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
127 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32,
128 v4s64, v8s1, v8s8, v8s16, v8s32, v8s64, v16s1,
129 v16s8, v16s16, v16s32, v16s64};
130
131 auto allShaderVectors = {v2s1, v2s8, v2s16, v2s32, v2s64,
132 v3s1, v3s8, v3s16, v3s32, v3s64,
133 v4s1, v4s8, v4s16, v4s32, v4s64};
134
135 auto allScalars = {s1, s8, s16, s32, s64};
136
137 auto allScalarsAndVectors = {
138 s1, s8, s16, s32, s64, s128, v2s1, v2s8,
139 v2s16, v2s32, v2s64, v3s1, v3s8, v3s16, v3s32, v3s64,
140 v4s1, v4s8, v4s16, v4s32, v4s64, v8s1, v8s8, v8s16,
141 v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
142
143 auto allShaderScalarsAndVectors = {
144 s1, s8, s16, s32, s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64,
145 v3s1, v3s8, v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64};
146
147 auto &allowedScalarsAndVectors =
148 ST.isShader() ? allShaderScalarsAndVectors : allScalarsAndVectors;
149
150 auto allIntScalarsAndVectors = {
151 s8, s16, s32, s64, s128, v2s8, v2s16, v2s32, v2s64,
152 v3s8, v3s16, v3s32, v3s64, v4s8, v4s16, v4s32, v4s64, v8s8,
153 v8s16, v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
154
155 auto allBoolScalarsAndVectors = {s1, v2s1, v3s1, v4s1, v8s1, v16s1};
156 auto allBoolVectors = {v2s1, v3s1, v4s1, v8s1, v16s1};
157
158 auto allIntScalars = {s8, s16, s32, s64, s128};
159
160 auto allShaderIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16,
161 v3s32, v3s64, v4s8, v4s16, v4s32, v4s64};
162
163 auto allIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16, v3s32,
164 v3s64, v4s8, v4s16, v4s32, v4s64, v8s8, v8s16,
165 v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
166
167 auto &allowedIntVectorTypes =
168 ST.isShader() ? allShaderIntVectors : allIntVectors;
169
170 auto allFloatScalarsAndF16Vector2AndVector4s = {s16, s32, s64, v2s16, v4s16};
171
172 auto allFloatScalars = {s16, s32, s64};
173
174 auto allFloatScalarsAndVectors = {
175 s16, s32, s64, v2s16, v2s32, v2s64, v3s16, v3s32, v3s64,
176 v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};
177
178 auto allShaderFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
179 v3s64, v4s16, v4s32, v4s64};
180
181 auto allFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
182 v3s64, v4s16, v4s32, v4s64, v8s16,
183 v8s32, v8s64, v16s16, v16s32, v16s64};
184
185 auto &allowedFloatVectorTypes =
186 ST.isShader() ? allShaderFloatVectors : allFloatVectors;
187
188 auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
189 p2, p3, p4, p5, p6, p7,
190 p8, p9, p10, p11, p12, p13};
191
192 auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13};
193
194 auto &allowedVectorTypes = ST.isShader() ? allShaderVectors : allVectors;
195
196 bool HasArbitraryPrecisionInts = ST.canUseExtension(
197 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
198 bool IsExtendedInts =
199 HasArbitraryPrecisionInts ||
200 ST.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions) ||
201 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4);
202 bool IsLongVecs = ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector);
203 auto ExtendedIntScalarsAndVectors =
204 [IsExtendedInts](const LegalityQuery &Query) {
205 const LLT Ty = Query.Types[0];
206 return IsExtendedInts && Ty.isValid() &&
207 !Ty.isPointerOrPointerVector() && Ty.getScalarSizeInBits() > 1;
208 };
209 auto ExtendedScalarsAndVectorsProduct = [IsExtendedInts](
210 const LegalityQuery &Query) {
211 const LLT Ty1 = Query.Types[0], Ty2 = Query.Types[1];
212 return IsExtendedInts && Ty1.isValid() && Ty2.isValid() &&
213 !Ty1.isPointerOrPointerVector() && !Ty2.isPointerOrPointerVector();
214 };
215 auto ExtendedPtrsScalarsAndVectors =
216 [IsExtendedInts](const LegalityQuery &Query) {
217 const LLT Ty = Query.Types[0];
218 return IsExtendedInts && Ty.isValid();
219 };
220
221 // The universal validation rules in the SPIR-V specification state that
222 // vector sizes are typically limited to 2, 3, or 4. However, larger vector
223 // sizes (8 and 16) are enabled when the Kernel capability is present. For
224 // shader execution models, vector sizes are strictly limited to 4. In
225 // non-shader contexts, vector sizes of 8 and 16 are also permitted, but
226 // arbitrary sizes (e.g., 6 or 11) are not.
227 uint32_t MaxVectorSize = ST.isShader() ? 4 : 16;
228 LLVM_DEBUG(dbgs() << "MaxVectorSize: " << MaxVectorSize << "\n");
229
230 for (auto Opc : getTypeFoldingSupportedOpcodes()) {
231 switch (Opc) {
232 case G_EXTRACT_VECTOR_ELT:
233 case G_UREM:
234 case G_SREM:
235 case G_UDIV:
236 case G_SDIV:
237 case G_FREM:
238 case G_SELECT:
239 break;
240 default:
242 .customFor(allScalars)
243 .customFor(allowedVectorTypes)
244 .customIf(typeOfLongVectors(0, IsLongVecs))
248 0, ElementCount::getFixed(MaxVectorSize)))
249 .custom();
250 break;
251 }
252 }
253
254 getActionDefinitionsBuilder({G_UREM, G_SREM, G_SDIV, G_UDIV, G_FREM})
255 .customFor(allScalars)
256 .customFor(allowedVectorTypes)
257 .customIf(typeOfLongVectors(0, IsLongVecs))
261 0, ElementCount::getFixed(MaxVectorSize)))
262 .custom();
263
265 .customFor(allScalars)
266 .customFor(allowedVectorTypes)
267 .customIf(typeOfLongVectors(0, IsLongVecs))
270 0, ElementCount::getFixed(MaxVectorSize)))
271 .custom();
272
273 getActionDefinitionsBuilder({G_FMA, G_STRICT_FMA})
274 .legalFor(allScalars)
275 .legalFor(allowedVectorTypes)
276 .legalIf(typeOfLongVectors(0, IsLongVecs))
280 0, ElementCount::getFixed(MaxVectorSize)))
281 .alwaysLegal();
282
283 getActionDefinitionsBuilder(G_INTRINSIC_W_SIDE_EFFECTS).custom();
284
285 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
286 .legalForCartesianProduct(allowedVectorTypes, allowedVectorTypes)
287 .legalIf(typeOfLongVectors(0, IsLongVecs))
288 .legalIf(typeOfLongVectors(1, IsLongVecs))
290 .lowerIf(vectorElementCountIsGreaterThan(0, MaxVectorSize))
292 .lowerIf(vectorElementCountIsGreaterThan(1, MaxVectorSize));
293
294 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
295 .customIf(typeOfLongVectors(1, IsLongVecs))
299 1, ElementCount::getFixed(MaxVectorSize)))
300 .custom();
301
302 getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
303 .customIf(typeOfLongVectors(0, IsLongVecs))
307 0, ElementCount::getFixed(MaxVectorSize)))
308 .custom();
309
310 // Illegal G_UNMERGE_VALUES instructions should be handled
311 // during the combine phase.
312 getActionDefinitionsBuilder(G_BUILD_VECTOR)
313 .legalIf(typeOfLongVectors(0, IsLongVecs))
317 0, ElementCount::getFixed(MaxVectorSize)));
318
319 // When entering the legalizer, there should be no G_BITCAST instructions.
320 // They should all be calls to the `spv_bitcast` intrinsic. The call to
321 // the intrinsic will be converted to a G_BITCAST during legalization if
322 // the vectors are not legal. After using the rules to legalize a G_BITCAST,
323 // we turn it back into a call to the intrinsic with a custom rule to avoid
324 // potential machine verifier failures.
326 .customIf(typeOfLongVectors(0, IsLongVecs))
331 0, ElementCount::getFixed(MaxVectorSize)))
332 .lowerIf(vectorElementCountIsGreaterThan(1, MaxVectorSize))
333 .custom();
334
335 // If the result is still illegal, the combiner should be able to remove it.
336 getActionDefinitionsBuilder(G_CONCAT_VECTORS)
337 .legalForCartesianProduct(allowedVectorTypes, allowedVectorTypes)
339 typeOfLongVectors(1, IsLongVecs)));
340
341 getActionDefinitionsBuilder(G_SPLAT_VECTOR)
342 .legalFor(allowedVectorTypes)
343 .legalIf(typeOfLongVectors(0, IsLongVecs))
347 .alwaysLegal();
348
349 // Vector Reduction Operations
351 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
352 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
353 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
354 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
355 .legalFor(allowedVectorTypes)
356 .legalIf(typeOfLongVectors(0, IsLongVecs))
357 .scalarize(1)
358 .lower();
359
360 getActionDefinitionsBuilder({G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
361 .scalarize(2)
362 .lower();
363
364 // Illegal G_UNMERGE_VALUES instructions should be handled
365 // during the combine phase.
366 getActionDefinitionsBuilder(G_UNMERGE_VALUES)
368 typeOfLongVectors(1, IsLongVecs)))
370
371 getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE})
372 .unsupportedIf(LegalityPredicates::any(typeIs(0, p9), typeIs(1, p9)))
373 .legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allPtrs)));
374
375 getActionDefinitionsBuilder({G_MEMSET, G_MEMSET_INLINE})
376 .unsupportedIf(typeIs(0, p9))
377 .legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allIntScalars)));
378
379 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
380 .legalForCartesianProduct(allPtrs, allPtrs);
381
382 // Should we be legalizing bad scalar sizes like s5 here instead
383 // of handling them in the instruction selector?
384 getActionDefinitionsBuilder({G_LOAD, G_STORE})
385 .unsupportedIf(typeIs(1, p9))
386 .legalForCartesianProduct(allowedVectorTypes, allPtrs)
387 .legalForCartesianProduct(allPtrs, allPtrs)
388 .legalIf(isScalar(0))
389 .legalIf(typeOfLongVectors(0, IsLongVecs))
390 .custom();
391
392 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS,
393 G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,
394 G_USUBSAT, G_SCMP, G_UCMP})
395 .legalFor(allIntScalars)
396 .legalFor(allowedIntVectorTypes)
397 .legalIf(ExtendedIntScalarsAndVectors)
398 // LLVM i1 maps to OpTypeBool, not OpTypeInt.
399 .scalarizeIf(typeInSet(0, allBoolVectors), 0)
400 .minScalar(0, s32)
403 0, ElementCount::getFixed(MaxVectorSize)))
405
406 getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT}).lower();
407
408 getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
409 .legalForCartesianProduct(allFloatScalarsAndVectors, allIntScalars);
410
411 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
412 .legalForCartesianProduct(allIntScalarsAndVectors,
413 allFloatScalarsAndVectors)
414 // LLVM i1 maps to OpTypeBool, not OpTypeInt.
415 .widenScalarIf(typeInSet(0, allBoolScalarsAndVectors),
417
418 getActionDefinitionsBuilder({G_FPTOSI_SAT, G_FPTOUI_SAT})
419 .legalForCartesianProduct(allIntScalarsAndVectors,
420 allFloatScalarsAndVectors);
421
422 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
423 .legalForCartesianProduct(allFloatScalarsAndVectors,
424 allScalarsAndVectors);
425
427 .legalForCartesianProduct(allIntScalarsAndVectors)
428 .legalIf(ExtendedScalarsAndVectorsProduct)
429 .legalIf(typeOfLongVectors(0, IsLongVecs));
430
431 getActionDefinitionsBuilder({G_TRUNC, G_ZEXT, G_SEXT, G_ANYEXT})
432 .legalForCartesianProduct(allowedScalarsAndVectors)
433 .legalIf(ExtendedScalarsAndVectorsProduct)
434 .legalIf(typeOfLongVectors(0, IsLongVecs))
438 0, ElementCount::getFixed(MaxVectorSize)));
439
440 getActionDefinitionsBuilder(G_SEXT_INREG)
441 .lowerIf(typeOfLongVectors(0, IsLongVecs))
445 0, ElementCount::getFixed(MaxVectorSize)))
446 .lower();
447
449 .legalIf(typeOfLongVectors(0, IsLongVecs))
452 0, ElementCount::getFixed(MaxVectorSize)))
453 .legalFor(allPtrsScalarsAndVectors)
454 .legalIf(ExtendedPtrsScalarsAndVectors)
455 .legalIf([](const LegalityQuery &Query) {
456 return Query.Types[0].isPointerVector();
457 })
458 .moreElementsToNextPow2(0);
459
461 all(typeInSet(0, allPtrsScalarsAndVectors),
462 typeInSet(1, allPtrsScalarsAndVectors)));
463
464 getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
465 .legalFor({s1, s128})
466 .legalFor(allFloatAndIntScalarsAndPtrs)
467 .legalFor(allowedVectorTypes)
468 .legalIf([](const LegalityQuery &Query) {
469 return Query.Types[0].isPointerVector();
470 })
471 .legalIf(typeOfLongVectors(0, IsLongVecs))
475 0, ElementCount::getFixed(MaxVectorSize)));
476
477 getActionDefinitionsBuilder({G_STACKSAVE, G_STACKRESTORE}).alwaysLegal();
478
480 .legalForCartesianProduct(allPtrs, allIntScalars)
481 .legalIf(
482 all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)))
483 .legalIf([](const LegalityQuery &Query) {
484 const LLT DstTy = Query.Types[0];
485 const LLT SrcTy = Query.Types[1];
486 return DstTy.isPointerVector() && SrcTy.isVector() &&
487 !SrcTy.isPointer() &&
488 DstTy.getNumElements() == SrcTy.getNumElements();
489 });
491 .legalForCartesianProduct(allIntScalars, allPtrs)
492 .legalIf(
493 all(typeOfExtendedScalars(0, IsExtendedInts), typeInSet(1, allPtrs)))
494 .legalIf([](const LegalityQuery &Query) {
495 const LLT DstTy = Query.Types[0];
496 const LLT SrcTy = Query.Types[1];
497 return SrcTy.isPointerVector() && DstTy.isVector() &&
498 !DstTy.isPointer() &&
499 DstTy.getNumElements() == SrcTy.getNumElements();
500 });
502 .legalForCartesianProduct(allPtrs, allIntScalars)
503 .legalIf(
504 all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)));
505
507 .legalForCartesianProduct(allPtrs, allIntScalars)
508 .legalIf(
509 all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)))
510 .legalIf([](const LegalityQuery &Query) {
511 const LLT PtrTy = Query.Types[0];
512 const LLT MaskTy = Query.Types[1];
513 return PtrTy.isPointerVector() && MaskTy.isVector() &&
514 !MaskTy.isPointer() &&
515 PtrTy.getNumElements() == MaskTy.getNumElements();
516 });
517
518 // ST.canDirectlyComparePointers() for pointer args is supported in
519 // legalizeCustom().
522 all(typeIs(0, p9), typeInSet(1, allPtrs), typeIsNot(1, p9)),
523 all(typeInSet(0, allPtrs), typeIsNot(0, p9), typeIs(1, p9))))
524 .legalIf(
525 all(typeOfLongVectors(1, IsLongVecs), predNot(isPointerVector(1))))
528 1, ElementCount::getFixed(MaxVectorSize)))
529 .legalIf([IsExtendedInts, &ST](const LegalityQuery &Query) {
530 const LLT Ty = Query.Types[1];
531 return IsExtendedInts && Ty.isValid() &&
532 !Ty.isPointerOrPointerVector() &&
534 })
535 .customIf(all(typeInSet(0, allBoolScalarsAndVectors),
536 typeInSet(1, allPtrsScalarsAndVectors)))
538
540 .legalIf(typeOfLongVectors(1, IsLongVecs))
543 1, ElementCount::getFixed(MaxVectorSize)))
544 .legalIf(all(typeInSet(0, allBoolScalarsAndVectors),
545 typeInSet(1, allFloatScalarsAndVectors)))
547
548 getActionDefinitionsBuilder({G_ATOMICRMW_OR, G_ATOMICRMW_ADD, G_ATOMICRMW_AND,
549 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,
550 G_ATOMICRMW_SUB, G_ATOMICRMW_XOR,
551 G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
552 .legalForCartesianProduct(allIntScalars, allPtrs);
553
555 {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
556 .legalForCartesianProduct(allFloatScalarsAndF16Vector2AndVector4s,
557 allPtrs);
558
559 getActionDefinitionsBuilder(G_ATOMICRMW_XCHG)
560 .legalForCartesianProduct(allFloatAndIntScalarsAndPtrs, allPtrs);
561
562 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS).lower();
563 // TODO: add proper legalization rules.
564 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG).alwaysLegal();
566
567 getActionDefinitionsBuilder({G_UADDO, G_USUBO, G_UMULO, G_SMULO})
568 .alwaysLegal();
569
570 getActionDefinitionsBuilder({G_SADDO, G_SSUBO}).lower();
571
572 // Lowering widens s64 to s128, which needs
573 // SPV_ALTERA_arbitrary_precision_integers. Mark s64 unsupported otherwise.
574 auto &MulFix = getActionDefinitionsBuilder(
575 {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT});
576 if (!HasArbitraryPrecisionInts)
577 MulFix.unsupportedFor({s64});
578 MulFix.lower();
579
580 getActionDefinitionsBuilder({G_TRUNC_SSAT_S, G_TRUNC_USAT_U}).lower();
581
582 getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
583 .legalForCartesianProduct(allIntScalarsAndVectors,
584 allFloatScalarsAndVectors);
585
586 // FP conversions.
587 getActionDefinitionsBuilder({G_FPTRUNC, G_FPEXT})
588 .legalForCartesianProduct(allFloatScalarsAndVectors);
589
590 // Pointer-handling.
591 getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
592
593 getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor(allPtrs);
594
595 // Control-flow. In some cases (e.g. constants) s1 may be promoted to s32.
597 getActionDefinitionsBuilder(G_BRCOND).legalFor({s1, s32});
598
600 allFloatScalarsAndVectors, {s32, v2s32, v3s32, v4s32, v8s32, v16s32});
601
602 // TODO: Review the target OpenCL and GLSL Extended Instruction Set specs to
603 // tighten these requirements. Many of these math functions are only legal on
604 // specific bitwidths, so they are not selectable for
605 // allFloatScalarsAndVectors.
606 // clang-format off
607 getActionDefinitionsBuilder({G_STRICT_FSQRT,
608 G_FPOW,
609 G_FEXP,
610 G_FMODF,
611 G_FSINCOS,
612 G_FEXP2,
613 G_FEXP10,
614 G_FLOG,
615 G_FLOG2,
616 G_FLOG10,
617 G_FABS,
618 G_FMINNUM,
619 G_FMAXNUM,
620 G_FCEIL,
621 G_FCOS,
622 G_FSIN,
623 G_FTAN,
624 G_FACOS,
625 G_FASIN,
626 G_FATAN,
627 G_FATAN2,
628 G_FCOSH,
629 G_FSINH,
630 G_FTANH,
631 G_FSQRT,
632 G_FFLOOR,
633 G_FRINT,
634 G_FNEARBYINT,
635 G_INTRINSIC_ROUND,
636 G_INTRINSIC_TRUNC,
637 G_INTRINSIC_ROUNDEVEN})
638 .legalFor(allFloatScalars)
639 .legalFor(allowedFloatVectorTypes)
642 0, ElementCount::getFixed(MaxVectorSize)))
644 // clang-format on
645
646 getActionDefinitionsBuilder(G_FCOPYSIGN)
647 .legalForCartesianProduct(allFloatScalarsAndVectors,
648 allFloatScalarsAndVectors)
651 0, ElementCount::getFixed(MaxVectorSize)))
653
655 allFloatScalarsAndVectors, allIntScalarsAndVectors);
656
657 if (ST.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
659 {G_CTTZ, G_CTTZ_ZERO_POISON, G_CTLZ, G_CTLZ_ZERO_POISON})
660 .legalForCartesianProduct(allIntScalarsAndVectors,
661 allIntScalarsAndVectors);
662
663 // Struct return types become a single scalar, so cannot easily legalize.
664 getActionDefinitionsBuilder({G_SMULH, G_UMULH}).alwaysLegal();
665 }
666
667 getActionDefinitionsBuilder(G_IS_FPCLASS).custom();
668
669 getActionDefinitionsBuilder({G_INTRINSIC, G_INTRINSIC_CONVERGENT,
670 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
671 .alwaysLegal();
673 getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
674
675 verify(*ST.getInstrInfo());
676}
677
679 MachineInstr &MI) {
680 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
681 Register DstReg = MI.getOperand(0).getReg();
682 Register SrcReg = MI.getOperand(1).getReg();
683 Register IdxReg = MI.getOperand(2).getReg();
684
685 MIRBuilder
686 .buildIntrinsic(Intrinsic::spv_extractelt, ArrayRef<Register>{DstReg})
687 .addUse(SrcReg)
688 .addUse(IdxReg);
689 MI.eraseFromParent();
690 return true;
691}
692
694 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
695 Register DstReg = MI.getOperand(0).getReg();
696 Register SrcReg = MI.getOperand(1).getReg();
697 Register ValReg = MI.getOperand(2).getReg();
698 Register IdxReg = MI.getOperand(3).getReg();
699
700 MIRBuilder
701 .buildIntrinsic(Intrinsic::spv_insertelt, ArrayRef<Register>{DstReg})
702 .addUse(SrcReg)
703 .addUse(ValReg)
704 .addUse(IdxReg);
705 MI.eraseFromParent();
706 return true;
707}
708
710 LegalizerHelper &Helper,
713 Register ConvReg = MRI.createGenericVirtualRegister(ConvTy);
714 MRI.setRegClass(ConvReg, GR->getRegClass(SpvType));
715 GR->assignSPIRVTypeToVReg(SpvType, ConvReg, Helper.MIRBuilder.getMF());
716 Helper.MIRBuilder.buildInstr(TargetOpcode::G_PTRTOINT)
717 .addDef(ConvReg)
718 .addUse(Reg);
719 return ConvReg;
720}
721
724 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
725 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
726 Register DstReg = MI.getOperand(0).getReg();
727 Register PtrReg = MI.getOperand(1).getReg();
728 LLT DstTy = MRI.getType(DstReg);
729
730 if (!DstTy.isVector())
731 return true;
732
733 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
734 if (!needsVectorLegalization(DstTy, ST))
735 return true;
736
737 SmallVector<Register, 8> SplitRegs;
738 LLT EltTy = DstTy.getElementType();
739 unsigned NumElts = DstTy.getNumElements();
740
741 LLT PtrTy = MRI.getType(PtrReg);
742 auto Zero = MIRBuilder.buildConstant(LLT::scalar(32), 0);
743
744 for (unsigned i = 0; i < NumElts; ++i) {
745 auto Idx = MIRBuilder.buildConstant(LLT::scalar(32), i);
746 Register EltPtr = MRI.createGenericVirtualRegister(PtrTy);
747
748 MIRBuilder.buildIntrinsic(Intrinsic::spv_gep, ArrayRef<Register>{EltPtr})
749 .addImm(1) // InBounds
750 .addUse(PtrReg)
751 .addUse(Zero.getReg(0))
752 .addUse(Idx.getReg(0));
753
754 MachinePointerInfo EltPtrInfo;
755 Align EltAlign = Align(1);
756 if (!MI.memoperands_empty()) {
757 MachineMemOperand *MMO = *MI.memoperands_begin();
758 EltPtrInfo =
759 MMO->getPointerInfo().getWithOffset(i * EltTy.getSizeInBytes());
760 EltAlign = commonAlignment(MMO->getAlign(), i * EltTy.getSizeInBytes());
761 }
762
763 Register EltReg = MRI.createGenericVirtualRegister(EltTy);
764 MIRBuilder.buildLoad(EltReg, EltPtr, EltPtrInfo, EltAlign);
765 SplitRegs.push_back(EltReg);
766 }
767
768 MIRBuilder.buildBuildVector(DstReg, SplitRegs);
769 MI.eraseFromParent();
770 return true;
771}
772
775 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
776 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
777 Register ValReg = MI.getOperand(0).getReg();
778 Register PtrReg = MI.getOperand(1).getReg();
779 LLT ValTy = MRI.getType(ValReg);
780
781 assert(ValTy.isVector() && "Expected vector store");
782
783 SmallVector<Register, 8> SplitRegs;
784 LLT EltTy = ValTy.getElementType();
785 unsigned NumElts = ValTy.getNumElements();
786
787 for (unsigned i = 0; i < NumElts; ++i)
788 SplitRegs.push_back(MRI.createGenericVirtualRegister(EltTy));
789
790 MIRBuilder.buildUnmerge(SplitRegs, ValReg);
791
792 LLT PtrTy = MRI.getType(PtrReg);
793 auto Zero = MIRBuilder.buildConstant(LLT::scalar(32), 0);
794
795 for (unsigned i = 0; i < NumElts; ++i) {
796 auto Idx = MIRBuilder.buildConstant(LLT::scalar(32), i);
797 Register EltPtr = MRI.createGenericVirtualRegister(PtrTy);
798
799 MIRBuilder.buildIntrinsic(Intrinsic::spv_gep, ArrayRef<Register>{EltPtr})
800 .addImm(1) // InBounds
801 .addUse(PtrReg)
802 .addUse(Zero.getReg(0))
803 .addUse(Idx.getReg(0));
804
805 MachinePointerInfo EltPtrInfo;
806 Align EltAlign = Align(1);
807 if (!MI.memoperands_empty()) {
808 MachineMemOperand *MMO = *MI.memoperands_begin();
809 EltPtrInfo =
810 MMO->getPointerInfo().getWithOffset(i * EltTy.getSizeInBytes());
811 EltAlign = commonAlignment(MMO->getAlign(), i * EltTy.getSizeInBytes());
812 }
813
814 MIRBuilder.buildStore(SplitRegs[i], EltPtr, EltPtrInfo, EltAlign);
815 }
816
817 MI.eraseFromParent();
818 return true;
819}
820
823 LostDebugLocObserver &LocObserver) const {
824 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
825 switch (MI.getOpcode()) {
826 default:
827 // TODO: implement legalization for other opcodes.
828 return true;
829 case TargetOpcode::G_BITCAST:
830 return legalizeBitcast(Helper, MI);
831 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
832 return legalizeExtractVectorElt(Helper, MI);
833 case TargetOpcode::G_INSERT_VECTOR_ELT:
834 return legalizeInsertVectorElt(Helper, MI);
835 case TargetOpcode::G_INTRINSIC:
836 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
837 return legalizeIntrinsic(Helper, MI);
838 case TargetOpcode::G_IS_FPCLASS:
839 return legalizeIsFPClass(Helper, MI, LocObserver);
840 case TargetOpcode::G_ICMP: {
841 auto &Op0 = MI.getOperand(2);
842 auto &Op1 = MI.getOperand(3);
843 Register Reg0 = Op0.getReg();
844 Register Reg1 = Op1.getReg();
846 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
847 if ((!ST->canDirectlyComparePointers() ||
849 MRI.getType(Reg0).isPointer() && MRI.getType(Reg1).isPointer()) {
850 LLT ConvT = LLT::scalar(ST->getPointerSize());
851 Type *LLVMTy = IntegerType::get(MI.getMF()->getFunction().getContext(),
852 ST->getPointerSize());
853 SPIRVTypeInst SpirvTy = GR->getOrCreateSPIRVType(
854 LLVMTy, Helper.MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);
855 Op0.setReg(convertPtrToInt(Reg0, ConvT, SpirvTy, Helper, MRI, GR));
856 Op1.setReg(convertPtrToInt(Reg1, ConvT, SpirvTy, Helper, MRI, GR));
857 }
858 return true;
859 }
860 case TargetOpcode::G_LOAD:
861 return legalizeLoad(Helper, MI, GR);
862 case TargetOpcode::G_STORE:
863 return legalizeStore(Helper, MI, GR);
864 }
865}
866
869 Register SrcReg, LLT SrcTy,
870 MachinePointerInfo &PtrInfo, Align &VecAlign) {
871 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
872 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
873
874 VecAlign = Helper.getStackTemporaryAlignment(SrcTy);
875 auto StackTemp = Helper.createStackTemporary(
876 TypeSize::getFixed(SrcTy.getSizeInBytes()), VecAlign, PtrInfo);
877
878 // Set the type of StackTemp to a pointer to an array of the element type.
879 SPIRVTypeInst SpvSrcTy = GR->getSPIRVTypeForVReg(SrcReg);
880 SPIRVTypeInst EltSpvTy = GR->getScalarOrVectorComponentType(SpvSrcTy);
881 const Type *LLVMEltTy = GR->getTypeForSPIRVType(EltSpvTy);
882 const Type *LLVMArrTy =
883 ArrayType::get(const_cast<Type *>(LLVMEltTy), SrcTy.getNumElements());
884 SPIRVTypeInst ArrSpvTy = GR->getOrCreateSPIRVType(
885 LLVMArrTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);
886 SPIRVTypeInst PtrToArrSpvTy = GR->getOrCreateSPIRVPointerType(
887 ArrSpvTy, MIRBuilder, SPIRV::StorageClass::Function);
888
889 Register StackReg = StackTemp.getReg(0);
890 MRI.setRegClass(StackReg, GR->getRegClass(PtrToArrSpvTy));
891 GR->assignSPIRVTypeToVReg(PtrToArrSpvTy, StackReg, MIRBuilder.getMF());
892
893 return StackTemp;
894}
895
898 LLVM_DEBUG(dbgs() << "Found a bitcast instruction\n");
899 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
900 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
901 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
902
903 Register DstReg = MI.getOperand(0).getReg();
904 Register SrcReg = MI.getOperand(2).getReg();
905 LLT DstTy = MRI.getType(DstReg);
906 LLT SrcTy = MRI.getType(SrcReg);
907
908 // If an spv_bitcast needs to be legalized, we convert it to G_BITCAST to
909 // allow using the generic legalization rules.
910 if (needsVectorLegalization(DstTy, ST) ||
911 needsVectorLegalization(SrcTy, ST)) {
912 LLVM_DEBUG(dbgs() << "Replacing with a G_BITCAST\n");
913 MIRBuilder.buildBitcast(DstReg, SrcReg);
914 MI.eraseFromParent();
915 }
916 return true;
917}
918
921 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
922 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
923 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
924
925 Register DstReg = MI.getOperand(0).getReg();
926 LLT DstTy = MRI.getType(DstReg);
927
928 if (needsVectorLegalization(DstTy, ST)) {
929 Register SrcReg = MI.getOperand(2).getReg();
930 Register ValReg = MI.getOperand(3).getReg();
931 LLT SrcTy = MRI.getType(SrcReg);
932 MachineOperand &IdxOperand = MI.getOperand(4);
933
934 if (getImm(IdxOperand, &MRI)) {
935 uint64_t IdxVal = foldImm(IdxOperand, &MRI);
936 if (IdxVal < SrcTy.getNumElements()) {
938 SPIRVTypeInst ElementType =
940 LLT ElementLLTTy = GR->getRegType(ElementType);
941 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
942 Register Reg = MRI.createGenericVirtualRegister(ElementLLTTy);
943 MRI.setRegClass(Reg, GR->getRegClass(ElementType));
944 GR->assignSPIRVTypeToVReg(ElementType, Reg, *MI.getMF());
945 Regs.push_back(Reg);
946 }
947 MIRBuilder.buildUnmerge(Regs, SrcReg);
948 Regs[IdxVal] = ValReg;
949 MIRBuilder.buildBuildVector(DstReg, Regs);
950 MI.eraseFromParent();
951 return true;
952 }
953 }
954
955 LLT EltTy = SrcTy.getElementType();
956 Align VecAlign;
957 MachinePointerInfo PtrInfo;
958 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
959 PtrInfo, VecAlign);
960
961 MIRBuilder.buildStore(SrcReg, StackTemp, PtrInfo, VecAlign);
962
963 Register IdxReg = IdxOperand.getReg();
964 LLT PtrTy = MRI.getType(StackTemp.getReg(0));
965 Register EltPtr = MRI.createGenericVirtualRegister(PtrTy);
966 auto Zero = MIRBuilder.buildConstant(LLT::scalar(32), 0);
967
968 MIRBuilder.buildIntrinsic(Intrinsic::spv_gep, ArrayRef<Register>{EltPtr})
969 .addImm(1) // InBounds
970 .addUse(StackTemp.getReg(0))
971 .addUse(Zero.getReg(0))
972 .addUse(IdxReg);
973
975 Align EltAlign = Helper.getStackTemporaryAlignment(EltTy);
976 MIRBuilder.buildStore(ValReg, EltPtr, EltPtrInfo, EltAlign);
977
978 MIRBuilder.buildLoad(DstReg, StackTemp, PtrInfo, VecAlign);
979 MI.eraseFromParent();
980 return true;
981 }
982 return true;
983}
984
987 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
988 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
989 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
990
991 Register SrcReg = MI.getOperand(2).getReg();
992 LLT SrcTy = MRI.getType(SrcReg);
993
994 if (needsVectorLegalization(SrcTy, ST)) {
995 Register DstReg = MI.getOperand(0).getReg();
996 MachineOperand &IdxOperand = MI.getOperand(3);
997
998 if (getImm(IdxOperand, &MRI)) {
999 uint64_t IdxVal = foldImm(IdxOperand, &MRI);
1000 if (IdxVal < SrcTy.getNumElements()) {
1001 LLT DstTy = MRI.getType(DstReg);
1003 SPIRVTypeInst DstSpvTy = GR->getSPIRVTypeForVReg(DstReg);
1004 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
1005 if (I == IdxVal) {
1006 Regs.push_back(DstReg);
1007 } else {
1008 Register Reg = MRI.createGenericVirtualRegister(DstTy);
1009 MRI.setRegClass(Reg, GR->getRegClass(DstSpvTy));
1010 GR->assignSPIRVTypeToVReg(DstSpvTy, Reg, *MI.getMF());
1011 Regs.push_back(Reg);
1012 }
1013 }
1014 MIRBuilder.buildUnmerge(Regs, SrcReg);
1015 MI.eraseFromParent();
1016 return true;
1017 }
1018 }
1019
1020 LLT EltTy = SrcTy.getElementType();
1021 Align VecAlign;
1022 MachinePointerInfo PtrInfo;
1023 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
1024 PtrInfo, VecAlign);
1025
1026 MIRBuilder.buildStore(SrcReg, StackTemp, PtrInfo, VecAlign);
1027
1028 Register IdxReg = IdxOperand.getReg();
1029 LLT PtrTy = MRI.getType(StackTemp.getReg(0));
1030 Register EltPtr = MRI.createGenericVirtualRegister(PtrTy);
1031 auto Zero = MIRBuilder.buildConstant(LLT::scalar(32), 0);
1032
1033 MIRBuilder.buildIntrinsic(Intrinsic::spv_gep, ArrayRef<Register>{EltPtr})
1034 .addImm(1) // InBounds
1035 .addUse(StackTemp.getReg(0))
1036 .addUse(Zero.getReg(0))
1037 .addUse(IdxReg);
1038
1040 Align EltAlign = Helper.getStackTemporaryAlignment(EltTy);
1041 MIRBuilder.buildLoad(DstReg, EltPtr, EltPtrInfo, EltAlign);
1042
1043 MI.eraseFromParent();
1044 return true;
1045 }
1046 return true;
1047}
1048
1050 SPIRVGlobalRegistry *GR) {
1051 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1052 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1053 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
1054
1055 Register DstReg = MI.getOperand(0).getReg();
1056 LLT DstTy = MRI.getType(DstReg);
1057
1058 if (!needsVectorLegalization(DstTy, ST))
1059 return true;
1060
1062 if (MI.getNumOperands() == 2) {
1063 // The "null" case: no values are attached.
1064 LLT EltTy = DstTy.getElementType();
1065 auto Zero = MIRBuilder.buildConstant(EltTy, 0);
1066 SPIRVTypeInst SpvDstTy = GR->getSPIRVTypeForVReg(DstReg);
1067 SPIRVTypeInst SpvEltTy = GR->getScalarOrVectorComponentType(SpvDstTy);
1068 GR->assignSPIRVTypeToVReg(SpvEltTy, Zero.getReg(0), MIRBuilder.getMF());
1069 for (unsigned i = 0; i < DstTy.getNumElements(); ++i)
1070 SrcRegs.push_back(Zero.getReg(0));
1071 } else {
1072 for (unsigned i = 2; i < MI.getNumOperands(); ++i) {
1073 SrcRegs.push_back(MI.getOperand(i).getReg());
1074 }
1075 }
1076 MIRBuilder.buildBuildVector(DstReg, SrcRegs);
1077 MI.eraseFromParent();
1078 return true;
1079}
1080
1082 MachineIRBuilder &MIRBuilder,
1083 SPIRVGlobalRegistry *GR) {
1084 LLT Ty = MIRBuilder.getMRI()->getType(Reg);
1085 if (!Ty.isVector())
1086 return {Reg};
1087 SPIRVTypeInst EltSpvTy =
1089 unsigned NumElts = Ty.getNumElements();
1091 for (unsigned I = 0; I < NumElts; ++I)
1092 Elts.push_back(createVirtualRegister(EltSpvTy, GR, MIRBuilder));
1093 MIRBuilder.buildUnmerge(Elts, Reg);
1094 return Elts;
1095}
1096
1098 MachineIRBuilder &MIRBuilder,
1099 SPIRVGlobalRegistry *GR) {
1100 if (Elts.size() == 1)
1101 return Elts[0];
1102 SPIRVTypeInst PartSpvTy =
1104 Elts.size(), MIRBuilder, /*EmitIR=*/true);
1105 Register Part = createVirtualRegister(PartSpvTy, GR, MIRBuilder);
1106 MIRBuilder.buildBuildVector(Part, Elts);
1107 return Part;
1108}
1109
1110// Split an elementwise intrinsic with an illegal vector width into intrinsics
1111// on legal vector widths.
1113 GIntrinsic &MI,
1114 SPIRVGlobalRegistry *GR) {
1115 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1116 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1117 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
1118
1119 if (!Intrinsic::isTriviallyScalarizable(MI.getIntrinsicID()))
1120 return true;
1121 Register DstReg = MI.getReg(0);
1122 LLT DstTy = MRI.getType(DstReg);
1123 if (!needsVectorLegalization(DstTy, ST))
1124 return true;
1125
1126 unsigned NumElts = DstTy.getNumElements();
1127 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
1128 unsigned PartSize = NumElts > MaxVectorSize ? MaxVectorSize : 4;
1129
1131 for (const MachineOperand &MO : drop_begin(MI.explicit_uses())) {
1132 if (!MO.isReg() || !MRI.getType(MO.getReg()).isVector())
1133 continue;
1134 auto [It, Inserted] = OpElts.try_emplace(MO.getReg());
1135 if (Inserted)
1136 It->second = unmergeToScalars(MO.getReg(), MIRBuilder, GR);
1137 }
1138
1139 SPIRVTypeInst DstEltSpvTy =
1142 for (unsigned Offset = 0; Offset < NumElts; Offset += PartSize) {
1143 unsigned Size = std::min(PartSize, NumElts - Offset);
1144 SPIRVTypeInst PartSpvTy =
1145 Size == 1 ? DstEltSpvTy
1146 : GR->getOrCreateSPIRVVectorType(DstEltSpvTy, Size,
1147 MIRBuilder, /*EmitIR=*/true);
1150 for (const MachineOperand &MO : drop_begin(MI.explicit_uses())) {
1151 auto EltsIt = MO.isReg() ? OpElts.find(MO.getReg()) : OpElts.end();
1152 if (EltsIt == OpElts.end()) {
1153 PartOps.push_back(MO);
1154 continue;
1155 }
1156 auto [It, Inserted] = PartRegs.try_emplace(MO.getReg());
1157 if (Inserted)
1158 It->second = buildVectorPart(
1159 ArrayRef(EltsIt->second).slice(Offset, Size), MIRBuilder, GR);
1160 PartOps.push_back(MachineOperand::CreateReg(It->second, /*isDef=*/false));
1161 }
1162 Register PartDst = createVirtualRegister(PartSpvTy, GR, MIRBuilder);
1163 auto Part = MIRBuilder.buildIntrinsic(
1164 MI.getIntrinsicID(), ArrayRef<Register>{PartDst}, MI.hasSideEffects(),
1165 MI.isConvergent());
1166 for (const MachineOperand &MO : PartOps)
1167 Part.add(MO);
1168 Part->setFlags(MI.getFlags());
1169 append_range(DstElts, unmergeToScalars(PartDst, MIRBuilder, GR));
1170 }
1171
1172 MIRBuilder.buildBuildVector(DstReg, DstElts);
1173 MI.eraseFromParent();
1174 return true;
1175}
1176
1178 MachineInstr &MI) const {
1179 LLVM_DEBUG(dbgs() << "legalizeIntrinsic: " << MI);
1180 auto IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
1181 switch (IntrinsicID) {
1182 case Intrinsic::spv_bitcast:
1183 return legalizeSpvBitcast(Helper, MI, GR);
1184 case Intrinsic::spv_insertelt:
1185 return legalizeSpvInsertElt(Helper, MI, GR);
1186 case Intrinsic::spv_extractelt:
1187 return legalizeSpvExtractElt(Helper, MI, GR);
1188 case Intrinsic::spv_const_composite:
1189 return legalizeSpvConstComposite(Helper, MI, GR);
1190 }
1192}
1193
1194bool SPIRVLegalizerInfo::legalizeBitcast(LegalizerHelper &Helper,
1195 MachineInstr &MI) const {
1196 // Once the G_BITCAST is using vectors that are allowed, we turn it back into
1197 // an spv_bitcast to avoid verifier problems when the register types are the
1198 // same for the source and the result. Note that the SPIR-V types associated
1199 // with the bitcast can be different even if the register types are the same.
1200 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1201 Register DstReg = MI.getOperand(0).getReg();
1202 Register SrcReg = MI.getOperand(1).getReg();
1203 SmallVector<Register, 1> DstRegs = {DstReg};
1204 MIRBuilder.buildIntrinsic(Intrinsic::spv_bitcast, DstRegs).addUse(SrcReg);
1205 MI.eraseFromParent();
1206 return true;
1207}
1208
1209// Note this code was copied from LegalizerHelper::lowerISFPCLASS and adjusted
1210// to ensure that all instructions created during the lowering have SPIR-V types
1211// assigned to them.
1212bool SPIRVLegalizerInfo::legalizeIsFPClass(
1214 LostDebugLocObserver &LocObserver) const {
1215 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1216 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(2).getImm());
1217
1218 auto &MIRBuilder = Helper.MIRBuilder;
1219 auto &MF = MIRBuilder.getMF();
1220 MachineRegisterInfo &MRI = MF.getRegInfo();
1221
1222 Type *LLVMDstTy =
1223 IntegerType::get(MIRBuilder.getContext(), DstTy.getScalarSizeInBits());
1224 if (DstTy.isVector())
1225 LLVMDstTy = VectorType::get(LLVMDstTy, DstTy.getElementCount());
1226 SPIRVTypeInst SPIRVDstTy = GR->getOrCreateSPIRVType(
1227 LLVMDstTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
1228 /*EmitIR*/ true);
1229
1230 unsigned BitSize = SrcTy.getScalarSizeInBits();
1231 const fltSemantics &Semantics = getFltSemanticForLLT(SrcTy.getScalarType());
1232
1233 LLT IntTy = LLT::scalar(BitSize);
1234 Type *LLVMIntTy = IntegerType::get(MIRBuilder.getContext(), BitSize);
1235 if (SrcTy.isVector()) {
1236 IntTy = LLT::vector(SrcTy.getElementCount(), IntTy);
1237 LLVMIntTy = VectorType::get(LLVMIntTy, SrcTy.getElementCount());
1238 }
1239 SPIRVTypeInst SPIRVIntTy = GR->getOrCreateSPIRVType(
1240 LLVMIntTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
1241 /*EmitIR*/ true);
1242
1243 // Clang doesn't support capture of structured bindings:
1244 LLT DstTyCopy = DstTy;
1245 const auto assignSPIRVTy = [&](MachineInstrBuilder &&MI) {
1246 // Assign this MI's (assumed only) destination to one of the two types we
1247 // expect: either the G_IS_FPCLASS's destination type, or the integer type
1248 // bitcast from the source type.
1249 LLT MITy = MRI.getType(MI.getReg(0));
1250 assert((MITy == IntTy || MITy == DstTyCopy) &&
1251 "Unexpected LLT type while lowering G_IS_FPCLASS");
1252 SPIRVTypeInst SPVTy = MITy == IntTy ? SPIRVIntTy : SPIRVDstTy;
1253 GR->assignSPIRVTypeToVReg(SPVTy, MI.getReg(0), MF);
1254 return MI;
1255 };
1256
1257 // Helper to build and assign a constant in one go
1258 const auto buildSPIRVConstant = [&](LLT Ty, auto &&C) -> MachineInstrBuilder {
1259 if (!Ty.isFixedVector())
1260 return assignSPIRVTy(MIRBuilder.buildConstant(Ty, C));
1261 auto ScalarC = MIRBuilder.buildConstant(Ty.getScalarType(), C);
1262 assert((Ty == IntTy || Ty == DstTyCopy) &&
1263 "Unexpected LLT type while lowering constant for G_IS_FPCLASS");
1264 SPIRVTypeInst VecEltTy = GR->getOrCreateSPIRVType(
1265 (Ty == IntTy ? LLVMIntTy : LLVMDstTy)->getScalarType(), MIRBuilder,
1266 SPIRV::AccessQualifier::ReadWrite,
1267 /*EmitIR*/ true);
1268 GR->assignSPIRVTypeToVReg(VecEltTy, ScalarC.getReg(0), MF);
1269 return assignSPIRVTy(MIRBuilder.buildSplatBuildVector(Ty, ScalarC));
1270 };
1271
1272 if (Mask == fcNone) {
1273 MIRBuilder.buildCopy(DstReg, buildSPIRVConstant(DstTy, 0));
1274 MI.eraseFromParent();
1275 return true;
1276 }
1277 if (Mask == fcAllFlags) {
1278 MIRBuilder.buildCopy(DstReg, buildSPIRVConstant(DstTy, 1));
1279 MI.eraseFromParent();
1280 return true;
1281 }
1282
1283 // Note that rather than creating a COPY here (between a floating-point and
1284 // integer type of the same size) we create a SPIR-V bitcast immediately. We
1285 // can't create a G_BITCAST because the LLTs are the same, and we can't seem
1286 // to correctly lower COPYs to SPIR-V bitcasts at this moment.
1287 Register ResVReg = MRI.createGenericVirtualRegister(IntTy);
1288 MRI.setRegClass(ResVReg, GR->getRegClass(SPIRVIntTy));
1289 GR->assignSPIRVTypeToVReg(SPIRVIntTy, ResVReg, Helper.MIRBuilder.getMF());
1290 auto AsInt = MIRBuilder.buildInstr(SPIRV::OpBitcast)
1291 .addDef(ResVReg)
1292 .addUse(GR->getSPIRVTypeID(SPIRVIntTy))
1293 .addUse(SrcReg);
1294 AsInt = assignSPIRVTy(std::move(AsInt));
1295
1296 // Various masks.
1297 APInt SignBit = APInt::getSignMask(BitSize);
1298 APInt ValueMask = APInt::getSignedMaxValue(BitSize); // All bits but sign.
1299 APInt Inf = APFloat::getInf(Semantics).bitcastToAPInt(); // Exp and int bit.
1300 APInt ExpMask = Inf;
1301 APInt AllOneMantissa = APFloat::getLargest(Semantics).bitcastToAPInt() & ~Inf;
1302 APInt QNaNBitMask =
1303 APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1);
1304 APInt InversionMask = APInt::getAllOnes(DstTy.getScalarSizeInBits());
1305
1306 auto SignBitC = buildSPIRVConstant(IntTy, SignBit);
1307 auto ValueMaskC = buildSPIRVConstant(IntTy, ValueMask);
1308 auto InfC = buildSPIRVConstant(IntTy, Inf);
1309 auto ExpMaskC = buildSPIRVConstant(IntTy, ExpMask);
1310 auto ZeroC = buildSPIRVConstant(IntTy, 0);
1311
1312 auto Abs = assignSPIRVTy(MIRBuilder.buildAnd(IntTy, AsInt, ValueMaskC));
1313 auto Sign = assignSPIRVTy(
1314 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_NE, DstTy, AsInt, Abs));
1315
1316 MachineInstrBuilder Res;
1317
1318 const auto appendToRes = [&](MachineInstrBuilder &&ToAppend) {
1319 auto Appended = assignSPIRVTy(std::move(ToAppend));
1320 Res = Res.getInstr()
1321 ? assignSPIRVTy(MIRBuilder.buildOr(DstTyCopy, Res, Appended))
1322 : Appended;
1323 };
1324
1325 // Tests that involve more than one class should be processed first.
1326 if ((Mask & fcFinite) == fcFinite) {
1327 // finite(V) ==> abs(V) u< exp_mask
1328 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs,
1329 ExpMaskC));
1330 Mask &= ~fcFinite;
1331 } else if ((Mask & fcFinite) == fcPosFinite) {
1332 // finite(V) && V > 0 ==> V u< exp_mask
1333 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, AsInt,
1334 ExpMaskC));
1335 Mask &= ~fcPosFinite;
1336 } else if ((Mask & fcFinite) == fcNegFinite) {
1337 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
1338 auto Cmp = assignSPIRVTy(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT,
1339 DstTy, Abs, ExpMaskC));
1340 appendToRes(MIRBuilder.buildAnd(DstTy, Cmp, Sign));
1341 Mask &= ~fcNegFinite;
1342 }
1343
1344 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
1345 // fcZero | fcSubnormal => test all exponent bits are 0
1346 // TODO: Handle sign bit specific cases
1347 // TODO: Handle inverted case
1348 if (PartialCheck == (fcZero | fcSubnormal)) {
1349 auto ExpBits = assignSPIRVTy(MIRBuilder.buildAnd(IntTy, AsInt, ExpMaskC));
1350 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
1351 ExpBits, ZeroC));
1352 Mask &= ~PartialCheck;
1353 }
1354 }
1355
1356 // Check for individual classes.
1357 if (FPClassTest PartialCheck = Mask & fcZero) {
1358 if (PartialCheck == fcPosZero)
1359 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
1360 AsInt, ZeroC));
1361 else if (PartialCheck == fcZero)
1362 appendToRes(
1363 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, ZeroC));
1364 else // fcNegZero
1365 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
1366 AsInt, SignBitC));
1367 }
1368
1369 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
1370 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
1371 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
1372 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
1373 auto OneC = buildSPIRVConstant(IntTy, 1);
1374 auto VMinusOne = MIRBuilder.buildSub(IntTy, V, OneC);
1375 auto SubnormalRes = assignSPIRVTy(
1376 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, VMinusOne,
1377 buildSPIRVConstant(IntTy, AllOneMantissa)));
1378 if (PartialCheck == fcNegSubnormal)
1379 SubnormalRes = MIRBuilder.buildAnd(DstTy, SubnormalRes, Sign);
1380 appendToRes(std::move(SubnormalRes));
1381 }
1382
1383 if (FPClassTest PartialCheck = Mask & fcInf) {
1384 if (PartialCheck == fcPosInf)
1385 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
1386 AsInt, InfC));
1387 else if (PartialCheck == fcInf)
1388 appendToRes(
1389 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, InfC));
1390 else { // fcNegInf
1391 APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt();
1392 auto NegInfC = buildSPIRVConstant(IntTy, NegInf);
1393 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
1394 AsInt, NegInfC));
1395 }
1396 }
1397
1398 if (FPClassTest PartialCheck = Mask & fcNan) {
1399 auto InfWithQnanBitC =
1400 buildSPIRVConstant(IntTy, std::move(Inf) | QNaNBitMask);
1401 if (PartialCheck == fcNan) {
1402 // isnan(V) ==> abs(V) u> int(inf)
1403 appendToRes(
1404 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC));
1405 } else if (PartialCheck == fcQNan) {
1406 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
1407 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGE, DstTy, Abs,
1408 InfWithQnanBitC));
1409 } else { // fcSNan
1410 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
1411 // abs(V) u< (unsigned(Inf) | quiet_bit)
1412 auto IsNan = assignSPIRVTy(
1413 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC));
1414 auto IsNotQnan = assignSPIRVTy(MIRBuilder.buildICmp(
1415 CmpInst::Predicate::ICMP_ULT, DstTy, Abs, InfWithQnanBitC));
1416 appendToRes(MIRBuilder.buildAnd(DstTy, IsNan, IsNotQnan));
1417 }
1418 }
1419
1420 if (FPClassTest PartialCheck = Mask & fcNormal) {
1421 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
1422 // (max_exp-1))
1423 APInt ExpLSB = ExpMask & ~(ExpMask.shl(1));
1424 auto ExpMinusOne = assignSPIRVTy(
1425 MIRBuilder.buildSub(IntTy, Abs, buildSPIRVConstant(IntTy, ExpLSB)));
1426 APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
1427 auto NormalRes = assignSPIRVTy(
1428 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, ExpMinusOne,
1429 buildSPIRVConstant(IntTy, MaxExpMinusOne)));
1430 if (PartialCheck == fcNegNormal)
1431 NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, Sign);
1432 else if (PartialCheck == fcPosNormal) {
1433 auto PosSign = assignSPIRVTy(MIRBuilder.buildXor(
1434 DstTy, Sign, buildSPIRVConstant(DstTy, InversionMask)));
1435 NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, PosSign);
1436 }
1437 appendToRes(std::move(NormalRes));
1438 }
1439
1440 MIRBuilder.buildCopy(DstReg, Res);
1441 MI.eraseFromParent();
1442 return true;
1443}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
bpf unwind lower
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static void scalarize(Instruction *I, SmallVectorImpl< Instruction * > &Worklist)
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition MD5.cpp:57
This file declares the MachineIRBuilder class.
Register Reg
Promote Memory to Register
Definition Mem2Reg.cpp:110
ppc ctr loops verify
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
SPIR-V address space definition.
static bool legalizeSpvInsertElt(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static bool needsVectorLegalization(const LLT &Ty, const SPIRVSubtarget &ST)
static SmallVector< Register, 16 > unmergeToScalars(Register Reg, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool legalizeInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI)
static MachineInstrBuilder createStackTemporaryForVector(LegalizerHelper &Helper, SPIRVGlobalRegistry *GR, Register SrcReg, LLT SrcTy, MachinePointerInfo &PtrInfo, Align &VecAlign)
static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType, LegalizerHelper &Helper, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR)
LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts)
static bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static Register buildVectorPart(ArrayRef< Register > Elts, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool legalizeExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI)
static bool legalizeSpvExtractElt(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
LegalityPredicate typeOfLongVectors(unsigned TypeIdx, bool IsLongVecs)
static bool legalizeSpvBitcast(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static bool legalizeElementwiseIntrinsic(LegalizerHelper &Helper, GIntrinsic &MI, SPIRVGlobalRegistry *GR)
static bool legalizeSpvConstComposite(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
#define LLVM_DEBUG(...)
Definition Debug.h:119
APInt bitcastToAPInt() const
Definition APFloat.h:1475
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition APFloat.h:1242
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition APInt.h:875
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ ICMP_NE
not equal
Definition InstrTypes.h:762
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
Represents a call to an intrinsic.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
LLT getScalarType() const
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr bool isPointer() const
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isPointerOrPointerVector() const
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & unsupportedFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & alwaysLegal()
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LLVM_ABI MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, MachinePointerInfo &PtrInfo)
Create a stack temporary based on the size in bytes and the alignment.
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LLVM_ABI Align getStackTemporaryAlignment(LLT Type, Align MinAlign=Align()) const
Return the alignment to use for a stack temporary object with the given type.
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
Helper class to build MachineInstr.
LLVMContext & getContext() const
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void setFlags(unsigned flags)
A description of a memory reference used in the backend.
const MachinePointerInfo & getPointerInfo() const
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
void assignSPIRVTypeToVReg(SPIRVTypeInst Type, Register VReg, const MachineFunction &MF)
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
const TargetRegisterClass * getRegClass(SPIRVTypeInst SpvType) const
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
LLT getRegType(SPIRVTypeInst SpvType) const
SPIRVTypeInst getScalarOrVectorComponentType(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
SPIRVLegalizerInfo(const SPIRVSubtarget &ST)
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
SPIRVGlobalRegistry * getSPIRVGlobalRegistry() const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI bool isTriviallyScalarizable(ID id)
Returns true if the intrinsic is trivially scalarizable.
LLVM_ABI LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LLVM_ABI LegalityPredicate numElementsNotPow2(unsigned TypeIdx)
True iff the specified type index is a vector whose element count is not a power of 2.
LLVM_ABI LegalityPredicate isPointerVector(unsigned TypeIdx)
True iff the specified type index is a vector of pointers (with any address space).
LLVM_ABI LegalityPredicate vectorElementCountIsLessThanOrEqualTo(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a vector with a number of elements that's less than or equal to ...
LLVM_ABI LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LLVM_ABI LegalityPredicate vectorElementCountIsGreaterThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a vector with a number of elements that's greater than the given...
Predicate any(Predicate P0, Predicate P1)
True iff P0 or P1 are true.
LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type)
True iff the given type index is not the specified type.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LLVM_ABI LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Predicate predNot(Predicate P)
True iff P is false.
LLVM_ABI LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as TypeIdx, but take the number of elements from FromTypeIdx.
LLVM_ABI LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as the given type index.
LLVM_ABI LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
Invariant opcodes: All instruction sets have these as their low opcodes.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
LLVM_ABI const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
std::function< bool(const LegalityQuery &)> LegalityPredicate
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
const std::set< unsigned > & getTypeFoldingSupportedOpcodes()
int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< LLT > Types
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const