LLVM 22.0.0git
AMDGPULegalizerInfo.cpp
Go to the documentation of this file.
1//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPULegalizerInfo.h"
15
16#include "AMDGPU.h"
18#include "AMDGPUInstrInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUTargetMachine.h"
22#include "SIInstrInfo.h"
24#include "SIRegisterInfo.h"
26#include "llvm/ADT/ScopeExit.h"
35#include "llvm/IR/IntrinsicsAMDGPU.h"
36#include "llvm/IR/IntrinsicsR600.h"
37
38#define DEBUG_TYPE "amdgpu-legalinfo"
39
40using namespace llvm;
41using namespace LegalizeActions;
42using namespace LegalizeMutations;
43using namespace LegalityPredicates;
44using namespace MIPatternMatch;
45
46// Hack until load/store selection patterns support any tuple of legal types.
48 "amdgpu-global-isel-new-legality",
49 cl::desc("Use GlobalISel desired legality, rather than try to use"
50 "rules compatible with selection patterns"),
51 cl::init(false),
53
54static constexpr unsigned MaxRegisterSize = 1024;
55
56// Round the number of elements to the next power of two elements
58 unsigned NElts = Ty.getNumElements();
59 unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts);
60 return Ty.changeElementCount(ElementCount::getFixed(Pow2NElts));
61}
62
63// Round the number of bits to the next power of two bits
65 unsigned Bits = Ty.getSizeInBits();
66 unsigned Pow2Bits = 1 << Log2_32_Ceil(Bits);
67 return LLT::scalar(Pow2Bits);
68}
69
70/// \returns true if this is an odd sized vector which should widen by adding an
71/// additional element. This is mostly to handle <3 x s16> -> <4 x s16>. This
72/// excludes s1 vectors, which should always be scalarized.
73static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
74 return [=](const LegalityQuery &Query) {
75 const LLT Ty = Query.Types[TypeIdx];
76 if (!Ty.isVector())
77 return false;
78
79 const LLT EltTy = Ty.getElementType();
80 const unsigned EltSize = EltTy.getSizeInBits();
81 return Ty.getNumElements() % 2 != 0 &&
82 EltSize > 1 && EltSize < 32 &&
83 Ty.getSizeInBits() % 32 != 0;
84 };
85}
86
87static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx) {
88 return [=](const LegalityQuery &Query) {
89 const LLT Ty = Query.Types[TypeIdx];
90 return Ty.getSizeInBits() % 32 == 0;
91 };
92}
93
94static LegalityPredicate isWideVec16(unsigned TypeIdx) {
95 return [=](const LegalityQuery &Query) {
96 const LLT Ty = Query.Types[TypeIdx];
97 const LLT EltTy = Ty.getScalarType();
98 return EltTy.getSizeInBits() == 16 && Ty.getNumElements() > 2;
99 };
100}
101
102static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
103 return [=](const LegalityQuery &Query) {
104 const LLT Ty = Query.Types[TypeIdx];
105 const LLT EltTy = Ty.getElementType();
106 return std::pair(TypeIdx,
107 LLT::fixed_vector(Ty.getNumElements() + 1, EltTy));
108 };
109}
110
112 return [=](const LegalityQuery &Query) {
113 const LLT Ty = Query.Types[TypeIdx];
114 const LLT EltTy = Ty.getElementType();
115 unsigned Size = Ty.getSizeInBits();
116 unsigned Pieces = (Size + 63) / 64;
117 unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
118 return std::pair(TypeIdx, LLT::scalarOrVector(
119 ElementCount::getFixed(NewNumElts), EltTy));
120 };
121}
122
123// Increase the number of vector elements to reach the next multiple of 32-bit
124// type.
125static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
126 return [=](const LegalityQuery &Query) {
127 const LLT Ty = Query.Types[TypeIdx];
128
129 const LLT EltTy = Ty.getElementType();
130 const int Size = Ty.getSizeInBits();
131 const int EltSize = EltTy.getSizeInBits();
132 const int NextMul32 = (Size + 31) / 32;
133
134 assert(EltSize < 32);
135
136 const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
137 return std::pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltTy));
138 };
139}
140
141// Retrieves the scalar type that's the same size as the mem desc
143 return [=](const LegalityQuery &Query) {
144 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
145 return std::make_pair(TypeIdx, LLT::scalar(MemSize));
146 };
147}
148
149// Increase the number of vector elements to reach the next legal RegClass.
151 return [=](const LegalityQuery &Query) {
152 const LLT Ty = Query.Types[TypeIdx];
153 const unsigned NumElts = Ty.getNumElements();
154 const unsigned EltSize = Ty.getElementType().getSizeInBits();
155 const unsigned MaxNumElts = MaxRegisterSize / EltSize;
156
157 assert(EltSize == 32 || EltSize == 64);
158 assert(Ty.getSizeInBits() < MaxRegisterSize);
159
160 unsigned NewNumElts;
161 // Find the nearest legal RegClass that is larger than the current type.
162 for (NewNumElts = NumElts; NewNumElts < MaxNumElts; ++NewNumElts) {
163 if (SIRegisterInfo::getSGPRClassForBitWidth(NewNumElts * EltSize))
164 break;
165 }
166 return std::pair(TypeIdx,
167 LLT::fixed_vector(NewNumElts, Ty.getElementType()));
168 };
169}
170
172 if (!Ty.isVector())
173 return LLT::scalar(128);
174 const ElementCount NumElems = Ty.getElementCount();
175 return LLT::vector(NumElems, LLT::scalar(128));
176}
177
179 if (!Ty.isVector())
180 return LLT::fixed_vector(4, LLT::scalar(32));
181 const unsigned NumElems = Ty.getElementCount().getFixedValue();
182 return LLT::fixed_vector(NumElems * 4, LLT::scalar(32));
183}
184
186 const unsigned Size = Ty.getSizeInBits();
187
188 if (Size <= 32) {
189 // <2 x s8> -> s16
190 // <4 x s8> -> s32
191 return LLT::scalar(Size);
192 }
193
195}
196
197static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
198 return [=](const LegalityQuery &Query) {
199 const LLT Ty = Query.Types[TypeIdx];
200 return std::pair(TypeIdx, getBitcastRegisterType(Ty));
201 };
202}
203
205 return [=](const LegalityQuery &Query) {
206 const LLT Ty = Query.Types[TypeIdx];
207 unsigned Size = Ty.getSizeInBits();
208 assert(Size % 32 == 0);
209 return std::pair(
211 };
212}
213
214static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
215 return [=](const LegalityQuery &Query) {
216 const LLT QueryTy = Query.Types[TypeIdx];
217 return QueryTy.isVector() && QueryTy.getSizeInBits() < Size;
218 };
219}
220
221static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
222 return [=](const LegalityQuery &Query) {
223 const LLT QueryTy = Query.Types[TypeIdx];
224 return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
225 };
226}
227
228static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
229 return [=](const LegalityQuery &Query) {
230 const LLT QueryTy = Query.Types[TypeIdx];
231 return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
232 };
233}
234
235static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size) {
236 return ((ST.useRealTrue16Insts() && Size == 16) || Size % 32 == 0) &&
238}
239
241 const int EltSize = EltTy.getSizeInBits();
242 return EltSize == 16 || EltSize % 32 == 0;
243}
244
245static bool isRegisterVectorType(LLT Ty) {
246 const int EltSize = Ty.getElementType().getSizeInBits();
247 return EltSize == 32 || EltSize == 64 ||
248 (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
249 EltSize == 128 || EltSize == 256;
250}
251
252// TODO: replace all uses of isRegisterType with isRegisterClassType
253static bool isRegisterType(const GCNSubtarget &ST, LLT Ty) {
254 if (!isRegisterSize(ST, Ty.getSizeInBits()))
255 return false;
256
257 if (Ty.isVector())
258 return isRegisterVectorType(Ty);
259
260 return true;
261}
262
263// Any combination of 32 or 64-bit elements up the maximum register size, and
264// multiples of v2s16.
266 unsigned TypeIdx) {
267 return [=, &ST](const LegalityQuery &Query) {
268 return isRegisterType(ST, Query.Types[TypeIdx]);
269 };
270}
271
272// RegisterType that doesn't have a corresponding RegClass.
273// TODO: Once `isRegisterType` is replaced with `isRegisterClassType` this
274// should be removed.
276 unsigned TypeIdx) {
277 return [=, &ST](const LegalityQuery &Query) {
278 LLT Ty = Query.Types[TypeIdx];
279 return isRegisterType(ST, Ty) &&
280 !SIRegisterInfo::getSGPRClassForBitWidth(Ty.getSizeInBits());
281 };
282}
283
284static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
285 return [=](const LegalityQuery &Query) {
286 const LLT QueryTy = Query.Types[TypeIdx];
287 if (!QueryTy.isVector())
288 return false;
289 const LLT EltTy = QueryTy.getElementType();
290 return EltTy == LLT::scalar(16) || EltTy.getSizeInBits() >= 32;
291 };
292}
293
294constexpr LLT S1 = LLT::scalar(1);
295constexpr LLT S8 = LLT::scalar(8);
296constexpr LLT S16 = LLT::scalar(16);
297constexpr LLT S32 = LLT::scalar(32);
298constexpr LLT F32 = LLT::float32();
299constexpr LLT S64 = LLT::scalar(64);
300constexpr LLT F64 = LLT::float64();
301constexpr LLT S96 = LLT::scalar(96);
302constexpr LLT S128 = LLT::scalar(128);
303constexpr LLT S160 = LLT::scalar(160);
304constexpr LLT S192 = LLT::scalar(192);
305constexpr LLT S224 = LLT::scalar(224);
306constexpr LLT S256 = LLT::scalar(256);
307constexpr LLT S512 = LLT::scalar(512);
308constexpr LLT S1024 = LLT::scalar(1024);
310
311constexpr LLT V2S8 = LLT::fixed_vector(2, 8);
312constexpr LLT V2S16 = LLT::fixed_vector(2, 16);
313constexpr LLT V4S16 = LLT::fixed_vector(4, 16);
314constexpr LLT V6S16 = LLT::fixed_vector(6, 16);
315constexpr LLT V8S16 = LLT::fixed_vector(8, 16);
316constexpr LLT V10S16 = LLT::fixed_vector(10, 16);
317constexpr LLT V12S16 = LLT::fixed_vector(12, 16);
318constexpr LLT V16S16 = LLT::fixed_vector(16, 16);
319
321constexpr LLT V2BF16 = V2F16; // FIXME
322
323constexpr LLT V2S32 = LLT::fixed_vector(2, 32);
324constexpr LLT V3S32 = LLT::fixed_vector(3, 32);
325constexpr LLT V4S32 = LLT::fixed_vector(4, 32);
326constexpr LLT V5S32 = LLT::fixed_vector(5, 32);
327constexpr LLT V6S32 = LLT::fixed_vector(6, 32);
328constexpr LLT V7S32 = LLT::fixed_vector(7, 32);
329constexpr LLT V8S32 = LLT::fixed_vector(8, 32);
330constexpr LLT V9S32 = LLT::fixed_vector(9, 32);
331constexpr LLT V10S32 = LLT::fixed_vector(10, 32);
332constexpr LLT V11S32 = LLT::fixed_vector(11, 32);
333constexpr LLT V12S32 = LLT::fixed_vector(12, 32);
334constexpr LLT V16S32 = LLT::fixed_vector(16, 32);
335constexpr LLT V32S32 = LLT::fixed_vector(32, 32);
336
337constexpr LLT V2S64 = LLT::fixed_vector(2, 64);
338constexpr LLT V3S64 = LLT::fixed_vector(3, 64);
339constexpr LLT V4S64 = LLT::fixed_vector(4, 64);
340constexpr LLT V5S64 = LLT::fixed_vector(5, 64);
341constexpr LLT V6S64 = LLT::fixed_vector(6, 64);
342constexpr LLT V7S64 = LLT::fixed_vector(7, 64);
343constexpr LLT V8S64 = LLT::fixed_vector(8, 64);
344constexpr LLT V16S64 = LLT::fixed_vector(16, 64);
345
346constexpr LLT V2S128 = LLT::fixed_vector(2, 128);
347constexpr LLT V4S128 = LLT::fixed_vector(4, 128);
348
349constexpr std::initializer_list<LLT> AllScalarTypes = {
351
352constexpr std::initializer_list<LLT> AllS16Vectors{
354
355constexpr std::initializer_list<LLT> AllS32Vectors = {
358
359constexpr std::initializer_list<LLT> AllS64Vectors = {
361
367
368// Checks whether a type is in the list of legal register types.
369static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty) {
370 if (Ty.isPointerOrPointerVector())
371 Ty = Ty.changeElementType(LLT::scalar(Ty.getScalarSizeInBits()));
372
375 (ST.useRealTrue16Insts() && Ty == S16) ||
377}
378
380 unsigned TypeIdx) {
381 return [&ST, TypeIdx](const LegalityQuery &Query) {
382 return isRegisterClassType(ST, Query.Types[TypeIdx]);
383 };
384}
385
386// If we have a truncating store or an extending load with a data size larger
387// than 32-bits, we need to reduce to a 32-bit type.
389 return [=](const LegalityQuery &Query) {
390 const LLT Ty = Query.Types[TypeIdx];
391 return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
392 Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
393 };
394}
395
396// If we have a truncating store or an extending load with a data size larger
397// than 32-bits and mem location is a power of 2
399 return [=](const LegalityQuery &Query) {
400 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
401 return isWideScalarExtLoadTruncStore(TypeIdx)(Query) &&
402 isPowerOf2_64(MemSize);
403 };
404}
405
406// TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
407// handle some operations by just promoting the register during
408// selection. There are also d16 loads on GFX9+ which preserve the high bits.
409static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
410 bool IsLoad, bool IsAtomic) {
411 switch (AS) {
413 // FIXME: Private element size.
414 return ST.enableFlatScratch() ? 128 : 32;
416 return ST.useDS128() ? 128 : 64;
421 // Treat constant and global as identical. SMRD loads are sometimes usable for
422 // global loads (ideally constant address space should be eliminated)
423 // depending on the context. Legality cannot be context dependent, but
424 // RegBankSelect can split the load as necessary depending on the pointer
425 // register bank/uniformity and if the memory is invariant or not written in a
426 // kernel.
427 return IsLoad ? 512 : 128;
428 default:
429 // FIXME: Flat addresses may contextually need to be split to 32-bit parts
430 // if they may alias scratch depending on the subtarget. This needs to be
431 // moved to custom handling to use addressMayBeAccessedAsPrivate
432 return ST.hasMultiDwordFlatScratchAddressing() || IsAtomic ? 128 : 32;
433 }
434}
435
436static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
437 const LegalityQuery &Query) {
438 const LLT Ty = Query.Types[0];
439
440 // Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
441 const bool IsLoad = Query.Opcode != AMDGPU::G_STORE;
442
443 unsigned RegSize = Ty.getSizeInBits();
444 uint64_t MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
445 uint64_t AlignBits = Query.MMODescrs[0].AlignInBits;
446 unsigned AS = Query.Types[1].getAddressSpace();
447
448 // All of these need to be custom lowered to cast the pointer operand.
450 return false;
451
452 // Do not handle extending vector loads.
453 if (Ty.isVector() && MemSize != RegSize)
454 return false;
455
456 // TODO: We should be able to widen loads if the alignment is high enough, but
457 // we also need to modify the memory access size.
458#if 0
459 // Accept widening loads based on alignment.
460 if (IsLoad && MemSize < Size)
461 MemSize = std::max(MemSize, Align);
462#endif
463
464 // Only 1-byte and 2-byte to 32-bit extloads are valid.
465 if (MemSize != RegSize && RegSize != 32)
466 return false;
467
468 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
469 Query.MMODescrs[0].Ordering !=
471 return false;
472
473 switch (MemSize) {
474 case 8:
475 case 16:
476 case 32:
477 case 64:
478 case 128:
479 break;
480 case 96:
481 if (!ST.hasDwordx3LoadStores())
482 return false;
483 break;
484 case 256:
485 case 512:
486 // These may contextually need to be broken down.
487 break;
488 default:
489 return false;
490 }
491
492 assert(RegSize >= MemSize);
493
494 if (AlignBits < MemSize) {
495 const SITargetLowering *TLI = ST.getTargetLowering();
496 if (!TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS,
497 Align(AlignBits / 8)))
498 return false;
499 }
500
501 return true;
502}
503
504// The newer buffer intrinsic forms take their resource arguments as
505// pointers in address space 8, aka s128 values. However, in order to not break
506// SelectionDAG, the underlying operations have to continue to take v4i32
507// arguments. Therefore, we convert resource pointers - or vectors of them
508// to integer values here.
509static bool hasBufferRsrcWorkaround(const LLT Ty) {
510 if (Ty.isPointer() && Ty.getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
511 return true;
512 if (Ty.isVector()) {
513 const LLT ElemTy = Ty.getElementType();
514 return hasBufferRsrcWorkaround(ElemTy);
515 }
516 return false;
517}
518
519// The current selector can't handle <6 x s16>, <8 x s16>, s96, s128 etc, so
520// workaround this. Eventually it should ignore the type for loads and only care
521// about the size. Return true in cases where we will workaround this for now by
522// bitcasting.
523static bool loadStoreBitcastWorkaround(const LLT Ty) {
525 return false;
526
527 const unsigned Size = Ty.getSizeInBits();
528 if (Ty.isPointerVector())
529 return true;
530 if (Size <= 64)
531 return false;
532 // Address space 8 pointers get their own workaround.
534 return false;
535 if (!Ty.isVector())
536 return true;
537
538 unsigned EltSize = Ty.getScalarSizeInBits();
539 return EltSize != 32 && EltSize != 64;
540}
541
542static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
543 const LLT Ty = Query.Types[0];
544 return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
546}
547
548/// Return true if a load or store of the type should be lowered with a bitcast
549/// to a different type.
550static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
551 const LLT MemTy) {
552 const unsigned MemSizeInBits = MemTy.getSizeInBits();
553 const unsigned Size = Ty.getSizeInBits();
554 if (Size != MemSizeInBits)
555 return Size <= 32 && Ty.isVector();
556
558 return true;
559
560 // Don't try to handle bitcasting vector ext loads for now.
561 return Ty.isVector() && (!MemTy.isVector() || MemTy == Ty) &&
562 (Size <= 32 || isRegisterSize(ST, Size)) &&
563 !isRegisterVectorElementType(Ty.getElementType());
564}
565
566/// Return true if we should legalize a load by widening an odd sized memory
567/// access up to the alignment. Note this case when the memory access itself
568/// changes, not the size of the result register.
569static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy,
570 uint64_t AlignInBits, unsigned AddrSpace,
571 unsigned Opcode) {
572 unsigned SizeInBits = MemoryTy.getSizeInBits();
573 // We don't want to widen cases that are naturally legal.
574 if (isPowerOf2_32(SizeInBits))
575 return false;
576
577 // If we have 96-bit memory operations, we shouldn't touch them. Note we may
578 // end up widening these for a scalar load during RegBankSelect, if we don't
579 // have 96-bit scalar loads.
580 if (SizeInBits == 96 && ST.hasDwordx3LoadStores())
581 return false;
582
583 if (SizeInBits >= maxSizeForAddrSpace(ST, AddrSpace, Opcode, false))
584 return false;
585
586 // A load is known dereferenceable up to the alignment, so it's legal to widen
587 // to it.
588 //
589 // TODO: Could check dereferenceable for less aligned cases.
590 unsigned RoundedSize = NextPowerOf2(SizeInBits);
591 if (AlignInBits < RoundedSize)
592 return false;
593
594 // Do not widen if it would introduce a slow unaligned load.
595 const SITargetLowering *TLI = ST.getTargetLowering();
596 unsigned Fast = 0;
598 RoundedSize, AddrSpace, Align(AlignInBits / 8),
600 Fast;
601}
602
603static bool shouldWidenLoad(const GCNSubtarget &ST, const LegalityQuery &Query,
604 unsigned Opcode) {
605 if (Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)
606 return false;
607
608 return shouldWidenLoad(ST, Query.MMODescrs[0].MemoryTy,
609 Query.MMODescrs[0].AlignInBits,
610 Query.Types[1].getAddressSpace(), Opcode);
611}
612
613/// Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial
614/// type of the operand `idx` and then to transform it to a `p8` via bitcasts
615/// and inttoptr. In addition, handle vectors of p8. Returns the new type.
617 MachineRegisterInfo &MRI, unsigned Idx) {
618 MachineOperand &MO = MI.getOperand(Idx);
619
620 const LLT PointerTy = MRI.getType(MO.getReg());
621
622 // Paranoidly prevent us from doing this multiple times.
624 return PointerTy;
625
626 const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
627 const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);
628 if (!PointerTy.isVector()) {
629 // Happy path: (4 x s32) -> (s32, s32, s32, s32) -> (p8)
630 const unsigned NumParts = PointerTy.getSizeInBits() / 32;
631 const LLT S32 = LLT::scalar(32);
632
633 Register VectorReg = MRI.createGenericVirtualRegister(VectorTy);
634 std::array<Register, 4> VectorElems;
635 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
636 for (unsigned I = 0; I < NumParts; ++I)
637 VectorElems[I] =
638 B.buildExtractVectorElementConstant(S32, VectorReg, I).getReg(0);
639 B.buildMergeValues(MO, VectorElems);
640 MO.setReg(VectorReg);
641 return VectorTy;
642 }
643 Register BitcastReg = MRI.createGenericVirtualRegister(VectorTy);
644 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
645 auto Scalar = B.buildBitcast(ScalarTy, BitcastReg);
646 B.buildIntToPtr(MO, Scalar);
647 MO.setReg(BitcastReg);
648
649 return VectorTy;
650}
651
652/// Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is
653/// the form in which the value must be in order to be passed to the low-level
654/// representations used for MUBUF/MTBUF intrinsics. This is a hack, which is
655/// needed in order to account for the fact that we can't define a register
656/// class for s128 without breaking SelectionDAG.
658 MachineRegisterInfo &MRI = *B.getMRI();
659 const LLT PointerTy = MRI.getType(Pointer);
660 const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
661 const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);
662
663 if (!PointerTy.isVector()) {
664 // Special case: p8 -> (s32, s32, s32, s32) -> (4xs32)
665 SmallVector<Register, 4> PointerParts;
666 const unsigned NumParts = PointerTy.getSizeInBits() / 32;
667 auto Unmerged = B.buildUnmerge(LLT::scalar(32), Pointer);
668 for (unsigned I = 0; I < NumParts; ++I)
669 PointerParts.push_back(Unmerged.getReg(I));
670 return B.buildBuildVector(VectorTy, PointerParts).getReg(0);
671 }
672 Register Scalar = B.buildPtrToInt(ScalarTy, Pointer).getReg(0);
673 return B.buildBitcast(VectorTy, Scalar).getReg(0);
674}
675
677 unsigned Idx) {
678 MachineOperand &MO = MI.getOperand(Idx);
679
680 const LLT PointerTy = B.getMRI()->getType(MO.getReg());
681 // Paranoidly prevent us from doing this multiple times.
683 return;
685}
686
688 const GCNTargetMachine &TM)
689 : ST(ST_) {
690 using namespace TargetOpcode;
691
692 auto GetAddrSpacePtr = [&TM](unsigned AS) {
693 return LLT::pointer(AS, TM.getPointerSizeInBits(AS));
694 };
695
696 const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
697 const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
698 const LLT Constant32Ptr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS_32BIT);
699 const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
700 const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS);
701 const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
702 const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
703 const LLT BufferFatPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_FAT_POINTER);
704 const LLT RsrcPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_RESOURCE);
705 const LLT BufferStridedPtr =
706 GetAddrSpacePtr(AMDGPUAS::BUFFER_STRIDED_POINTER);
707
708 const LLT CodePtr = FlatPtr;
709
710 const std::initializer_list<LLT> AddrSpaces64 = {
711 GlobalPtr, ConstantPtr, FlatPtr
712 };
713
714 const std::initializer_list<LLT> AddrSpaces32 = {
715 LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
716 };
717
718 const std::initializer_list<LLT> AddrSpaces128 = {RsrcPtr};
719
720 const std::initializer_list<LLT> FPTypesBase = {
721 S32, S64
722 };
723
724 const std::initializer_list<LLT> FPTypes16 = {
725 S32, S64, S16
726 };
727
728 const std::initializer_list<LLT> FPTypesPK16 = {
729 S32, S64, S16, V2S16
730 };
731
732 const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32;
733
734 // s1 for VCC branches, s32 for SCC branches.
736
737 // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
738 // elements for v3s16
741 .legalFor(AllS32Vectors)
743 .legalFor(AddrSpaces64)
744 .legalFor(AddrSpaces32)
745 .legalFor(AddrSpaces128)
746 .legalIf(isPointer(0))
747 .clampScalar(0, S16, S256)
749 .clampMaxNumElements(0, S32, 16)
751 .scalarize(0);
752
753 if (ST.hasVOP3PInsts() && ST.hasAddNoCarry() && ST.hasIntClamp()) {
754 // Full set of gfx9 features.
755 if (ST.hasScalarAddSub64()) {
756 getActionDefinitionsBuilder({G_ADD, G_SUB})
757 .legalFor({S64, S32, S16, V2S16})
758 .clampMaxNumElementsStrict(0, S16, 2)
759 .scalarize(0)
760 .minScalar(0, S16)
762 .maxScalar(0, S32);
763 } else {
764 getActionDefinitionsBuilder({G_ADD, G_SUB})
765 .legalFor({S32, S16, V2S16})
766 .clampMaxNumElementsStrict(0, S16, 2)
767 .scalarize(0)
768 .minScalar(0, S16)
770 .maxScalar(0, S32);
771 }
772
773 if (ST.hasScalarSMulU64()) {
775 .legalFor({S64, S32, S16, V2S16})
776 .clampMaxNumElementsStrict(0, S16, 2)
777 .scalarize(0)
778 .minScalar(0, S16)
780 .custom();
781 } else {
783 .legalFor({S32, S16, V2S16})
784 .clampMaxNumElementsStrict(0, S16, 2)
785 .scalarize(0)
786 .minScalar(0, S16)
788 .custom();
789 }
790 assert(ST.hasMad64_32());
791
792 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
793 .legalFor({S32, S16, V2S16}) // Clamp modifier
794 .minScalarOrElt(0, S16)
796 .scalarize(0)
798 .lower();
799 } else if (ST.has16BitInsts()) {
800 getActionDefinitionsBuilder({G_ADD, G_SUB})
801 .legalFor({S32, S16})
802 .minScalar(0, S16)
804 .maxScalar(0, S32)
805 .scalarize(0);
806
808 .legalFor({S32, S16})
809 .scalarize(0)
810 .minScalar(0, S16)
812 .custom();
813 assert(ST.hasMad64_32());
814
815 // Technically the saturating operations require clamp bit support, but this
816 // was introduced at the same time as 16-bit operations.
817 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
818 .legalFor({S32, S16}) // Clamp modifier
819 .minScalar(0, S16)
820 .scalarize(0)
822 .lower();
823
824 // We're just lowering this, but it helps get a better result to try to
825 // coerce to the desired type first.
826 getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
827 .minScalar(0, S16)
828 .scalarize(0)
829 .lower();
830 } else {
831 getActionDefinitionsBuilder({G_ADD, G_SUB})
832 .legalFor({S32})
833 .widenScalarToNextMultipleOf(0, 32)
834 .clampScalar(0, S32, S32)
835 .scalarize(0);
836
837 auto &Mul = getActionDefinitionsBuilder(G_MUL)
838 .legalFor({S32})
839 .scalarize(0)
840 .minScalar(0, S32)
842
843 if (ST.hasMad64_32())
844 Mul.custom();
845 else
846 Mul.maxScalar(0, S32);
847
848 if (ST.hasIntClamp()) {
849 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
850 .legalFor({S32}) // Clamp modifier.
851 .scalarize(0)
853 .lower();
854 } else {
855 // Clamp bit support was added in VI, along with 16-bit operations.
856 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
857 .minScalar(0, S32)
858 .scalarize(0)
859 .lower();
860 }
861
862 // FIXME: DAG expansion gets better results. The widening uses the smaller
863 // range values and goes for the min/max lowering directly.
864 getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
865 .minScalar(0, S32)
866 .scalarize(0)
867 .lower();
868 }
869
871 {G_SDIV, G_UDIV, G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
872 .customFor({S32, S64})
873 .clampScalar(0, S32, S64)
875 .scalarize(0);
876
877 auto &Mulh = getActionDefinitionsBuilder({G_UMULH, G_SMULH})
878 .legalFor({S32})
879 .maxScalar(0, S32);
880
881 if (ST.hasVOP3PInsts()) {
882 Mulh
883 .clampMaxNumElements(0, S8, 2)
884 .lowerFor({V2S8});
885 }
886
887 Mulh
888 .scalarize(0)
889 .lower();
890
891 // Report legal for any types we can handle anywhere. For the cases only legal
892 // on the SALU, RegBankSelect will be able to re-legalize.
893 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
894 .legalFor({S32, S1, S64, V2S32, S16, V2S16, V4S16})
895 .clampScalar(0, S32, S64)
901 .scalarize(0);
902
904 {G_UADDO, G_USUBO, G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
905 .legalFor({{S32, S1}, {S32, S32}})
906 .clampScalar(0, S32, S32)
907 .scalarize(0);
908
910 // Don't worry about the size constraint.
912 .lower();
913
915 .legalFor({S1, S32, S64, S16, GlobalPtr,
916 LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
917 .legalIf(isPointer(0))
918 .clampScalar(0, S32, S64)
920
921 getActionDefinitionsBuilder(G_FCONSTANT)
922 .legalFor({S32, S64, S16})
923 .clampScalar(0, S16, S64);
924
925 getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
926 .legalIf(isRegisterClassType(ST, 0))
927 // s1 and s16 are special cases because they have legal operations on
928 // them, but don't really occupy registers in the normal way.
929 .legalFor({S1, S16})
930 .clampNumElements(0, V16S32, V32S32)
934 .clampMaxNumElements(0, S32, 16);
935
936 getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr});
937
938 // If the amount is divergent, we have to do a wave reduction to get the
939 // maximum value, so this is expanded during RegBankSelect.
940 getActionDefinitionsBuilder(G_DYN_STACKALLOC)
941 .legalFor({{PrivatePtr, S32}});
942
943 getActionDefinitionsBuilder(G_STACKSAVE)
944 .customFor({PrivatePtr});
945 getActionDefinitionsBuilder(G_STACKRESTORE)
946 .legalFor({PrivatePtr});
947
948 getActionDefinitionsBuilder({G_GET_FPENV, G_SET_FPENV}).customFor({S64});
949
950 getActionDefinitionsBuilder(G_GLOBAL_VALUE)
951 .customIf(typeIsNot(0, PrivatePtr));
952
953 getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({CodePtr});
954
955 auto &FPOpActions = getActionDefinitionsBuilder(
956 { G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE,
957 G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
958 .legalFor({S32, S64});
959 auto &TrigActions = getActionDefinitionsBuilder({G_FSIN, G_FCOS})
960 .customFor({S32, S64});
961 auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV)
962 .customFor({S32, S64});
963
964 if (ST.has16BitInsts()) {
965 if (ST.hasVOP3PInsts())
966 FPOpActions.legalFor({S16, V2S16});
967 else
968 FPOpActions.legalFor({S16});
969
970 TrigActions.customFor({S16});
971 FDIVActions.customFor({S16});
972 }
973
974 if (ST.hasPackedFP32Ops()) {
975 FPOpActions.legalFor({V2S32});
976 FPOpActions.clampMaxNumElementsStrict(0, S32, 2);
977 }
978
979 auto &MinNumMaxNum = getActionDefinitionsBuilder(
980 {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM, G_FMINNUM_IEEE,
981 G_FMAXNUM_IEEE});
982
983 if (ST.hasVOP3PInsts()) {
984 MinNumMaxNum.customFor(FPTypesPK16)
985 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
986 .clampMaxNumElements(0, S16, 2)
987 .clampScalar(0, S16, S64)
988 .scalarize(0);
989 } else if (ST.has16BitInsts()) {
990 MinNumMaxNum.customFor(FPTypes16)
991 .clampScalar(0, S16, S64)
992 .scalarize(0);
993 } else {
994 MinNumMaxNum.customFor(FPTypesBase)
995 .clampScalar(0, S32, S64)
996 .scalarize(0);
997 }
998
999 if (ST.hasVOP3PInsts())
1000 FPOpActions.clampMaxNumElementsStrict(0, S16, 2);
1001
1002 FPOpActions
1003 .scalarize(0)
1004 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
1005
1006 TrigActions
1007 .scalarize(0)
1008 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
1009
1010 FDIVActions
1011 .scalarize(0)
1012 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
1013
1014 getActionDefinitionsBuilder({G_FNEG, G_FABS})
1015 .legalFor(FPTypesPK16)
1017 .scalarize(0)
1018 .clampScalar(0, S16, S64);
1019
1020 if (ST.has16BitInsts()) {
1022 .legalFor({S16})
1023 .customFor({S32, S64})
1024 .scalarize(0)
1025 .unsupported();
1027 .legalFor({S32, S64, S16})
1028 .scalarize(0)
1029 .clampScalar(0, S16, S64);
1030
1031 getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
1032 .legalFor({{S32, S32}, {S64, S32}, {S16, S16}})
1033 .scalarize(0)
1034 .maxScalarIf(typeIs(0, S16), 1, S16)
1035 .clampScalar(1, S32, S32)
1036 .lower();
1037
1039 .customFor({{S32, S32}, {S64, S32}, {S16, S16}, {S16, S32}})
1040 .scalarize(0)
1041 .lower();
1042 } else {
1044 .customFor({S32, S64, S16})
1045 .scalarize(0)
1046 .unsupported();
1047
1048
1049 if (ST.hasFractBug()) {
1051 .customFor({S64})
1052 .legalFor({S32, S64})
1053 .scalarize(0)
1054 .clampScalar(0, S32, S64);
1055 } else {
1057 .legalFor({S32, S64})
1058 .scalarize(0)
1059 .clampScalar(0, S32, S64);
1060 }
1061
1062 getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
1063 .legalFor({{S32, S32}, {S64, S32}})
1064 .scalarize(0)
1065 .clampScalar(0, S32, S64)
1066 .clampScalar(1, S32, S32)
1067 .lower();
1068
1070 .customFor({{S32, S32}, {S64, S32}})
1071 .scalarize(0)
1072 .minScalar(0, S32)
1073 .clampScalar(1, S32, S32)
1074 .lower();
1075 }
1076
1077 auto &FPTruncActions = getActionDefinitionsBuilder(G_FPTRUNC);
1078 if (ST.hasCvtPkF16F32Inst()) {
1079 FPTruncActions.legalFor({{S32, S64}, {S16, S32}, {V2S16, V2S32}})
1080 .clampMaxNumElements(0, S16, 2);
1081 } else {
1082 FPTruncActions.legalFor({{S32, S64}, {S16, S32}});
1083 }
1084 FPTruncActions.scalarize(0).lower();
1085
1087 .legalFor({{S64, S32}, {S32, S16}})
1088 .narrowScalarFor({{S64, S16}}, changeTo(0, S32))
1089 .scalarize(0);
1090
1091 auto &FSubActions = getActionDefinitionsBuilder({G_FSUB, G_STRICT_FSUB});
1092 if (ST.has16BitInsts()) {
1093 FSubActions
1094 // Use actual fsub instruction
1095 .legalFor({S32, S16})
1096 // Must use fadd + fneg
1097 .lowerFor({S64, V2S16});
1098 } else {
1099 FSubActions
1100 // Use actual fsub instruction
1101 .legalFor({S32})
1102 // Must use fadd + fneg
1103 .lowerFor({S64, S16, V2S16});
1104 }
1105
1106 FSubActions
1107 .scalarize(0)
1108 .clampScalar(0, S32, S64);
1109
1110 // Whether this is legal depends on the floating point mode for the function.
1111 auto &FMad = getActionDefinitionsBuilder(G_FMAD);
1112 if (ST.hasMadF16() && ST.hasMadMacF32Insts())
1113 FMad.customFor({S32, S16});
1114 else if (ST.hasMadMacF32Insts())
1115 FMad.customFor({S32});
1116 else if (ST.hasMadF16())
1117 FMad.customFor({S16});
1118 FMad.scalarize(0)
1119 .lower();
1120
1121 auto &FRem = getActionDefinitionsBuilder(G_FREM);
1122 if (ST.has16BitInsts()) {
1123 FRem.customFor({S16, S32, S64});
1124 } else {
1125 FRem.minScalar(0, S32)
1126 .customFor({S32, S64});
1127 }
1128 FRem.scalarize(0);
1129
1130 // TODO: Do we need to clamp maximum bitwidth?
1132 .legalIf(isScalar(0))
1133 .legalFor({{V2S16, V2S32}})
1134 .clampMaxNumElements(0, S16, 2)
1135 // Avoid scalarizing in cases that should be truly illegal. In unresolvable
1136 // situations (like an invalid implicit use), we don't want to infinite loop
1137 // in the legalizer.
1139 .alwaysLegal();
1140
1141 getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
1142 .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
1143 {S32, S1}, {S64, S1}, {S16, S1}})
1144 .scalarize(0)
1145 .clampScalar(0, S32, S64)
1146 .widenScalarToNextPow2(1, 32);
1147
1148 // TODO: Split s1->s64 during regbankselect for VALU.
1149 auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
1150 .legalFor({{S32, S32}, {S64, S32}, {S16, S32}})
1151 .lowerIf(typeIs(1, S1))
1152 .customFor({{S32, S64}, {S64, S64}});
1153 if (ST.has16BitInsts())
1154 IToFP.legalFor({{S16, S16}});
1155 IToFP.clampScalar(1, S32, S64)
1156 .minScalar(0, S32)
1157 .scalarize(0)
1159
1160 auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
1161 .legalFor({{S32, S32}, {S32, S64}, {S32, S16}})
1162 .customFor({{S64, S32}, {S64, S64}})
1163 .narrowScalarFor({{S64, S16}}, changeTo(0, S32));
1164 if (ST.has16BitInsts())
1165 FPToI.legalFor({{S16, S16}});
1166 else
1167 FPToI.minScalar(1, S32);
1168
1169 FPToI.minScalar(0, S32)
1170 .widenScalarToNextPow2(0, 32)
1171 .scalarize(0)
1172 .lower();
1173
1174 getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
1175 .clampScalar(0, S16, S64)
1176 .scalarize(0)
1177 .lower();
1178
1179 getActionDefinitionsBuilder(G_INTRINSIC_FPTRUNC_ROUND)
1180 .legalFor({S16, S32})
1181 .scalarize(0)
1182 .lower();
1183
1184 // Lower G_FNEARBYINT and G_FRINT into G_INTRINSIC_ROUNDEVEN
1185 getActionDefinitionsBuilder({G_INTRINSIC_ROUND, G_FRINT, G_FNEARBYINT})
1186 .scalarize(0)
1187 .lower();
1188
1189 getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
1190 .clampScalar(0, S16, S64)
1191 .scalarize(0)
1192 .lower();
1193
1194 if (ST.has16BitInsts()) {
1196 {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
1197 .legalFor({S16, S32, S64})
1198 .clampScalar(0, S16, S64)
1199 .scalarize(0);
1200 } else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
1202 {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
1203 .legalFor({S32, S64})
1204 .clampScalar(0, S32, S64)
1205 .scalarize(0);
1206 } else {
1208 {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
1209 .legalFor({S32})
1210 .customFor({S64})
1211 .clampScalar(0, S32, S64)
1212 .scalarize(0);
1213 }
1214
1216 .unsupportedFor({BufferFatPtr, BufferStridedPtr, RsrcPtr})
1217 .legalIf(all(isPointer(0), sameSize(0, 1)))
1218 .scalarize(0)
1219 .scalarSameSizeAs(1, 0);
1220
1222 .legalIf(all(sameSize(0, 1), typeInSet(1, {S64, S32})))
1223 .scalarSameSizeAs(1, 0)
1224 .scalarize(0);
1225
1226 auto &CmpBuilder =
1228 // The compare output type differs based on the register bank of the output,
1229 // so make both s1 and s32 legal.
1230 //
1231 // Scalar compares producing output in scc will be promoted to s32, as that
1232 // is the allocatable register type that will be needed for the copy from
1233 // scc. This will be promoted during RegBankSelect, and we assume something
1234 // before that won't try to use s32 result types.
1235 //
1236 // Vector compares producing an output in vcc/SGPR will use s1 in VCC reg
1237 // bank.
1239 {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
1240 .legalForCartesianProduct(
1241 {S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
1242 if (ST.has16BitInsts()) {
1243 CmpBuilder.legalFor({{S1, S16}});
1244 }
1245
1246 CmpBuilder
1248 .clampScalar(1, S32, S64)
1249 .scalarize(0)
1250 .legalIf(all(typeInSet(0, {S1, S32}), isPointer(1)));
1251
1252 auto &FCmpBuilder =
1254 {S1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase);
1255
1256 if (ST.hasSALUFloatInsts())
1257 FCmpBuilder.legalForCartesianProduct({S32}, {S16, S32});
1258
1259 FCmpBuilder
1261 .clampScalar(1, S32, S64)
1262 .scalarize(0);
1263
1264 // FIXME: fpow has a selection pattern that should move to custom lowering.
1265 auto &ExpOps = getActionDefinitionsBuilder(G_FPOW);
1266 if (ST.has16BitInsts())
1267 ExpOps.customFor({{S32}, {S16}});
1268 else
1269 ExpOps.customFor({S32});
1270 ExpOps.clampScalar(0, MinScalarFPTy, S32)
1271 .scalarize(0);
1272
1274 .clampScalar(0, MinScalarFPTy, S32)
1275 .lower();
1276
1277 auto &Log2Ops = getActionDefinitionsBuilder({G_FLOG2, G_FEXP2});
1278 Log2Ops.customFor({S32});
1279 if (ST.has16BitInsts())
1280 Log2Ops.legalFor({S16});
1281 else
1282 Log2Ops.customFor({S16});
1283 Log2Ops.scalarize(0)
1284 .lower();
1285
1286 auto &LogOps =
1287 getActionDefinitionsBuilder({G_FLOG, G_FLOG10, G_FEXP, G_FEXP10});
1288 LogOps.customFor({S32, S16});
1289 LogOps.clampScalar(0, MinScalarFPTy, S32)
1290 .scalarize(0);
1291
1292 // The 64-bit versions produce 32-bit results, but only on the SALU.
1294 .legalFor({{S32, S32}, {S32, S64}})
1295 .clampScalar(0, S32, S32)
1296 .widenScalarToNextPow2(1, 32)
1297 .clampScalar(1, S32, S64)
1298 .scalarize(0)
1299 .widenScalarToNextPow2(0, 32);
1300
1301 // If no 16 bit instr is available, lower into different instructions.
1302 if (ST.has16BitInsts())
1303 getActionDefinitionsBuilder(G_IS_FPCLASS)
1304 .legalForCartesianProduct({S1}, FPTypes16)
1305 .widenScalarToNextPow2(1)
1306 .scalarize(0)
1307 .lower();
1308 else
1309 getActionDefinitionsBuilder(G_IS_FPCLASS)
1310 .legalForCartesianProduct({S1}, FPTypesBase)
1311 .lowerFor({S1, S16})
1312 .widenScalarToNextPow2(1)
1313 .scalarize(0)
1314 .lower();
1315
1316 // The hardware instructions return a different result on 0 than the generic
1317 // instructions expect. The hardware produces -1, but these produce the
1318 // bitwidth.
1319 getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
1320 .scalarize(0)
1321 .clampScalar(0, S32, S32)
1322 .clampScalar(1, S32, S64)
1323 .widenScalarToNextPow2(0, 32)
1324 .widenScalarToNextPow2(1, 32)
1325 .custom();
1326
1327 // The 64-bit versions produce 32-bit results, but only on the SALU.
1328 getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF)
1329 .legalFor({{S32, S32}, {S32, S64}})
1330 .customIf(scalarNarrowerThan(1, 32))
1331 .clampScalar(0, S32, S32)
1332 .clampScalar(1, S32, S64)
1333 .scalarize(0)
1334 .widenScalarToNextPow2(0, 32)
1335 .widenScalarToNextPow2(1, 32);
1336
1337 getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF)
1338 .legalFor({{S32, S32}, {S32, S64}})
1339 .clampScalar(0, S32, S32)
1340 .clampScalar(1, S32, S64)
1341 .scalarize(0)
1342 .widenScalarToNextPow2(0, 32)
1343 .widenScalarToNextPow2(1, 32);
1344
1345 // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
1346 // RegBankSelect.
1347 getActionDefinitionsBuilder(G_BITREVERSE)
1348 .legalFor({S32, S64})
1349 .clampScalar(0, S32, S64)
1350 .scalarize(0)
1352
1353 if (ST.has16BitInsts()) {
1355 .legalFor({S16, S32, V2S16})
1356 .clampMaxNumElementsStrict(0, S16, 2)
1357 // FIXME: Fixing non-power-of-2 before clamp is workaround for
1358 // narrowScalar limitation.
1360 .clampScalar(0, S16, S32)
1361 .scalarize(0);
1362
1363 if (ST.hasVOP3PInsts()) {
1365 .legalFor({S32, S16, V2S16})
1366 .clampMaxNumElements(0, S16, 2)
1367 .minScalar(0, S16)
1369 .scalarize(0)
1370 .lower();
1371 if (ST.hasIntMinMax64()) {
1372 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1373 .legalFor({S32, S16, S64, V2S16})
1374 .clampMaxNumElements(0, S16, 2)
1375 .minScalar(0, S16)
1377 .scalarize(0)
1378 .lower();
1379 } else {
1380 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1381 .legalFor({S32, S16, V2S16})
1382 .clampMaxNumElements(0, S16, 2)
1383 .minScalar(0, S16)
1385 .scalarize(0)
1386 .lower();
1387 }
1388 } else {
1389 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1390 .legalFor({S32, S16})
1391 .widenScalarToNextPow2(0)
1392 .minScalar(0, S16)
1393 .scalarize(0)
1394 .lower();
1395 }
1396 } else {
1397 // TODO: Should have same legality without v_perm_b32
1399 .legalFor({S32})
1400 .lowerIf(scalarNarrowerThan(0, 32))
1401 // FIXME: Fixing non-power-of-2 before clamp is workaround for
1402 // narrowScalar limitation.
1404 .maxScalar(0, S32)
1405 .scalarize(0)
1406 .lower();
1407
1408 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1409 .legalFor({S32})
1410 .minScalar(0, S32)
1412 .scalarize(0)
1413 .lower();
1414 }
1415
1416 getActionDefinitionsBuilder(G_INTTOPTR)
1417 // List the common cases
1418 .legalForCartesianProduct(AddrSpaces64, {S64})
1419 .legalForCartesianProduct(AddrSpaces32, {S32})
1420 .scalarize(0)
1421 // Accept any address space as long as the size matches
1422 .legalIf(sameSize(0, 1))
1424 [](const LegalityQuery &Query) {
1425 return std::pair(
1426 1, LLT::scalar(Query.Types[0].getSizeInBits()));
1427 })
1428 .narrowScalarIf(largerThan(1, 0), [](const LegalityQuery &Query) {
1429 return std::pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
1430 });
1431
1432 getActionDefinitionsBuilder(G_PTRTOINT)
1433 // List the common cases
1434 .legalForCartesianProduct(AddrSpaces64, {S64})
1435 .legalForCartesianProduct(AddrSpaces32, {S32})
1436 .scalarize(0)
1437 // Accept any address space as long as the size matches
1438 .legalIf(sameSize(0, 1))
1440 [](const LegalityQuery &Query) {
1441 return std::pair(
1442 0, LLT::scalar(Query.Types[1].getSizeInBits()));
1443 })
1444 .narrowScalarIf(largerThan(0, 1), [](const LegalityQuery &Query) {
1445 return std::pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
1446 });
1447
1448 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
1449 .scalarize(0)
1450 .custom();
1451
1452 const auto needToSplitMemOp = [=](const LegalityQuery &Query,
1453 bool IsLoad) -> bool {
1454 const LLT DstTy = Query.Types[0];
1455
1456 // Split vector extloads.
1457 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1458
1459 if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
1460 return true;
1461
1462 const LLT PtrTy = Query.Types[1];
1463 unsigned AS = PtrTy.getAddressSpace();
1464 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
1465 Query.MMODescrs[0].Ordering !=
1467 return true;
1468
1469 // Catch weird sized loads that don't evenly divide into the access sizes
1470 // TODO: May be able to widen depending on alignment etc.
1471 unsigned NumRegs = (MemSize + 31) / 32;
1472 if (NumRegs == 3) {
1473 if (!ST.hasDwordx3LoadStores())
1474 return true;
1475 } else {
1476 // If the alignment allows, these should have been widened.
1477 if (!isPowerOf2_32(NumRegs))
1478 return true;
1479 }
1480
1481 return false;
1482 };
1483
1484 unsigned GlobalAlign32 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
1485 unsigned GlobalAlign16 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
1486 unsigned GlobalAlign8 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;
1487
1488 // TODO: Refine based on subtargets which support unaligned access or 128-bit
1489 // LDS
1490 // TODO: Unsupported flat for SI.
1491
1492 for (unsigned Op : {G_LOAD, G_STORE}) {
1493 const bool IsStore = Op == G_STORE;
1494
1495 auto &Actions = getActionDefinitionsBuilder(Op);
1496 // Explicitly list some common cases.
1497 // TODO: Does this help compile time at all?
1498 Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, S32, GlobalAlign32},
1499 {V2S32, GlobalPtr, V2S32, GlobalAlign32},
1500 {V4S32, GlobalPtr, V4S32, GlobalAlign32},
1501 {S64, GlobalPtr, S64, GlobalAlign32},
1502 {V2S64, GlobalPtr, V2S64, GlobalAlign32},
1503 {V2S16, GlobalPtr, V2S16, GlobalAlign32},
1504 {S32, GlobalPtr, S8, GlobalAlign8},
1505 {S32, GlobalPtr, S16, GlobalAlign16},
1506
1507 {S32, LocalPtr, S32, 32},
1508 {S64, LocalPtr, S64, 32},
1509 {V2S32, LocalPtr, V2S32, 32},
1510 {S32, LocalPtr, S8, 8},
1511 {S32, LocalPtr, S16, 16},
1512 {V2S16, LocalPtr, S32, 32},
1513
1514 {S32, PrivatePtr, S32, 32},
1515 {S32, PrivatePtr, S8, 8},
1516 {S32, PrivatePtr, S16, 16},
1517 {V2S16, PrivatePtr, S32, 32},
1518
1519 {S32, ConstantPtr, S32, GlobalAlign32},
1520 {V2S32, ConstantPtr, V2S32, GlobalAlign32},
1521 {V4S32, ConstantPtr, V4S32, GlobalAlign32},
1522 {S64, ConstantPtr, S64, GlobalAlign32},
1523 {V2S32, ConstantPtr, V2S32, GlobalAlign32}});
1524 Actions.legalIf(
1525 [=](const LegalityQuery &Query) -> bool {
1526 return isLoadStoreLegal(ST, Query);
1527 });
1528
1529 // The custom pointers (fat pointers, buffer resources) don't work with load
1530 // and store at this level. Fat pointers should have been lowered to
1531 // intrinsics before the translation to MIR.
1532 Actions.unsupportedIf(
1533 typeInSet(1, {BufferFatPtr, BufferStridedPtr, RsrcPtr}));
1534
1535 // Address space 8 pointers are handled by a 4xs32 load, bitcast, and
1536 // ptrtoint. This is needed to account for the fact that we can't have i128
1537 // as a register class for SelectionDAG reasons.
1538 Actions.customIf([=](const LegalityQuery &Query) -> bool {
1539 return hasBufferRsrcWorkaround(Query.Types[0]);
1540 });
1541
1542 // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
1543 // 64-bits.
1544 //
1545 // TODO: Should generalize bitcast action into coerce, which will also cover
1546 // inserting addrspacecasts.
1547 Actions.customIf(typeIs(1, Constant32Ptr));
1548
1549 // Turn any illegal element vectors into something easier to deal
1550 // with. These will ultimately produce 32-bit scalar shifts to extract the
1551 // parts anyway.
1552 //
1553 // For odd 16-bit element vectors, prefer to split those into pieces with
1554 // 16-bit vector parts.
1555 Actions.bitcastIf(
1556 [=](const LegalityQuery &Query) -> bool {
1557 return shouldBitcastLoadStoreType(ST, Query.Types[0],
1558 Query.MMODescrs[0].MemoryTy);
1559 }, bitcastToRegisterType(0));
1560
1561 if (!IsStore) {
1562 // Widen suitably aligned loads by loading extra bytes. The standard
1563 // legalization actions can't properly express widening memory operands.
1564 Actions.customIf([=](const LegalityQuery &Query) -> bool {
1565 return shouldWidenLoad(ST, Query, G_LOAD);
1566 });
1567 }
1568
1569 // FIXME: load/store narrowing should be moved to lower action
1570 Actions
1571 .narrowScalarIf(
1572 [=](const LegalityQuery &Query) -> bool {
1573 return !Query.Types[0].isVector() &&
1574 needToSplitMemOp(Query, Op == G_LOAD);
1575 },
1576 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1577 const LLT DstTy = Query.Types[0];
1578 const LLT PtrTy = Query.Types[1];
1579
1580 const unsigned DstSize = DstTy.getSizeInBits();
1581 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1582
1583 // Split extloads.
1584 if (DstSize > MemSize)
1585 return std::pair(0, LLT::scalar(MemSize));
1586
1587 unsigned MaxSize = maxSizeForAddrSpace(
1588 ST, PtrTy.getAddressSpace(), Op == G_LOAD,
1589 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
1590 if (MemSize > MaxSize)
1591 return std::pair(0, LLT::scalar(MaxSize));
1592
1593 uint64_t Align = Query.MMODescrs[0].AlignInBits;
1594 return std::pair(0, LLT::scalar(Align));
1595 })
1596 .fewerElementsIf(
1597 [=](const LegalityQuery &Query) -> bool {
1598 return Query.Types[0].isVector() &&
1599 needToSplitMemOp(Query, Op == G_LOAD);
1600 },
1601 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1602 const LLT DstTy = Query.Types[0];
1603 const LLT PtrTy = Query.Types[1];
1604
1605 LLT EltTy = DstTy.getElementType();
1606 unsigned MaxSize = maxSizeForAddrSpace(
1607 ST, PtrTy.getAddressSpace(), Op == G_LOAD,
1608 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
1609
1610 // FIXME: Handle widened to power of 2 results better. This ends
1611 // up scalarizing.
1612 // FIXME: 3 element stores scalarized on SI
1613
1614 // Split if it's too large for the address space.
1615 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1616 if (MemSize > MaxSize) {
1617 unsigned NumElts = DstTy.getNumElements();
1618 unsigned EltSize = EltTy.getSizeInBits();
1619
1620 if (MaxSize % EltSize == 0) {
1621 return std::pair(
1623 ElementCount::getFixed(MaxSize / EltSize), EltTy));
1624 }
1625
1626 unsigned NumPieces = MemSize / MaxSize;
1627
1628 // FIXME: Refine when odd breakdowns handled
1629 // The scalars will need to be re-legalized.
1630 if (NumPieces == 1 || NumPieces >= NumElts ||
1631 NumElts % NumPieces != 0)
1632 return std::pair(0, EltTy);
1633
1634 return std::pair(0,
1635 LLT::fixed_vector(NumElts / NumPieces, EltTy));
1636 }
1637
1638 // FIXME: We could probably handle weird extending loads better.
1639 if (DstTy.getSizeInBits() > MemSize)
1640 return std::pair(0, EltTy);
1641
1642 unsigned EltSize = EltTy.getSizeInBits();
1643 unsigned DstSize = DstTy.getSizeInBits();
1644 if (!isPowerOf2_32(DstSize)) {
1645 // We're probably decomposing an odd sized store. Try to split
1646 // to the widest type. TODO: Account for alignment. As-is it
1647 // should be OK, since the new parts will be further legalized.
1648 unsigned FloorSize = llvm::bit_floor(DstSize);
1649 return std::pair(
1651 ElementCount::getFixed(FloorSize / EltSize), EltTy));
1652 }
1653
1654 // May need relegalization for the scalars.
1655 return std::pair(0, EltTy);
1656 })
1657 .minScalar(0, S32)
1658 .narrowScalarIf(isTruncStoreToSizePowerOf2(0),
1660 .widenScalarToNextPow2(0)
1661 .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
1662 .lower();
1663 }
1664
1665 // FIXME: Unaligned accesses not lowered.
1666 auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
1667 .legalForTypesWithMemDesc({{S32, GlobalPtr, S8, 8},
1668 {S32, GlobalPtr, S16, 2 * 8},
1669 {S32, LocalPtr, S8, 8},
1670 {S32, LocalPtr, S16, 16},
1671 {S32, PrivatePtr, S8, 8},
1672 {S32, PrivatePtr, S16, 16},
1673 {S32, ConstantPtr, S8, 8},
1674 {S32, ConstantPtr, S16, 2 * 8}})
1675 .legalIf(
1676 [=](const LegalityQuery &Query) -> bool {
1677 return isLoadStoreLegal(ST, Query);
1678 });
1679
1680 if (ST.hasFlatAddressSpace()) {
1681 ExtLoads.legalForTypesWithMemDesc(
1682 {{S32, FlatPtr, S8, 8}, {S32, FlatPtr, S16, 16}});
1683 }
1684
1685 // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
1686 // 64-bits.
1687 //
1688 // TODO: Should generalize bitcast action into coerce, which will also cover
1689 // inserting addrspacecasts.
1690 ExtLoads.customIf(typeIs(1, Constant32Ptr));
1691
1692 ExtLoads.clampScalar(0, S32, S32)
1694 .lower();
1695
1696 auto &Atomics = getActionDefinitionsBuilder(
1697 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
1698 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
1699 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
1700 G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
1701 .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
1702 {S64, GlobalPtr}, {S64, LocalPtr},
1703 {S32, RegionPtr}, {S64, RegionPtr}});
1704 if (ST.hasFlatAddressSpace()) {
1705 Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
1706 }
1707
1708 // TODO: v2bf16 operations, and fat buffer pointer support.
1709 auto &Atomic = getActionDefinitionsBuilder(G_ATOMICRMW_FADD);
1710 if (ST.hasLDSFPAtomicAddF32()) {
1711 Atomic.legalFor({{S32, LocalPtr}, {S32, RegionPtr}});
1712 if (ST.hasLdsAtomicAddF64())
1713 Atomic.legalFor({{S64, LocalPtr}});
1714 if (ST.hasAtomicDsPkAdd16Insts())
1715 Atomic.legalFor({{V2F16, LocalPtr}, {V2BF16, LocalPtr}});
1716 }
1717 if (ST.hasAtomicFaddInsts())
1718 Atomic.legalFor({{S32, GlobalPtr}});
1719 if (ST.hasFlatAtomicFaddF32Inst())
1720 Atomic.legalFor({{S32, FlatPtr}});
1721
1722 if (ST.hasGFX90AInsts() || ST.hasGFX1250Insts()) {
1723 // These are legal with some caveats, and should have undergone expansion in
1724 // the IR in most situations
1725 // TODO: Move atomic expansion into legalizer
1726 Atomic.legalFor({
1727 {S32, GlobalPtr},
1728 {S64, GlobalPtr},
1729 {S64, FlatPtr}
1730 });
1731 }
1732
1733 if (ST.hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
1734 ST.hasAtomicBufferGlobalPkAddF16Insts())
1735 Atomic.legalFor({{V2F16, GlobalPtr}, {V2F16, BufferFatPtr}});
1736 if (ST.hasAtomicGlobalPkAddBF16Inst())
1737 Atomic.legalFor({{V2BF16, GlobalPtr}});
1738 if (ST.hasAtomicFlatPkAdd16Insts())
1739 Atomic.legalFor({{V2F16, FlatPtr}, {V2BF16, FlatPtr}});
1740
1741
1742 // Most of the legalization work here is done by AtomicExpand. We could
1743 // probably use a simpler legality rule that just assumes anything is OK.
1744 auto &AtomicFMinFMax =
1745 getActionDefinitionsBuilder({G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
1746 .legalFor({{F32, LocalPtr}, {F64, LocalPtr}});
1747
1748 if (ST.hasAtomicFMinFMaxF32GlobalInsts())
1749 AtomicFMinFMax.legalFor({{F32, GlobalPtr},{F32, BufferFatPtr}});
1750 if (ST.hasAtomicFMinFMaxF64GlobalInsts())
1751 AtomicFMinFMax.legalFor({{F64, GlobalPtr}, {F64, BufferFatPtr}});
1752 if (ST.hasAtomicFMinFMaxF32FlatInsts())
1753 AtomicFMinFMax.legalFor({F32, FlatPtr});
1754 if (ST.hasAtomicFMinFMaxF64FlatInsts())
1755 AtomicFMinFMax.legalFor({F64, FlatPtr});
1756
1757 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
1758 // demarshalling
1759 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
1760 .customFor({{S32, GlobalPtr}, {S64, GlobalPtr},
1761 {S32, FlatPtr}, {S64, FlatPtr}})
1762 .legalFor({{S32, LocalPtr}, {S64, LocalPtr},
1763 {S32, RegionPtr}, {S64, RegionPtr}});
1764 // TODO: Pointer types, any 32-bit or 64-bit vector
1765
1766 // Condition should be s32 for scalar, s1 for vector.
1769 LocalPtr, FlatPtr, PrivatePtr,
1770 LLT::fixed_vector(2, LocalPtr),
1771 LLT::fixed_vector(2, PrivatePtr)},
1772 {S1, S32})
1773 .clampScalar(0, S16, S64)
1774 .scalarize(1)
1777 .clampMaxNumElements(0, S32, 2)
1778 .clampMaxNumElements(0, LocalPtr, 2)
1779 .clampMaxNumElements(0, PrivatePtr, 2)
1780 .scalarize(0)
1782 .legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));
1783
1784 // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
1785 // be more flexible with the shift amount type.
1786 auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
1787 .legalFor({{S32, S32}, {S64, S32}});
1788 if (ST.has16BitInsts()) {
1789 if (ST.hasVOP3PInsts()) {
1790 Shifts.legalFor({{S16, S16}, {V2S16, V2S16}})
1791 .clampMaxNumElements(0, S16, 2);
1792 } else
1793 Shifts.legalFor({{S16, S16}});
1794
1795 // TODO: Support 16-bit shift amounts for all types
1796 Shifts.widenScalarIf(
1797 [=](const LegalityQuery &Query) {
1798 // Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
1799 // 32-bit amount.
1800 const LLT ValTy = Query.Types[0];
1801 const LLT AmountTy = Query.Types[1];
1802 return ValTy.isScalar() && ValTy.getSizeInBits() <= 16 &&
1803 AmountTy.getSizeInBits() < 16;
1804 }, changeTo(1, S16));
1805 Shifts.maxScalarIf(typeIs(0, S16), 1, S16);
1806 Shifts.clampScalar(1, S32, S32);
1807 Shifts.widenScalarToNextPow2(0, 16);
1808 Shifts.clampScalar(0, S16, S64);
1809
1810 getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
1811 .minScalar(0, S16)
1812 .scalarize(0)
1813 .lower();
1814 } else {
1815 // Make sure we legalize the shift amount type first, as the general
1816 // expansion for the shifted type will produce much worse code if it hasn't
1817 // been truncated already.
1818 Shifts.clampScalar(1, S32, S32);
1819 Shifts.widenScalarToNextPow2(0, 32);
1820 Shifts.clampScalar(0, S32, S64);
1821
1822 getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
1823 .minScalar(0, S32)
1824 .scalarize(0)
1825 .lower();
1826 }
1827 Shifts.scalarize(0);
1828
1829 for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
1830 unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
1831 unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
1832 unsigned IdxTypeIdx = 2;
1833
1835 .customIf([=](const LegalityQuery &Query) {
1836 const LLT EltTy = Query.Types[EltTypeIdx];
1837 const LLT VecTy = Query.Types[VecTypeIdx];
1838 const LLT IdxTy = Query.Types[IdxTypeIdx];
1839 const unsigned EltSize = EltTy.getSizeInBits();
1840 const bool isLegalVecType =
1842 // Address space 8 pointers are 128-bit wide values, but the logic
1843 // below will try to bitcast them to 2N x s64, which will fail.
1844 // Therefore, as an intermediate step, wrap extracts/insertions from a
1845 // ptrtoint-ing the vector and scalar arguments (or inttoptring the
1846 // extraction result) in order to produce a vector operation that can
1847 // be handled by the logic below.
1848 if (EltTy.isPointer() && EltSize > 64)
1849 return true;
1850 return (EltSize == 32 || EltSize == 64) &&
1851 VecTy.getSizeInBits() % 32 == 0 &&
1852 VecTy.getSizeInBits() <= MaxRegisterSize &&
1853 IdxTy.getSizeInBits() == 32 &&
1854 isLegalVecType;
1855 })
1856 .bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
1857 scalarOrEltNarrowerThan(VecTypeIdx, 32)),
1858 bitcastToVectorElement32(VecTypeIdx))
1859 //.bitcastIf(vectorSmallerThan(1, 32), bitcastToScalar(1))
1860 .bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
1861 scalarOrEltWiderThan(VecTypeIdx, 64)),
1862 [=](const LegalityQuery &Query) {
1863 // For > 64-bit element types, try to turn this into a
1864 // 64-bit element vector since we may be able to do better
1865 // indexing if this is scalar. If not, fall back to 32.
1866 const LLT EltTy = Query.Types[EltTypeIdx];
1867 const LLT VecTy = Query.Types[VecTypeIdx];
1868 const unsigned DstEltSize = EltTy.getSizeInBits();
1869 const unsigned VecSize = VecTy.getSizeInBits();
1870
1871 const unsigned TargetEltSize =
1872 DstEltSize % 64 == 0 ? 64 : 32;
1873 return std::pair(VecTypeIdx,
1874 LLT::fixed_vector(VecSize / TargetEltSize,
1875 TargetEltSize));
1876 })
1877 .clampScalar(EltTypeIdx, S32, S64)
1878 .clampScalar(VecTypeIdx, S32, S64)
1879 .clampScalar(IdxTypeIdx, S32, S32)
1880 .clampMaxNumElements(VecTypeIdx, S32, 32)
1881 // TODO: Clamp elements for 64-bit vectors?
1882 .moreElementsIf(isIllegalRegisterType(ST, VecTypeIdx),
1884 // It should only be necessary with variable indexes.
1885 // As a last resort, lower to the stack
1886 .lower();
1887 }
1888
1889 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
1890 .unsupportedIf([=](const LegalityQuery &Query) {
1891 const LLT &EltTy = Query.Types[1].getElementType();
1892 return Query.Types[0] != EltTy;
1893 });
1894
1895 for (unsigned Op : {G_EXTRACT, G_INSERT}) {
1896 unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
1897 unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
1898
1899 // FIXME: Doesn't handle extract of illegal sizes.
1901 .lowerIf(all(typeIs(LitTyIdx, S16), sizeIs(BigTyIdx, 32)))
1902 .lowerIf([=](const LegalityQuery &Query) {
1903 // Sub-vector(or single element) insert and extract.
1904 // TODO: verify immediate offset here since lower only works with
1905 // whole elements.
1906 const LLT BigTy = Query.Types[BigTyIdx];
1907 return BigTy.isVector();
1908 })
1909 // FIXME: Multiples of 16 should not be legal.
1910 .legalIf([=](const LegalityQuery &Query) {
1911 const LLT BigTy = Query.Types[BigTyIdx];
1912 const LLT LitTy = Query.Types[LitTyIdx];
1913 return (BigTy.getSizeInBits() % 32 == 0) &&
1914 (LitTy.getSizeInBits() % 16 == 0);
1915 })
1916 .widenScalarIf(
1917 [=](const LegalityQuery &Query) {
1918 const LLT BigTy = Query.Types[BigTyIdx];
1919 return (BigTy.getScalarSizeInBits() < 16);
1920 },
1922 .widenScalarIf(
1923 [=](const LegalityQuery &Query) {
1924 const LLT LitTy = Query.Types[LitTyIdx];
1925 return (LitTy.getScalarSizeInBits() < 16);
1926 },
1928 .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
1929 .widenScalarToNextPow2(BigTyIdx, 32);
1930
1931 }
1932
1933 auto &BuildVector =
1934 getActionDefinitionsBuilder(G_BUILD_VECTOR)
1936 .legalForCartesianProduct(AllS64Vectors, {S64})
1937 .clampNumElements(0, V16S32, V32S32)
1942
1943 if (ST.hasScalarPackInsts()) {
1944 BuildVector
1945 // FIXME: Should probably widen s1 vectors straight to s32
1946 .minScalarOrElt(0, S16)
1947 .minScalar(1, S16);
1948
1949 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
1950 .legalFor({V2S16, S32})
1951 .lower();
1952 } else {
1953 BuildVector.customFor({V2S16, S16});
1954 BuildVector.minScalarOrElt(0, S32);
1955
1956 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
1957 .customFor({V2S16, S32})
1958 .lower();
1959 }
1960
1961 BuildVector.legalIf(isRegisterType(ST, 0));
1962
1963 // FIXME: Clamp maximum size
1964 getActionDefinitionsBuilder(G_CONCAT_VECTORS)
1965 .legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
1966 .clampMaxNumElements(0, S32, 32)
1967 .clampMaxNumElements(1, S16, 2) // TODO: Make 4?
1968 .clampMaxNumElements(0, S16, 64);
1969
1970 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR).lower();
1971
1972 // Merge/Unmerge
1973 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
1974 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
1975 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
1976
1977 auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
1978 const LLT Ty = Query.Types[TypeIdx];
1979 if (Ty.isVector()) {
1980 const LLT &EltTy = Ty.getElementType();
1981 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 512)
1982 return true;
1984 return true;
1985 }
1986 return false;
1987 };
1988
1989 auto &Builder =
1991 .legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
1992 .lowerFor({{S16, V2S16}})
1993 .lowerIf([=](const LegalityQuery &Query) {
1994 const LLT BigTy = Query.Types[BigTyIdx];
1995 return BigTy.getSizeInBits() == 32;
1996 })
1997 // Try to widen to s16 first for small types.
1998 // TODO: Only do this on targets with legal s16 shifts
1999 .minScalarOrEltIf(scalarNarrowerThan(LitTyIdx, 16), LitTyIdx, S16)
2000 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
2002 oneMoreElement(BigTyIdx))
2004 elementTypeIs(1, S16)),
2005 changeTo(1, V2S16))
2006 // Clamp the little scalar to s8-s256 and make it a power of 2. It's
2007 // not worth considering the multiples of 64 since 2*192 and 2*384
2008 // are not valid.
2009 .clampScalar(LitTyIdx, S32, S512)
2010 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
2011 // Break up vectors with weird elements into scalars
2013 [=](const LegalityQuery &Query) {
2014 return notValidElt(Query, LitTyIdx);
2015 },
2016 scalarize(0))
2017 .fewerElementsIf(
2018 [=](const LegalityQuery &Query) {
2019 return notValidElt(Query, BigTyIdx);
2020 },
2021 scalarize(1))
2022 .clampScalar(BigTyIdx, S32, MaxScalar);
2023
2024 if (Op == G_MERGE_VALUES) {
2025 Builder.widenScalarIf(
2026 // TODO: Use 16-bit shifts if legal for 8-bit values?
2027 [=](const LegalityQuery &Query) {
2028 const LLT Ty = Query.Types[LitTyIdx];
2029 return Ty.getSizeInBits() < 32;
2030 },
2031 changeTo(LitTyIdx, S32));
2032 }
2033
2034 Builder.widenScalarIf(
2035 [=](const LegalityQuery &Query) {
2036 const LLT Ty = Query.Types[BigTyIdx];
2037 return Ty.getSizeInBits() % 16 != 0;
2038 },
2039 [=](const LegalityQuery &Query) {
2040 // Pick the next power of 2, or a multiple of 64 over 128.
2041 // Whichever is smaller.
2042 const LLT &Ty = Query.Types[BigTyIdx];
2043 unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
2044 if (NewSizeInBits >= 256) {
2045 unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
2046 if (RoundedTo < NewSizeInBits)
2047 NewSizeInBits = RoundedTo;
2048 }
2049 return std::pair(BigTyIdx, LLT::scalar(NewSizeInBits));
2050 })
2051 // Any vectors left are the wrong size. Scalarize them.
2052 .scalarize(0)
2053 .scalarize(1);
2054 }
2055
2056 // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
2057 // RegBankSelect.
2058 auto &SextInReg = getActionDefinitionsBuilder(G_SEXT_INREG)
2059 .legalFor({{S32}, {S64}})
2060 .clampScalar(0, S32, S64);
2061
2062 if (ST.hasVOP3PInsts()) {
2063 SextInReg.lowerFor({{V2S16}})
2064 // Prefer to reduce vector widths for 16-bit vectors before lowering, to
2065 // get more vector shift opportunities, since we'll get those when
2066 // expanded.
2067 .clampMaxNumElementsStrict(0, S16, 2);
2068 } else if (ST.has16BitInsts()) {
2069 SextInReg.lowerFor({{S32}, {S64}, {S16}});
2070 } else {
2071 // Prefer to promote to s32 before lowering if we don't have 16-bit
2072 // shifts. This avoid a lot of intermediate truncate and extend operations.
2073 SextInReg.lowerFor({{S32}, {S64}});
2074 }
2075
2076 SextInReg
2077 .scalarize(0)
2078 .clampScalar(0, S32, S64)
2079 .lower();
2080
2081 getActionDefinitionsBuilder({G_ROTR, G_ROTL})
2082 .scalarize(0)
2083 .lower();
2084
2085 auto &FSHRActionDefs = getActionDefinitionsBuilder(G_FSHR);
2086 FSHRActionDefs.legalFor({{S32, S32}})
2087 .clampMaxNumElementsStrict(0, S16, 2);
2088 if (ST.hasVOP3PInsts())
2089 FSHRActionDefs.lowerFor({{V2S16, V2S16}});
2090 FSHRActionDefs.scalarize(0).lower();
2091
2092 if (ST.hasVOP3PInsts()) {
2094 .lowerFor({{V2S16, V2S16}})
2095 .clampMaxNumElementsStrict(0, S16, 2)
2096 .scalarize(0)
2097 .lower();
2098 } else {
2100 .scalarize(0)
2101 .lower();
2102 }
2103
2104 getActionDefinitionsBuilder(G_READCYCLECOUNTER)
2105 .legalFor({S64});
2106
2107 getActionDefinitionsBuilder(G_READSTEADYCOUNTER).legalFor({S64});
2108
2110 .alwaysLegal();
2111
2112 getActionDefinitionsBuilder({G_SMULO, G_UMULO})
2113 .scalarize(0)
2114 .minScalar(0, S32)
2115 .lower();
2116
2117 getActionDefinitionsBuilder({G_SBFX, G_UBFX})
2118 .legalFor({{S32, S32}, {S64, S32}})
2119 .clampScalar(1, S32, S32)
2120 .clampScalar(0, S32, S64)
2122 .scalarize(0);
2123
2125 {// TODO: Verify V_BFI_B32 is generated from expanded bit ops
2126 G_FCOPYSIGN,
2127
2128 G_ATOMIC_CMPXCHG_WITH_SUCCESS, G_ATOMICRMW_NAND, G_ATOMICRMW_FSUB,
2129 G_READ_REGISTER, G_WRITE_REGISTER,
2130
2131 G_SADDO, G_SSUBO})
2132 .lower();
2133
2134 if (ST.hasIEEEMinimumMaximumInsts()) {
2135 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
2136 .legalFor(FPTypesPK16)
2137 .clampMaxNumElements(0, S16, 2)
2138 .scalarize(0);
2139 } else {
2140 // TODO: Implement
2141 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM}).lower();
2142 }
2143
2144 getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET})
2145 .lower();
2146
2147 getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();
2148
2149 getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
2150 G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
2151 G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
2152 .unsupported();
2153
2155
2157 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
2158 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
2159 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
2160 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
2161 .legalFor(AllVectors)
2162 .scalarize(1)
2163 .lower();
2164
2166 verify(*ST.getInstrInfo());
2167}
2168
2171 LostDebugLocObserver &LocObserver) const {
2172 MachineIRBuilder &B = Helper.MIRBuilder;
2173 MachineRegisterInfo &MRI = *B.getMRI();
2174
2175 switch (MI.getOpcode()) {
2176 case TargetOpcode::G_ADDRSPACE_CAST:
2177 return legalizeAddrSpaceCast(MI, MRI, B);
2178 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2179 return legalizeFroundeven(MI, MRI, B);
2180 case TargetOpcode::G_FCEIL:
2181 return legalizeFceil(MI, MRI, B);
2182 case TargetOpcode::G_FREM:
2183 return legalizeFrem(MI, MRI, B);
2184 case TargetOpcode::G_INTRINSIC_TRUNC:
2185 return legalizeIntrinsicTrunc(MI, MRI, B);
2186 case TargetOpcode::G_SITOFP:
2187 return legalizeITOFP(MI, MRI, B, true);
2188 case TargetOpcode::G_UITOFP:
2189 return legalizeITOFP(MI, MRI, B, false);
2190 case TargetOpcode::G_FPTOSI:
2191 return legalizeFPTOI(MI, MRI, B, true);
2192 case TargetOpcode::G_FPTOUI:
2193 return legalizeFPTOI(MI, MRI, B, false);
2194 case TargetOpcode::G_FMINNUM:
2195 case TargetOpcode::G_FMAXNUM:
2196 case TargetOpcode::G_FMINIMUMNUM:
2197 case TargetOpcode::G_FMAXIMUMNUM:
2198 case TargetOpcode::G_FMINNUM_IEEE:
2199 case TargetOpcode::G_FMAXNUM_IEEE:
2200 return legalizeMinNumMaxNum(Helper, MI);
2201 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
2202 return legalizeExtractVectorElt(MI, MRI, B);
2203 case TargetOpcode::G_INSERT_VECTOR_ELT:
2204 return legalizeInsertVectorElt(MI, MRI, B);
2205 case TargetOpcode::G_FSIN:
2206 case TargetOpcode::G_FCOS:
2207 return legalizeSinCos(MI, MRI, B);
2208 case TargetOpcode::G_GLOBAL_VALUE:
2209 return legalizeGlobalValue(MI, MRI, B);
2210 case TargetOpcode::G_LOAD:
2211 case TargetOpcode::G_SEXTLOAD:
2212 case TargetOpcode::G_ZEXTLOAD:
2213 return legalizeLoad(Helper, MI);
2214 case TargetOpcode::G_STORE:
2215 return legalizeStore(Helper, MI);
2216 case TargetOpcode::G_FMAD:
2217 return legalizeFMad(MI, MRI, B);
2218 case TargetOpcode::G_FDIV:
2219 return legalizeFDIV(MI, MRI, B);
2220 case TargetOpcode::G_FFREXP:
2221 return legalizeFFREXP(MI, MRI, B);
2222 case TargetOpcode::G_FSQRT:
2223 return legalizeFSQRT(MI, MRI, B);
2224 case TargetOpcode::G_UDIV:
2225 case TargetOpcode::G_UREM:
2226 case TargetOpcode::G_UDIVREM:
2227 return legalizeUnsignedDIV_REM(MI, MRI, B);
2228 case TargetOpcode::G_SDIV:
2229 case TargetOpcode::G_SREM:
2230 case TargetOpcode::G_SDIVREM:
2231 return legalizeSignedDIV_REM(MI, MRI, B);
2232 case TargetOpcode::G_ATOMIC_CMPXCHG:
2233 return legalizeAtomicCmpXChg(MI, MRI, B);
2234 case TargetOpcode::G_FLOG2:
2235 return legalizeFlog2(MI, B);
2236 case TargetOpcode::G_FLOG:
2237 case TargetOpcode::G_FLOG10:
2238 return legalizeFlogCommon(MI, B);
2239 case TargetOpcode::G_FEXP2:
2240 return legalizeFExp2(MI, B);
2241 case TargetOpcode::G_FEXP:
2242 case TargetOpcode::G_FEXP10:
2243 return legalizeFExp(MI, B);
2244 case TargetOpcode::G_FPOW:
2245 return legalizeFPow(MI, B);
2246 case TargetOpcode::G_FFLOOR:
2247 return legalizeFFloor(MI, MRI, B);
2248 case TargetOpcode::G_BUILD_VECTOR:
2249 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
2250 return legalizeBuildVector(MI, MRI, B);
2251 case TargetOpcode::G_MUL:
2252 return legalizeMul(Helper, MI);
2253 case TargetOpcode::G_CTLZ:
2254 case TargetOpcode::G_CTTZ:
2255 return legalizeCTLZ_CTTZ(MI, MRI, B);
2256 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2257 return legalizeCTLZ_ZERO_UNDEF(MI, MRI, B);
2258 case TargetOpcode::G_STACKSAVE:
2259 return legalizeStackSave(MI, B);
2260 case TargetOpcode::G_GET_FPENV:
2261 return legalizeGetFPEnv(MI, MRI, B);
2262 case TargetOpcode::G_SET_FPENV:
2263 return legalizeSetFPEnv(MI, MRI, B);
2264 case TargetOpcode::G_TRAP:
2265 return legalizeTrap(MI, MRI, B);
2266 case TargetOpcode::G_DEBUGTRAP:
2267 return legalizeDebugTrap(MI, MRI, B);
2268 default:
2269 return false;
2270 }
2271
2272 llvm_unreachable("expected switch to return");
2273}
2274
2276 unsigned AS,
2278 MachineIRBuilder &B) const {
2279 MachineFunction &MF = B.getMF();
2280 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2281 const LLT S32 = LLT::scalar(32);
2282 const LLT S64 = LLT::scalar(64);
2283
2285
2286 if (ST.hasApertureRegs()) {
2287 // Note: this register is somewhat broken. When used as a 32-bit operand,
2288 // it only returns zeroes. The real value is in the upper 32 bits.
2289 // Thus, we must emit extract the high 32 bits.
2290 const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
2291 ? AMDGPU::SRC_SHARED_BASE
2292 : AMDGPU::SRC_PRIVATE_BASE;
2293 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
2294 !ST.hasGloballyAddressableScratch()) &&
2295 "Cannot use src_private_base with globally addressable scratch!");
2296 Register Dst = MRI.createGenericVirtualRegister(S64);
2297 MRI.setRegClass(Dst, &AMDGPU::SReg_64RegClass);
2298 B.buildCopy({Dst}, {Register(ApertureRegNo)});
2299 return B.buildUnmerge(S32, Dst).getReg(1);
2300 }
2301
2302 // TODO: can we be smarter about machine pointer info?
2304 Register LoadAddr = MRI.createGenericVirtualRegister(
2306 // For code object version 5, private_base and shared_base are passed through
2307 // implicit kernargs.
2314 ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);
2315
2316 Register KernargPtrReg = MRI.createGenericVirtualRegister(
2318
2319 if (!loadInputValue(KernargPtrReg, B,
2321 return Register();
2322
2324 PtrInfo,
2328
2329 // Pointer address
2330 B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
2331 B.buildConstant(LLT::scalar(64), Offset).getReg(0));
2332 // Load address
2333 return B.buildLoad(S32, LoadAddr, *MMO).getReg(0);
2334 }
2335
2336 Register QueuePtr = MRI.createGenericVirtualRegister(
2338
2340 return Register();
2341
2342 // Offset into amd_queue_t for group_segment_aperture_base_hi /
2343 // private_segment_aperture_base_hi.
2344 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
2345
2347 PtrInfo,
2350 LLT::scalar(32), commonAlignment(Align(64), StructOffset));
2351
2352 B.buildObjectPtrOffset(
2353 LoadAddr, QueuePtr,
2354 B.buildConstant(LLT::scalar(64), StructOffset).getReg(0));
2355 return B.buildLoad(S32, LoadAddr, *MMO).getReg(0);
2356}
2357
2358/// Return true if the value is a known valid address, such that a null check is
2359/// not necessary.
2361 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
2362 MachineInstr *Def = MRI.getVRegDef(Val);
2363 switch (Def->getOpcode()) {
2364 case AMDGPU::G_FRAME_INDEX:
2365 case AMDGPU::G_GLOBAL_VALUE:
2366 case AMDGPU::G_BLOCK_ADDR:
2367 return true;
2368 case AMDGPU::G_CONSTANT: {
2369 const ConstantInt *CI = Def->getOperand(1).getCImm();
2370 return CI->getSExtValue() != TM.getNullPointerValue(AddrSpace);
2371 }
2372 default:
2373 return false;
2374 }
2375
2376 return false;
2377}
2378
2381 MachineIRBuilder &B) const {
2382 MachineFunction &MF = B.getMF();
2383
2384 // MI can either be a G_ADDRSPACE_CAST or a
2385 // G_INTRINSIC @llvm.amdgcn.addrspacecast.nonnull
2386 assert(MI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
2387 (isa<GIntrinsic>(MI) && cast<GIntrinsic>(MI).getIntrinsicID() ==
2388 Intrinsic::amdgcn_addrspacecast_nonnull));
2389
2390 const LLT S32 = LLT::scalar(32);
2391 Register Dst = MI.getOperand(0).getReg();
2392 Register Src = isa<GIntrinsic>(MI) ? MI.getOperand(2).getReg()
2393 : MI.getOperand(1).getReg();
2394 LLT DstTy = MRI.getType(Dst);
2395 LLT SrcTy = MRI.getType(Src);
2396 unsigned DestAS = DstTy.getAddressSpace();
2397 unsigned SrcAS = SrcTy.getAddressSpace();
2398
2399 // TODO: Avoid reloading from the queue ptr for each cast, or at least each
2400 // vector element.
2401 assert(!DstTy.isVector());
2402
2403 const AMDGPUTargetMachine &TM
2404 = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
2405
2406 if (TM.isNoopAddrSpaceCast(SrcAS, DestAS)) {
2407 MI.setDesc(B.getTII().get(TargetOpcode::G_BITCAST));
2408 return true;
2409 }
2410
2411 if (SrcAS == AMDGPUAS::FLAT_ADDRESS &&
2412 (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
2413 DestAS == AMDGPUAS::PRIVATE_ADDRESS)) {
2414 auto castFlatToLocalOrPrivate = [&](const DstOp &Dst) -> Register {
2415 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2416 ST.hasGloballyAddressableScratch()) {
2417 // flat -> private with globally addressable scratch: subtract
2418 // src_flat_scratch_base_lo.
2419 const LLT S32 = LLT::scalar(32);
2420 Register SrcLo = B.buildExtract(S32, Src, 0).getReg(0);
2421 Register FlatScratchBaseLo =
2422 B.buildInstr(AMDGPU::S_MOV_B32, {S32},
2423 {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)})
2424 .getReg(0);
2425 MRI.setRegClass(FlatScratchBaseLo, &AMDGPU::SReg_32RegClass);
2426 Register Sub = B.buildSub(S32, SrcLo, FlatScratchBaseLo).getReg(0);
2427 return B.buildIntToPtr(Dst, Sub).getReg(0);
2428 }
2429
2430 // Extract low 32-bits of the pointer.
2431 return B.buildExtract(Dst, Src, 0).getReg(0);
2432 };
2433
2434 // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
2435 // G_ADDRSPACE_CAST we need to guess.
2436 if (isa<GIntrinsic>(MI) || isKnownNonNull(Src, MRI, TM, SrcAS)) {
2437 castFlatToLocalOrPrivate(Dst);
2438 MI.eraseFromParent();
2439 return true;
2440 }
2441
2442 unsigned NullVal = TM.getNullPointerValue(DestAS);
2443
2444 auto SegmentNull = B.buildConstant(DstTy, NullVal);
2445 auto FlatNull = B.buildConstant(SrcTy, 0);
2446
2447 // Extract low 32-bits of the pointer.
2448 auto PtrLo32 = castFlatToLocalOrPrivate(DstTy);
2449
2450 auto CmpRes =
2451 B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, FlatNull.getReg(0));
2452 B.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));
2453
2454 MI.eraseFromParent();
2455 return true;
2456 }
2457
2458 if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
2459 (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
2460 SrcAS == AMDGPUAS::PRIVATE_ADDRESS)) {
2461 auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
2462 // Coerce the type of the low half of the result so we can use
2463 // merge_values.
2464 Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0);
2465
2466 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
2467 ST.hasGloballyAddressableScratch()) {
2468 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
2469 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
2470 Register AllOnes = B.buildConstant(S32, -1).getReg(0);
2471 Register ThreadID = B.buildConstant(S32, 0).getReg(0);
2472 ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {S32})
2473 .addUse(AllOnes)
2474 .addUse(ThreadID)
2475 .getReg(0);
2476 if (ST.isWave64()) {
2477 ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {S32})
2478 .addUse(AllOnes)
2479 .addUse(ThreadID)
2480 .getReg(0);
2481 }
2482 Register ShAmt =
2483 B.buildConstant(S32, 57 - 32 - ST.getWavefrontSizeLog2()).getReg(0);
2484 Register SrcHi = B.buildShl(S32, ThreadID, ShAmt).getReg(0);
2485 Register CvtPtr =
2486 B.buildMergeLikeInstr(DstTy, {SrcAsInt, SrcHi}).getReg(0);
2487 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
2488 // 64-bit hi:lo value.
2489 Register FlatScratchBase =
2490 B.buildInstr(AMDGPU::S_MOV_B64, {S64},
2491 {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE)})
2492 .getReg(0);
2493 MRI.setRegClass(FlatScratchBase, &AMDGPU::SReg_64RegClass);
2494 return B.buildPtrAdd(Dst, CvtPtr, FlatScratchBase).getReg(0);
2495 }
2496
2497 Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
2498 if (!ApertureReg.isValid())
2499 return false;
2500
2501 // TODO: Should we allow mismatched types but matching sizes in merges to
2502 // avoid the ptrtoint?
2503 return B.buildMergeLikeInstr(Dst, {SrcAsInt, ApertureReg}).getReg(0);
2504 };
2505
2506 // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
2507 // G_ADDRSPACE_CAST we need to guess.
2508 if (isa<GIntrinsic>(MI) || isKnownNonNull(Src, MRI, TM, SrcAS)) {
2509 castLocalOrPrivateToFlat(Dst);
2510 MI.eraseFromParent();
2511 return true;
2512 }
2513
2514 Register BuildPtr = castLocalOrPrivateToFlat(DstTy);
2515
2516 auto SegmentNull = B.buildConstant(SrcTy, TM.getNullPointerValue(SrcAS));
2517 auto FlatNull = B.buildConstant(DstTy, TM.getNullPointerValue(DestAS));
2518
2519 auto CmpRes = B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src,
2520 SegmentNull.getReg(0));
2521
2522 B.buildSelect(Dst, CmpRes, BuildPtr, FlatNull);
2523
2524 MI.eraseFromParent();
2525 return true;
2526 }
2527
2528 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
2529 SrcTy.getSizeInBits() == 64) {
2530 // Truncate.
2531 B.buildExtract(Dst, Src, 0);
2532 MI.eraseFromParent();
2533 return true;
2534 }
2535
2536 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
2537 DstTy.getSizeInBits() == 64) {
2539 uint32_t AddrHiVal = Info->get32BitAddressHighBits();
2540 auto PtrLo = B.buildPtrToInt(S32, Src);
2541 auto HighAddr = B.buildConstant(S32, AddrHiVal);
2542 B.buildMergeLikeInstr(Dst, {PtrLo, HighAddr});
2543 MI.eraseFromParent();
2544 return true;
2545 }
2546
2547 // Invalid casts are poison.
2548 // TODO: Should return poison
2549 B.buildUndef(Dst);
2550 MI.eraseFromParent();
2551 return true;
2552}
2553
2556 MachineIRBuilder &B) const {
2557 Register Src = MI.getOperand(1).getReg();
2558 LLT Ty = MRI.getType(Src);
2559 assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
2560
2561 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2562 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2563
2564 auto C1 = B.buildFConstant(Ty, C1Val);
2565 auto CopySign = B.buildFCopysign(Ty, C1, Src);
2566
2567 // TODO: Should this propagate fast-math-flags?
2568 auto Tmp1 = B.buildFAdd(Ty, Src, CopySign);
2569 auto Tmp2 = B.buildFSub(Ty, Tmp1, CopySign);
2570
2571 auto C2 = B.buildFConstant(Ty, C2Val);
2572 auto Fabs = B.buildFAbs(Ty, Src);
2573
2574 auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
2575 B.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
2576 MI.eraseFromParent();
2577 return true;
2578}
2579
2582 MachineIRBuilder &B) const {
2583
2584 const LLT S1 = LLT::scalar(1);
2585 const LLT S64 = LLT::scalar(64);
2586
2587 Register Src = MI.getOperand(1).getReg();
2588 assert(MRI.getType(Src) == S64);
2589
2590 // result = trunc(src)
2591 // if (src > 0.0 && src != result)
2592 // result += 1.0
2593
2594 auto Trunc = B.buildIntrinsicTrunc(S64, Src);
2595
2596 const auto Zero = B.buildFConstant(S64, 0.0);
2597 const auto One = B.buildFConstant(S64, 1.0);
2598 auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
2599 auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
2600 auto And = B.buildAnd(S1, Lt0, NeTrunc);
2601 auto Add = B.buildSelect(S64, And, One, Zero);
2602
2603 // TODO: Should this propagate fast-math-flags?
2604 B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
2605 MI.eraseFromParent();
2606 return true;
2607}
2608
2611 MachineIRBuilder &B) const {
2612 Register DstReg = MI.getOperand(0).getReg();
2613 Register Src0Reg = MI.getOperand(1).getReg();
2614 Register Src1Reg = MI.getOperand(2).getReg();
2615 auto Flags = MI.getFlags();
2616 LLT Ty = MRI.getType(DstReg);
2617
2618 auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags);
2619 auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
2620 auto Neg = B.buildFNeg(Ty, Trunc, Flags);
2621 B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
2622 MI.eraseFromParent();
2623 return true;
2624}
2625
2628 const unsigned FractBits = 52;
2629 const unsigned ExpBits = 11;
2630 LLT S32 = LLT::scalar(32);
2631
2632 auto Const0 = B.buildConstant(S32, FractBits - 32);
2633 auto Const1 = B.buildConstant(S32, ExpBits);
2634
2635 auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {S32})
2636 .addUse(Hi)
2637 .addUse(Const0.getReg(0))
2638 .addUse(Const1.getReg(0));
2639
2640 return B.buildSub(S32, ExpPart, B.buildConstant(S32, 1023));
2641}
2642
2645 MachineIRBuilder &B) const {
2646 const LLT S1 = LLT::scalar(1);
2647 const LLT S32 = LLT::scalar(32);
2648 const LLT S64 = LLT::scalar(64);
2649
2650 Register Src = MI.getOperand(1).getReg();
2651 assert(MRI.getType(Src) == S64);
2652
2653 // TODO: Should this use extract since the low half is unused?
2654 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
2655 Register Hi = Unmerge.getReg(1);
2656
2657 // Extract the upper half, since this is where we will find the sign and
2658 // exponent.
2659 auto Exp = extractF64Exponent(Hi, B);
2660
2661 const unsigned FractBits = 52;
2662
2663 // Extract the sign bit.
2664 const auto SignBitMask = B.buildConstant(S32, UINT32_C(1) << 31);
2665 auto SignBit = B.buildAnd(S32, Hi, SignBitMask);
2666
2667 const auto FractMask = B.buildConstant(S64, (UINT64_C(1) << FractBits) - 1);
2668
2669 const auto Zero32 = B.buildConstant(S32, 0);
2670
2671 // Extend back to 64-bits.
2672 auto SignBit64 = B.buildMergeLikeInstr(S64, {Zero32, SignBit});
2673
2674 auto Shr = B.buildAShr(S64, FractMask, Exp);
2675 auto Not = B.buildNot(S64, Shr);
2676 auto Tmp0 = B.buildAnd(S64, Src, Not);
2677 auto FiftyOne = B.buildConstant(S32, FractBits - 1);
2678
2679 auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
2680 auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);
2681
2682 auto Tmp1 = B.buildSelect(S64, ExpLt0, SignBit64, Tmp0);
2683 B.buildSelect(MI.getOperand(0).getReg(), ExpGt51, Src, Tmp1);
2684 MI.eraseFromParent();
2685 return true;
2686}
2687
2690 MachineIRBuilder &B, bool Signed) const {
2691
2692 Register Dst = MI.getOperand(0).getReg();
2693 Register Src = MI.getOperand(1).getReg();
2694
2695 const LLT S64 = LLT::scalar(64);
2696 const LLT S32 = LLT::scalar(32);
2697
2698 assert(MRI.getType(Src) == S64);
2699
2700 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
2701 auto ThirtyTwo = B.buildConstant(S32, 32);
2702
2703 if (MRI.getType(Dst) == S64) {
2704 auto CvtHi = Signed ? B.buildSITOFP(S64, Unmerge.getReg(1))
2705 : B.buildUITOFP(S64, Unmerge.getReg(1));
2706
2707 auto CvtLo = B.buildUITOFP(S64, Unmerge.getReg(0));
2708 auto LdExp = B.buildFLdexp(S64, CvtHi, ThirtyTwo);
2709
2710 // TODO: Should this propagate fast-math-flags?
2711 B.buildFAdd(Dst, LdExp, CvtLo);
2712 MI.eraseFromParent();
2713 return true;
2714 }
2715
2716 assert(MRI.getType(Dst) == S32);
2717
2718 auto One = B.buildConstant(S32, 1);
2719
2720 MachineInstrBuilder ShAmt;
2721 if (Signed) {
2722 auto ThirtyOne = B.buildConstant(S32, 31);
2723 auto X = B.buildXor(S32, Unmerge.getReg(0), Unmerge.getReg(1));
2724 auto OppositeSign = B.buildAShr(S32, X, ThirtyOne);
2725 auto MaxShAmt = B.buildAdd(S32, ThirtyTwo, OppositeSign);
2726 auto LS = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {S32})
2727 .addUse(Unmerge.getReg(1));
2728 auto LS2 = B.buildSub(S32, LS, One);
2729 ShAmt = B.buildUMin(S32, LS2, MaxShAmt);
2730 } else
2731 ShAmt = B.buildCTLZ(S32, Unmerge.getReg(1));
2732 auto Norm = B.buildShl(S64, Src, ShAmt);
2733 auto Unmerge2 = B.buildUnmerge({S32, S32}, Norm);
2734 auto Adjust = B.buildUMin(S32, One, Unmerge2.getReg(0));
2735 auto Norm2 = B.buildOr(S32, Unmerge2.getReg(1), Adjust);
2736 auto FVal = Signed ? B.buildSITOFP(S32, Norm2) : B.buildUITOFP(S32, Norm2);
2737 auto Scale = B.buildSub(S32, ThirtyTwo, ShAmt);
2738 B.buildFLdexp(Dst, FVal, Scale);
2739 MI.eraseFromParent();
2740 return true;
2741}
2742
2743// TODO: Copied from DAG implementation. Verify logic and document how this
2744// actually works.
2748 bool Signed) const {
2749
2750 Register Dst = MI.getOperand(0).getReg();
2751 Register Src = MI.getOperand(1).getReg();
2752
2753 const LLT S64 = LLT::scalar(64);
2754 const LLT S32 = LLT::scalar(32);
2755
2756 const LLT SrcLT = MRI.getType(Src);
2757 assert((SrcLT == S32 || SrcLT == S64) && MRI.getType(Dst) == S64);
2758
2759 unsigned Flags = MI.getFlags();
2760
2761 // The basic idea of converting a floating point number into a pair of 32-bit
2762 // integers is illustrated as follows:
2763 //
2764 // tf := trunc(val);
2765 // hif := floor(tf * 2^-32);
2766 // lof := tf - hif * 2^32; // lof is always positive due to floor.
2767 // hi := fptoi(hif);
2768 // lo := fptoi(lof);
2769 //
2770 auto Trunc = B.buildIntrinsicTrunc(SrcLT, Src, Flags);
2772 if (Signed && SrcLT == S32) {
2773 // However, a 32-bit floating point number has only 23 bits mantissa and
2774 // it's not enough to hold all the significant bits of `lof` if val is
2775 // negative. To avoid the loss of precision, We need to take the absolute
2776 // value after truncating and flip the result back based on the original
2777 // signedness.
2778 Sign = B.buildAShr(S32, Src, B.buildConstant(S32, 31));
2779 Trunc = B.buildFAbs(S32, Trunc, Flags);
2780 }
2781 MachineInstrBuilder K0, K1;
2782 if (SrcLT == S64) {
2783 K0 = B.buildFConstant(
2784 S64, llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)));
2785 K1 = B.buildFConstant(
2786 S64, llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)));
2787 } else {
2788 K0 = B.buildFConstant(
2789 S32, llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)));
2790 K1 = B.buildFConstant(
2791 S32, llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)));
2792 }
2793
2794 auto Mul = B.buildFMul(SrcLT, Trunc, K0, Flags);
2795 auto FloorMul = B.buildFFloor(SrcLT, Mul, Flags);
2796 auto Fma = B.buildFMA(SrcLT, FloorMul, K1, Trunc, Flags);
2797
2798 auto Hi = (Signed && SrcLT == S64) ? B.buildFPTOSI(S32, FloorMul)
2799 : B.buildFPTOUI(S32, FloorMul);
2800 auto Lo = B.buildFPTOUI(S32, Fma);
2801
2802 if (Signed && SrcLT == S32) {
2803 // Flip the result based on the signedness, which is either all 0s or 1s.
2804 Sign = B.buildMergeLikeInstr(S64, {Sign, Sign});
2805 // r := xor({lo, hi}, sign) - sign;
2806 B.buildSub(Dst, B.buildXor(S64, B.buildMergeLikeInstr(S64, {Lo, Hi}), Sign),
2807 Sign);
2808 } else
2809 B.buildMergeLikeInstr(Dst, {Lo, Hi});
2810 MI.eraseFromParent();
2811
2812 return true;
2813}
2814
2816 MachineInstr &MI) const {
2817 MachineFunction &MF = Helper.MIRBuilder.getMF();
2819
2820 const bool IsIEEEOp = MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE ||
2821 MI.getOpcode() == AMDGPU::G_FMAXNUM_IEEE;
2822
2823 // With ieee_mode disabled, the instructions have the correct behavior
2824 // already for G_FMINIMUMNUM/G_FMAXIMUMNUM.
2825 //
2826 // FIXME: G_FMINNUM/G_FMAXNUM should match the behavior with ieee_mode
2827 // enabled.
2828 if (!MFI->getMode().IEEE) {
2829 if (MI.getOpcode() == AMDGPU::G_FMINIMUMNUM ||
2830 MI.getOpcode() == AMDGPU::G_FMAXIMUMNUM)
2831 return true;
2832
2833 return !IsIEEEOp;
2834 }
2835
2836 if (IsIEEEOp)
2837 return true;
2838
2840}
2841
2844 MachineIRBuilder &B) const {
2845 // TODO: Should move some of this into LegalizerHelper.
2846
2847 // TODO: Promote dynamic indexing of s16 to s32
2848
2849 Register Dst = MI.getOperand(0).getReg();
2850 Register Vec = MI.getOperand(1).getReg();
2851
2852 LLT VecTy = MRI.getType(Vec);
2853 LLT EltTy = VecTy.getElementType();
2854 assert(EltTy == MRI.getType(Dst));
2855
2856 // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
2857 // but we can't go directly to that logic becasue you can't bitcast a vector
2858 // of pointers to a vector of integers. Therefore, introduce an intermediate
2859 // vector of integers using ptrtoint (and inttoptr on the output) in order to
2860 // drive the legalization forward.
2861 if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
2862 LLT IntTy = LLT::scalar(EltTy.getSizeInBits());
2863 LLT IntVecTy = VecTy.changeElementType(IntTy);
2864
2865 auto IntVec = B.buildPtrToInt(IntVecTy, Vec);
2866 auto IntElt = B.buildExtractVectorElement(IntTy, IntVec, MI.getOperand(2));
2867 B.buildIntToPtr(Dst, IntElt);
2868
2869 MI.eraseFromParent();
2870 return true;
2871 }
2872
2873 // FIXME: Artifact combiner probably should have replaced the truncated
2874 // constant before this, so we shouldn't need
2875 // getIConstantVRegValWithLookThrough.
2876 std::optional<ValueAndVReg> MaybeIdxVal =
2877 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
2878 if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
2879 return true;
2880 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
2881
2882 if (IdxVal < VecTy.getNumElements()) {
2883 auto Unmerge = B.buildUnmerge(EltTy, Vec);
2884 B.buildCopy(Dst, Unmerge.getReg(IdxVal));
2885 } else {
2886 B.buildUndef(Dst);
2887 }
2888
2889 MI.eraseFromParent();
2890 return true;
2891}
2892
2895 MachineIRBuilder &B) const {
2896 // TODO: Should move some of this into LegalizerHelper.
2897
2898 // TODO: Promote dynamic indexing of s16 to s32
2899
2900 Register Dst = MI.getOperand(0).getReg();
2901 Register Vec = MI.getOperand(1).getReg();
2902 Register Ins = MI.getOperand(2).getReg();
2903
2904 LLT VecTy = MRI.getType(Vec);
2905 LLT EltTy = VecTy.getElementType();
2906 assert(EltTy == MRI.getType(Ins));
2907
2908 // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
2909 // but we can't go directly to that logic becasue you can't bitcast a vector
2910 // of pointers to a vector of integers. Therefore, make the pointer vector
2911 // into an equivalent vector of integers with ptrtoint, insert the ptrtoint'd
2912 // new value, and then inttoptr the result vector back. This will then allow
2913 // the rest of legalization to take over.
2914 if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
2915 LLT IntTy = LLT::scalar(EltTy.getSizeInBits());
2916 LLT IntVecTy = VecTy.changeElementType(IntTy);
2917
2918 auto IntVecSource = B.buildPtrToInt(IntVecTy, Vec);
2919 auto IntIns = B.buildPtrToInt(IntTy, Ins);
2920 auto IntVecDest = B.buildInsertVectorElement(IntVecTy, IntVecSource, IntIns,
2921 MI.getOperand(3));
2922 B.buildIntToPtr(Dst, IntVecDest);
2923 MI.eraseFromParent();
2924 return true;
2925 }
2926
2927 // FIXME: Artifact combiner probably should have replaced the truncated
2928 // constant before this, so we shouldn't need
2929 // getIConstantVRegValWithLookThrough.
2930 std::optional<ValueAndVReg> MaybeIdxVal =
2931 getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
2932 if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
2933 return true;
2934
2935 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
2936
2937 unsigned NumElts = VecTy.getNumElements();
2938 if (IdxVal < NumElts) {
2940 for (unsigned i = 0; i < NumElts; ++i)
2941 SrcRegs.push_back(MRI.createGenericVirtualRegister(EltTy));
2942 B.buildUnmerge(SrcRegs, Vec);
2943
2944 SrcRegs[IdxVal] = MI.getOperand(2).getReg();
2945 B.buildMergeLikeInstr(Dst, SrcRegs);
2946 } else {
2947 B.buildUndef(Dst);
2948 }
2949
2950 MI.eraseFromParent();
2951 return true;
2952}
2953
2956 MachineIRBuilder &B) const {
2957
2958 Register DstReg = MI.getOperand(0).getReg();
2959 Register SrcReg = MI.getOperand(1).getReg();
2960 LLT Ty = MRI.getType(DstReg);
2961 unsigned Flags = MI.getFlags();
2962
2963 Register TrigVal;
2964 auto OneOver2Pi = B.buildFConstant(Ty, 0.5 * numbers::inv_pi);
2965 if (ST.hasTrigReducedRange()) {
2966 auto MulVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags);
2967 TrigVal = B.buildIntrinsic(Intrinsic::amdgcn_fract, {Ty})
2968 .addUse(MulVal.getReg(0))
2969 .setMIFlags(Flags)
2970 .getReg(0);
2971 } else
2972 TrigVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags).getReg(0);
2973
2974 Intrinsic::ID TrigIntrin = MI.getOpcode() == AMDGPU::G_FSIN ?
2975 Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
2976 B.buildIntrinsic(TrigIntrin, ArrayRef<Register>(DstReg))
2977 .addUse(TrigVal)
2978 .setMIFlags(Flags);
2979 MI.eraseFromParent();
2980 return true;
2981}
2982
2985 const GlobalValue *GV,
2986 int64_t Offset,
2987 unsigned GAFlags) const {
2988 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
2989 // In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
2990 // to the following code sequence:
2991 //
2992 // For constant address space:
2993 // s_getpc_b64 s[0:1]
2994 // s_add_u32 s0, s0, $symbol
2995 // s_addc_u32 s1, s1, 0
2996 //
2997 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
2998 // a fixup or relocation is emitted to replace $symbol with a literal
2999 // constant, which is a pc-relative offset from the encoding of the $symbol
3000 // operand to the global variable.
3001 //
3002 // For global address space:
3003 // s_getpc_b64 s[0:1]
3004 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
3005 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
3006 //
3007 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
3008 // fixups or relocations are emitted to replace $symbol@*@lo and
3009 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
3010 // which is a 64-bit pc-relative offset from the encoding of the $symbol
3011 // operand to the global variable.
3012
3014
3015 Register PCReg = PtrTy.getSizeInBits() != 32 ? DstReg :
3016 B.getMRI()->createGenericVirtualRegister(ConstPtrTy);
3017
3018 if (ST.has64BitLiterals()) {
3019 assert(GAFlags != SIInstrInfo::MO_NONE);
3020
3022 B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET64).addDef(PCReg);
3023 MIB.addGlobalAddress(GV, Offset, GAFlags + 2);
3024 } else {
3026 B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET).addDef(PCReg);
3027
3028 MIB.addGlobalAddress(GV, Offset, GAFlags);
3029 if (GAFlags == SIInstrInfo::MO_NONE)
3030 MIB.addImm(0);
3031 else
3032 MIB.addGlobalAddress(GV, Offset, GAFlags + 1);
3033 }
3034
3035 if (!B.getMRI()->getRegClassOrNull(PCReg))
3036 B.getMRI()->setRegClass(PCReg, &AMDGPU::SReg_64RegClass);
3037
3038 if (PtrTy.getSizeInBits() == 32)
3039 B.buildExtract(DstReg, PCReg, 0);
3040 return true;
3041}
3042
3043// Emit a ABS32_LO / ABS32_HI relocation stub.
3045 Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV,
3046 MachineRegisterInfo &MRI) const {
3047 bool RequiresHighHalf = PtrTy.getSizeInBits() != 32;
3048
3049 if (RequiresHighHalf && ST.has64BitLiterals()) {
3050 if (!MRI.getRegClassOrNull(DstReg))
3051 MRI.setRegClass(DstReg, &AMDGPU::SReg_64RegClass);
3052 B.buildInstr(AMDGPU::S_MOV_B64)
3053 .addDef(DstReg)
3054 .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS64);
3055 return;
3056 }
3057
3058 LLT S32 = LLT::scalar(32);
3059
3060 // Use the destination directly, if and only if we store the lower address
3061 // part only and we don't have a register class being set.
3062 Register AddrLo = !RequiresHighHalf && !MRI.getRegClassOrNull(DstReg)
3063 ? DstReg
3064 : MRI.createGenericVirtualRegister(S32);
3065
3066 if (!MRI.getRegClassOrNull(AddrLo))
3067 MRI.setRegClass(AddrLo, &AMDGPU::SReg_32RegClass);
3068
3069 // Write the lower half.
3070 B.buildInstr(AMDGPU::S_MOV_B32)
3071 .addDef(AddrLo)
3072 .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_LO);
3073
3074 // If required, write the upper half as well.
3075 if (RequiresHighHalf) {
3076 assert(PtrTy.getSizeInBits() == 64 &&
3077 "Must provide a 64-bit pointer type!");
3078
3079 Register AddrHi = MRI.createGenericVirtualRegister(S32);
3080 MRI.setRegClass(AddrHi, &AMDGPU::SReg_32RegClass);
3081
3082 B.buildInstr(AMDGPU::S_MOV_B32)
3083 .addDef(AddrHi)
3084 .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_HI);
3085
3086 // Use the destination directly, if and only if we don't have a register
3087 // class being set.
3088 Register AddrDst = !MRI.getRegClassOrNull(DstReg)
3089 ? DstReg
3090 : MRI.createGenericVirtualRegister(LLT::scalar(64));
3091
3092 if (!MRI.getRegClassOrNull(AddrDst))
3093 MRI.setRegClass(AddrDst, &AMDGPU::SReg_64RegClass);
3094
3095 B.buildMergeValues(AddrDst, {AddrLo, AddrHi});
3096
3097 // If we created a new register for the destination, cast the result into
3098 // the final output.
3099 if (AddrDst != DstReg)
3100 B.buildCast(DstReg, AddrDst);
3101 } else if (AddrLo != DstReg) {
3102 // If we created a new register for the destination, cast the result into
3103 // the final output.
3104 B.buildCast(DstReg, AddrLo);
3105 }
3106}
3107
3110 MachineIRBuilder &B) const {
3111 Register DstReg = MI.getOperand(0).getReg();
3112 LLT Ty = MRI.getType(DstReg);
3113 unsigned AS = Ty.getAddressSpace();
3114
3115 const GlobalValue *GV = MI.getOperand(1).getGlobal();
3116 MachineFunction &MF = B.getMF();
3118
3120 if (!MFI->isModuleEntryFunction() &&
3121 GV->getName() != "llvm.amdgcn.module.lds" &&
3123 const Function &Fn = MF.getFunction();
3125 Fn, "local memory global used by non-kernel function",
3126 MI.getDebugLoc(), DS_Warning));
3127
3128 // We currently don't have a way to correctly allocate LDS objects that
3129 // aren't directly associated with a kernel. We do force inlining of
3130 // functions that use local objects. However, if these dead functions are
3131 // not eliminated, we don't want a compile time error. Just emit a warning
3132 // and a trap, since there should be no callable path here.
3133 B.buildTrap();
3134 B.buildUndef(DstReg);
3135 MI.eraseFromParent();
3136 return true;
3137 }
3138
3139 // TODO: We could emit code to handle the initialization somewhere.
3140 // We ignore the initializer for now and legalize it to allow selection.
3141 // The initializer will anyway get errored out during assembly emission.
3142 const SITargetLowering *TLI = ST.getTargetLowering();
3143 if (!TLI->shouldUseLDSConstAddress(GV)) {
3144 MI.getOperand(1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
3145 return true; // Leave in place;
3146 }
3147
3148 if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
3149 Type *Ty = GV->getValueType();
3150 // HIP uses an unsized array `extern __shared__ T s[]` or similar
3151 // zero-sized type in other languages to declare the dynamic shared
3152 // memory which size is not known at the compile time. They will be
3153 // allocated by the runtime and placed directly after the static
3154 // allocated ones. They all share the same offset.
3155 if (B.getDataLayout().getTypeAllocSize(Ty).isZero()) {
3156 // Adjust alignment for that dynamic shared memory array.
3158 LLT S32 = LLT::scalar(32);
3159 auto Sz = B.buildIntrinsic(Intrinsic::amdgcn_groupstaticsize, {S32});
3160 B.buildIntToPtr(DstReg, Sz);
3161 MI.eraseFromParent();
3162 return true;
3163 }
3164 }
3165
3166 B.buildConstant(DstReg, MFI->allocateLDSGlobal(B.getDataLayout(),
3167 *cast<GlobalVariable>(GV)));
3168 MI.eraseFromParent();
3169 return true;
3170 }
3171
3172 if (ST.isAmdPalOS() || ST.isMesa3DOS()) {
3173 buildAbsGlobalAddress(DstReg, Ty, B, GV, MRI);
3174 MI.eraseFromParent();
3175 return true;
3176 }
3177
3178 const SITargetLowering *TLI = ST.getTargetLowering();
3179
3180 if (TLI->shouldEmitFixup(GV)) {
3181 buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0);
3182 MI.eraseFromParent();
3183 return true;
3184 }
3185
3186 if (TLI->shouldEmitPCReloc(GV)) {
3187 buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0, SIInstrInfo::MO_REL32);
3188 MI.eraseFromParent();
3189 return true;
3190 }
3191
3193 Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);
3194
3195 LLT LoadTy = Ty.getSizeInBits() == 32 ? PtrTy : Ty;
3200 LoadTy, Align(8));
3201
3202 buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);
3203
3204 if (Ty.getSizeInBits() == 32) {
3205 // Truncate if this is a 32-bit constant address.
3206 auto Load = B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
3207 B.buildExtract(DstReg, Load, 0);
3208 } else
3209 B.buildLoad(DstReg, GOTAddr, *GOTMMO);
3210
3211 MI.eraseFromParent();
3212 return true;
3213}
3214
3216 if (Ty.isVector())
3217 return Ty.changeElementCount(
3218 ElementCount::getFixed(PowerOf2Ceil(Ty.getNumElements())));
3219 return LLT::scalar(PowerOf2Ceil(Ty.getSizeInBits()));
3220}
3221
3223 MachineInstr &MI) const {
3224 MachineIRBuilder &B = Helper.MIRBuilder;
3225 MachineRegisterInfo &MRI = *B.getMRI();
3226 GISelChangeObserver &Observer = Helper.Observer;
3227
3228 Register PtrReg = MI.getOperand(1).getReg();
3229 LLT PtrTy = MRI.getType(PtrReg);
3230 unsigned AddrSpace = PtrTy.getAddressSpace();
3231
3232 if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
3234 auto Cast = B.buildAddrSpaceCast(ConstPtr, PtrReg);
3235 Observer.changingInstr(MI);
3236 MI.getOperand(1).setReg(Cast.getReg(0));
3237 Observer.changedInstr(MI);
3238 return true;
3239 }
3240
3241 if (MI.getOpcode() != AMDGPU::G_LOAD)
3242 return false;
3243
3244 Register ValReg = MI.getOperand(0).getReg();
3245 LLT ValTy = MRI.getType(ValReg);
3246
3247 if (hasBufferRsrcWorkaround(ValTy)) {
3248 Observer.changingInstr(MI);
3250 Observer.changedInstr(MI);
3251 return true;
3252 }
3253
3254 MachineMemOperand *MMO = *MI.memoperands_begin();
3255 const unsigned ValSize = ValTy.getSizeInBits();
3256 const LLT MemTy = MMO->getMemoryType();
3257 const Align MemAlign = MMO->getAlign();
3258 const unsigned MemSize = MemTy.getSizeInBits();
3259 const uint64_t AlignInBits = 8 * MemAlign.value();
3260
3261 // Widen non-power-of-2 loads to the alignment if needed
3262 if (shouldWidenLoad(ST, MemTy, AlignInBits, AddrSpace, MI.getOpcode())) {
3263 const unsigned WideMemSize = PowerOf2Ceil(MemSize);
3264
3265 // This was already the correct extending load result type, so just adjust
3266 // the memory type.
3267 if (WideMemSize == ValSize) {
3268 MachineFunction &MF = B.getMF();
3269
3270 MachineMemOperand *WideMMO =
3271 MF.getMachineMemOperand(MMO, 0, WideMemSize / 8);
3272 Observer.changingInstr(MI);
3273 MI.setMemRefs(MF, {WideMMO});
3274 Observer.changedInstr(MI);
3275 return true;
3276 }
3277
3278 // Don't bother handling edge case that should probably never be produced.
3279 if (ValSize > WideMemSize)
3280 return false;
3281
3282 LLT WideTy = widenToNextPowerOf2(ValTy);
3283
3284 Register WideLoad;
3285 if (!WideTy.isVector()) {
3286 WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3287 B.buildTrunc(ValReg, WideLoad).getReg(0);
3288 } else {
3289 // Extract the subvector.
3290
3291 if (isRegisterType(ST, ValTy)) {
3292 // If this a case where G_EXTRACT is legal, use it.
3293 // (e.g. <3 x s32> -> <4 x s32>)
3294 WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3295 B.buildExtract(ValReg, WideLoad, 0);
3296 } else {
3297 // For cases where the widened type isn't a nice register value, unmerge
3298 // from a widened register (e.g. <3 x s16> -> <4 x s16>)
3299 WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3300 B.buildDeleteTrailingVectorElements(ValReg, WideLoad);
3301 }
3302 }
3303
3304 MI.eraseFromParent();
3305 return true;
3306 }
3307
3308 return false;
3309}
3310
3312 MachineInstr &MI) const {
3313 MachineIRBuilder &B = Helper.MIRBuilder;
3314 MachineRegisterInfo &MRI = *B.getMRI();
3315 GISelChangeObserver &Observer = Helper.Observer;
3316
3317 Register DataReg = MI.getOperand(0).getReg();
3318 LLT DataTy = MRI.getType(DataReg);
3319
3320 if (hasBufferRsrcWorkaround(DataTy)) {
3321 Observer.changingInstr(MI);
3323 Observer.changedInstr(MI);
3324 return true;
3325 }
3326 return false;
3327}
3328
3331 MachineIRBuilder &B) const {
3332 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3333 assert(Ty.isScalar());
3334
3335 MachineFunction &MF = B.getMF();
3337
3338 // TODO: Always legal with future ftz flag.
3339 // FIXME: Do we need just output?
3340 if (Ty == LLT::float32() &&
3342 return true;
3343 if (Ty == LLT::float16() &&
3345 return true;
3346
3347 MachineIRBuilder HelperBuilder(MI);
3348 GISelObserverWrapper DummyObserver;
3349 LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
3350 return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
3351}
3352
3355 Register DstReg = MI.getOperand(0).getReg();
3356 Register PtrReg = MI.getOperand(1).getReg();
3357 Register CmpVal = MI.getOperand(2).getReg();
3358 Register NewVal = MI.getOperand(3).getReg();
3359
3360 assert(AMDGPU::isFlatGlobalAddrSpace(MRI.getType(PtrReg).getAddressSpace()) &&
3361 "this should not have been custom lowered");
3362
3363 LLT ValTy = MRI.getType(CmpVal);
3364 LLT VecTy = LLT::fixed_vector(2, ValTy);
3365
3366 Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0);
3367
3368 B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
3369 .addDef(DstReg)
3370 .addUse(PtrReg)
3371 .addUse(PackedVal)
3372 .setMemRefs(MI.memoperands());
3373
3374 MI.eraseFromParent();
3375 return true;
3376}
3377
3378/// Return true if it's known that \p Src can never be an f32 denormal value.
3380 Register Src) {
3381 const MachineInstr *DefMI = MRI.getVRegDef(Src);
3382 switch (DefMI->getOpcode()) {
3383 case TargetOpcode::G_INTRINSIC: {
3385 case Intrinsic::amdgcn_frexp_mant:
3386 return true;
3387 default:
3388 break;
3389 }
3390
3391 break;
3392 }
3393 case TargetOpcode::G_FFREXP: {
3394 if (DefMI->getOperand(0).getReg() == Src)
3395 return true;
3396 break;
3397 }
3398 case TargetOpcode::G_FPEXT: {
3399 return MRI.getType(DefMI->getOperand(1).getReg()) == LLT::scalar(16);
3400 }
3401 default:
3402 return false;
3403 }
3404
3405 return false;
3406}
3407
3408static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags) {
3409 return Flags & MachineInstr::FmAfn;
3410}
3411
3413 unsigned Flags) {
3414 return !valueIsKnownNeverF32Denorm(MF.getRegInfo(), Src) &&
3417}
3418
3419std::pair<Register, Register>
3421 unsigned Flags) const {
3422 if (!needsDenormHandlingF32(B.getMF(), Src, Flags))
3423 return {};
3424
3425 const LLT F32 = LLT::scalar(32);
3426 auto SmallestNormal = B.buildFConstant(
3428 auto IsLtSmallestNormal =
3429 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src, SmallestNormal);
3430
3431 auto Scale32 = B.buildFConstant(F32, 0x1.0p+32);
3432 auto One = B.buildFConstant(F32, 1.0);
3433 auto ScaleFactor =
3434 B.buildSelect(F32, IsLtSmallestNormal, Scale32, One, Flags);
3435 auto ScaledInput = B.buildFMul(F32, Src, ScaleFactor, Flags);
3436
3437 return {ScaledInput.getReg(0), IsLtSmallestNormal.getReg(0)};
3438}
3439
3441 MachineIRBuilder &B) const {
3442 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
3443 // If we have to handle denormals, scale up the input and adjust the result.
3444
3445 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
3446 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
3447
3448 Register Dst = MI.getOperand(0).getReg();
3449 Register Src = MI.getOperand(1).getReg();
3450 LLT Ty = B.getMRI()->getType(Dst);
3451 unsigned Flags = MI.getFlags();
3452
3453 if (Ty == LLT::scalar(16)) {
3454 const LLT F32 = LLT::scalar(32);
3455 // Nothing in half is a denormal when promoted to f32.
3456 auto Ext = B.buildFPExt(F32, Src, Flags);
3457 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {F32})
3458 .addUse(Ext.getReg(0))
3459 .setMIFlags(Flags);
3460 B.buildFPTrunc(Dst, Log2, Flags);
3461 MI.eraseFromParent();
3462 return true;
3463 }
3464
3465 assert(Ty == LLT::scalar(32));
3466
3467 auto [ScaledInput, IsLtSmallestNormal] = getScaledLogInput(B, Src, Flags);
3468 if (!ScaledInput) {
3469 B.buildIntrinsic(Intrinsic::amdgcn_log, {MI.getOperand(0)})
3470 .addUse(Src)
3471 .setMIFlags(Flags);
3472 MI.eraseFromParent();
3473 return true;
3474 }
3475
3476 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
3477 .addUse(ScaledInput)
3478 .setMIFlags(Flags);
3479
3480 auto ThirtyTwo = B.buildFConstant(Ty, 32.0);
3481 auto Zero = B.buildFConstant(Ty, 0.0);
3482 auto ResultOffset =
3483 B.buildSelect(Ty, IsLtSmallestNormal, ThirtyTwo, Zero, Flags);
3484 B.buildFSub(Dst, Log2, ResultOffset, Flags);
3485
3486 MI.eraseFromParent();
3487 return true;
3488}
3489
3491 Register Z, unsigned Flags) {
3492 auto FMul = B.buildFMul(Ty, X, Y, Flags);
3493 return B.buildFAdd(Ty, FMul, Z, Flags).getReg(0);
3494}
3495
3497 MachineIRBuilder &B) const {
3498 const bool IsLog10 = MI.getOpcode() == TargetOpcode::G_FLOG10;
3499 assert(IsLog10 || MI.getOpcode() == TargetOpcode::G_FLOG);
3500
3501 MachineRegisterInfo &MRI = *B.getMRI();
3502 Register Dst = MI.getOperand(0).getReg();
3503 Register X = MI.getOperand(1).getReg();
3504 unsigned Flags = MI.getFlags();
3505 const LLT Ty = MRI.getType(X);
3506 MachineFunction &MF = B.getMF();
3507
3508 const LLT F32 = LLT::scalar(32);
3509 const LLT F16 = LLT::scalar(16);
3510
3511 const AMDGPUTargetMachine &TM =
3512 static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
3513
3514 if (Ty == F16 || MI.getFlag(MachineInstr::FmAfn)) {
3515 if (Ty == F16 && !ST.has16BitInsts()) {
3516 Register LogVal = MRI.createGenericVirtualRegister(F32);
3517 auto PromoteSrc = B.buildFPExt(F32, X);
3518 legalizeFlogUnsafe(B, LogVal, PromoteSrc.getReg(0), IsLog10, Flags);
3519 B.buildFPTrunc(Dst, LogVal);
3520 } else {
3521 legalizeFlogUnsafe(B, Dst, X, IsLog10, Flags);
3522 }
3523
3524 MI.eraseFromParent();
3525 return true;
3526 }
3527
3528 auto [ScaledInput, IsScaled] = getScaledLogInput(B, X, Flags);
3529 if (ScaledInput)
3530 X = ScaledInput;
3531
3532 auto Y =
3533 B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty}).addUse(X).setMIFlags(Flags);
3534
3535 Register R;
3536 if (ST.hasFastFMAF32()) {
3537 // c+cc are ln(2)/ln(10) to more than 49 bits
3538 const float c_log10 = 0x1.344134p-2f;
3539 const float cc_log10 = 0x1.09f79ep-26f;
3540
3541 // c + cc is ln(2) to more than 49 bits
3542 const float c_log = 0x1.62e42ep-1f;
3543 const float cc_log = 0x1.efa39ep-25f;
3544
3545 auto C = B.buildFConstant(Ty, IsLog10 ? c_log10 : c_log);
3546 auto CC = B.buildFConstant(Ty, IsLog10 ? cc_log10 : cc_log);
3547
3548 R = B.buildFMul(Ty, Y, C, Flags).getReg(0);
3549 auto NegR = B.buildFNeg(Ty, R, Flags);
3550 auto FMA0 = B.buildFMA(Ty, Y, C, NegR, Flags);
3551 auto FMA1 = B.buildFMA(Ty, Y, CC, FMA0, Flags);
3552 R = B.buildFAdd(Ty, R, FMA1, Flags).getReg(0);
3553 } else {
3554 // ch+ct is ln(2)/ln(10) to more than 36 bits
3555 const float ch_log10 = 0x1.344000p-2f;
3556 const float ct_log10 = 0x1.3509f6p-18f;
3557
3558 // ch + ct is ln(2) to more than 36 bits
3559 const float ch_log = 0x1.62e000p-1f;
3560 const float ct_log = 0x1.0bfbe8p-15f;
3561
3562 auto CH = B.buildFConstant(Ty, IsLog10 ? ch_log10 : ch_log);
3563 auto CT = B.buildFConstant(Ty, IsLog10 ? ct_log10 : ct_log);
3564
3565 auto MaskConst = B.buildConstant(Ty, 0xfffff000);
3566 auto YH = B.buildAnd(Ty, Y, MaskConst);
3567 auto YT = B.buildFSub(Ty, Y, YH, Flags);
3568 auto YTCT = B.buildFMul(Ty, YT, CT, Flags);
3569
3570 Register Mad0 =
3571 getMad(B, Ty, YH.getReg(0), CT.getReg(0), YTCT.getReg(0), Flags);
3572 Register Mad1 = getMad(B, Ty, YT.getReg(0), CH.getReg(0), Mad0, Flags);
3573 R = getMad(B, Ty, YH.getReg(0), CH.getReg(0), Mad1, Flags);
3574 }
3575
3576 const bool IsFiniteOnly =
3577 (MI.getFlag(MachineInstr::FmNoNans) || TM.Options.NoNaNsFPMath) &&
3578 MI.getFlag(MachineInstr::FmNoInfs);
3579
3580 if (!IsFiniteOnly) {
3581 // Expand isfinite(x) => fabs(x) < inf
3582 auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
3583 auto Fabs = B.buildFAbs(Ty, Y);
3584 auto IsFinite =
3585 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
3586 R = B.buildSelect(Ty, IsFinite, R, Y, Flags).getReg(0);
3587 }
3588
3589 if (ScaledInput) {
3590 auto Zero = B.buildFConstant(Ty, 0.0);
3591 auto ShiftK =
3592 B.buildFConstant(Ty, IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f);
3593 auto Shift = B.buildSelect(Ty, IsScaled, ShiftK, Zero, Flags);
3594 B.buildFSub(Dst, R, Shift, Flags);
3595 } else {
3596 B.buildCopy(Dst, R);
3597 }
3598
3599 MI.eraseFromParent();
3600 return true;
3601}
3602
3604 Register Src, bool IsLog10,
3605 unsigned Flags) const {
3606 const double Log2BaseInverted =
3608
3609 LLT Ty = B.getMRI()->getType(Dst);
3610
3611 if (Ty == LLT::scalar(32)) {
3612 auto [ScaledInput, IsScaled] = getScaledLogInput(B, Src, Flags);
3613 if (ScaledInput) {
3614 auto LogSrc = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
3615 .addUse(Src)
3616 .setMIFlags(Flags);
3617 auto ScaledResultOffset = B.buildFConstant(Ty, -32.0 * Log2BaseInverted);
3618 auto Zero = B.buildFConstant(Ty, 0.0);
3619 auto ResultOffset =
3620 B.buildSelect(Ty, IsScaled, ScaledResultOffset, Zero, Flags);
3621 auto Log2Inv = B.buildFConstant(Ty, Log2BaseInverted);
3622
3623 if (ST.hasFastFMAF32())
3624 B.buildFMA(Dst, LogSrc, Log2Inv, ResultOffset, Flags);
3625 else {
3626 auto Mul = B.buildFMul(Ty, LogSrc, Log2Inv, Flags);
3627 B.buildFAdd(Dst, Mul, ResultOffset, Flags);
3628 }
3629
3630 return true;
3631 }
3632 }
3633
3634 auto Log2Operand = Ty == LLT::scalar(16)
3635 ? B.buildFLog2(Ty, Src, Flags)
3636 : B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
3637 .addUse(Src)
3638 .setMIFlags(Flags);
3639 auto Log2BaseInvertedOperand = B.buildFConstant(Ty, Log2BaseInverted);
3640 B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
3641 return true;
3642}
3643
3645 MachineIRBuilder &B) const {
3646 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
3647 // If we have to handle denormals, scale up the input and adjust the result.
3648
3649 Register Dst = MI.getOperand(0).getReg();
3650 Register Src = MI.getOperand(1).getReg();
3651 unsigned Flags = MI.getFlags();
3652 LLT Ty = B.getMRI()->getType(Dst);
3653 const LLT F16 = LLT::scalar(16);
3654 const LLT F32 = LLT::scalar(32);
3655
3656 if (Ty == F16) {
3657 // Nothing in half is a denormal when promoted to f32.
3658 auto Ext = B.buildFPExt(F32, Src, Flags);
3659 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {F32})
3660 .addUse(Ext.getReg(0))
3661 .setMIFlags(Flags);
3662 B.buildFPTrunc(Dst, Log2, Flags);
3663 MI.eraseFromParent();
3664 return true;
3665 }
3666
3667 assert(Ty == F32);
3668
3669 if (!needsDenormHandlingF32(B.getMF(), Src, Flags)) {
3670 B.buildIntrinsic(Intrinsic::amdgcn_exp2, ArrayRef<Register>{Dst})
3671 .addUse(Src)
3672 .setMIFlags(Flags);
3673 MI.eraseFromParent();
3674 return true;
3675 }
3676
3677 // bool needs_scaling = x < -0x1.f80000p+6f;
3678 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3679
3680 // -nextafter(128.0, -1)
3681 auto RangeCheckConst = B.buildFConstant(Ty, -0x1.f80000p+6f);
3682 auto NeedsScaling = B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src,
3683 RangeCheckConst, Flags);
3684
3685 auto SixtyFour = B.buildFConstant(Ty, 0x1.0p+6f);
3686 auto Zero = B.buildFConstant(Ty, 0.0);
3687 auto AddOffset = B.buildSelect(F32, NeedsScaling, SixtyFour, Zero, Flags);
3688 auto AddInput = B.buildFAdd(F32, Src, AddOffset, Flags);
3689
3690 auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
3691 .addUse(AddInput.getReg(0))
3692 .setMIFlags(Flags);
3693
3694 auto TwoExpNeg64 = B.buildFConstant(Ty, 0x1.0p-64f);
3695 auto One = B.buildFConstant(Ty, 1.0);
3696 auto ResultScale = B.buildSelect(F32, NeedsScaling, TwoExpNeg64, One, Flags);
3697 B.buildFMul(Dst, Exp2, ResultScale, Flags);
3698 MI.eraseFromParent();
3699 return true;
3700}
3701
3703 Register X, unsigned Flags) const {
3704 LLT Ty = B.getMRI()->getType(Dst);
3705 LLT F32 = LLT::scalar(32);
3706
3707 if (Ty != F32 || !needsDenormHandlingF32(B.getMF(), X, Flags)) {
3708 auto Log2E = B.buildFConstant(Ty, numbers::log2e);
3709 auto Mul = B.buildFMul(Ty, X, Log2E, Flags);
3710
3711 if (Ty == F32) {
3712 B.buildIntrinsic(Intrinsic::amdgcn_exp2, ArrayRef<Register>{Dst})
3713 .addUse(Mul.getReg(0))
3714 .setMIFlags(Flags);
3715 } else {
3716 B.buildFExp2(Dst, Mul.getReg(0), Flags);
3717 }
3718
3719 return true;
3720 }
3721
3722 auto Threshold = B.buildFConstant(Ty, -0x1.5d58a0p+6f);
3723 auto NeedsScaling =
3724 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold, Flags);
3725 auto ScaleOffset = B.buildFConstant(Ty, 0x1.0p+6f);
3726 auto ScaledX = B.buildFAdd(Ty, X, ScaleOffset, Flags);
3727 auto AdjustedX = B.buildSelect(Ty, NeedsScaling, ScaledX, X, Flags);
3728
3729 auto Log2E = B.buildFConstant(Ty, numbers::log2e);
3730 auto ExpInput = B.buildFMul(Ty, AdjustedX, Log2E, Flags);
3731
3732 auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
3733 .addUse(ExpInput.getReg(0))
3734 .setMIFlags(Flags);
3735
3736 auto ResultScaleFactor = B.buildFConstant(Ty, 0x1.969d48p-93f);
3737 auto AdjustedResult = B.buildFMul(Ty, Exp2, ResultScaleFactor, Flags);
3738 B.buildSelect(Dst, NeedsScaling, AdjustedResult, Exp2, Flags);
3739 return true;
3740}
3741
3743 MachineIRBuilder &B) const {
3744 Register Dst = MI.getOperand(0).getReg();
3745 Register X = MI.getOperand(1).getReg();
3746 const unsigned Flags = MI.getFlags();
3747 MachineFunction &MF = B.getMF();
3748 MachineRegisterInfo &MRI = *B.getMRI();
3749 LLT Ty = MRI.getType(Dst);
3750 const LLT F16 = LLT::scalar(16);
3751 const LLT F32 = LLT::scalar(32);
3752 const bool IsExp10 = MI.getOpcode() == TargetOpcode::G_FEXP10;
3753
3754 if (Ty == F16) {
3755 // v_exp_f16 (fmul x, log2e)
3756 if (allowApproxFunc(MF, Flags)) {
3757 // TODO: Does this really require fast?
3758 legalizeFExpUnsafe(B, Dst, X, Flags);
3759 MI.eraseFromParent();
3760 return true;
3761 }
3762
3763 // exp(f16 x) ->
3764 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3765
3766 // Nothing in half is a denormal when promoted to f32.
3767 auto Ext = B.buildFPExt(F32, X, Flags);
3768 Register Lowered = MRI.createGenericVirtualRegister(F32);
3769 legalizeFExpUnsafe(B, Lowered, Ext.getReg(0), Flags);
3770 B.buildFPTrunc(Dst, Lowered, Flags);
3771 MI.eraseFromParent();
3772 return true;
3773 }
3774
3775 assert(Ty == F32);
3776
3777 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3778 // library behavior. Also, is known-not-daz source sufficient?
3779 if (allowApproxFunc(MF, Flags)) {
3780 legalizeFExpUnsafe(B, Dst, X, Flags);
3781 MI.eraseFromParent();
3782 return true;
3783 }
3784
3785 // Algorithm:
3786 //
3787 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3788 //
3789 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3790 // n = 64*m + j, 0 <= j < 64
3791 //
3792 // e^x = 2^((64*m + j + f)/64)
3793 // = (2^m) * (2^(j/64)) * 2^(f/64)
3794 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3795 //
3796 // f = x*(64/ln(2)) - n
3797 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3798 //
3799 // e^x = (2^m) * (2^(j/64)) * e^r
3800 //
3801 // (2^(j/64)) is precomputed
3802 //
3803 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3804 // e^r = 1 + q
3805 //
3806 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3807 //
3808 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3809 const unsigned FlagsNoContract = Flags & ~MachineInstr::FmContract;
3810 Register PH, PL;
3811
3812 if (ST.hasFastFMAF32()) {
3813 const float c_exp = numbers::log2ef;
3814 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3815 const float c_exp10 = 0x1.a934f0p+1f;
3816 const float cc_exp10 = 0x1.2f346ep-24f;
3817
3818 auto C = B.buildFConstant(Ty, IsExp10 ? c_exp10 : c_exp);
3819 PH = B.buildFMul(Ty, X, C, Flags).getReg(0);
3820 auto NegPH = B.buildFNeg(Ty, PH, Flags);
3821 auto FMA0 = B.buildFMA(Ty, X, C, NegPH, Flags);
3822
3823 auto CC = B.buildFConstant(Ty, IsExp10 ? cc_exp10 : cc_exp);
3824 PL = B.buildFMA(Ty, X, CC, FMA0, Flags).getReg(0);
3825 } else {
3826 const float ch_exp = 0x1.714000p+0f;
3827 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3828
3829 const float ch_exp10 = 0x1.a92000p+1f;
3830 const float cl_exp10 = 0x1.4f0978p-11f;
3831
3832 auto MaskConst = B.buildConstant(Ty, 0xfffff000);
3833 auto XH = B.buildAnd(Ty, X, MaskConst);
3834 auto XL = B.buildFSub(Ty, X, XH, Flags);
3835
3836 auto CH = B.buildFConstant(Ty, IsExp10 ? ch_exp10 : ch_exp);
3837 PH = B.buildFMul(Ty, XH, CH, Flags).getReg(0);
3838
3839 auto CL = B.buildFConstant(Ty, IsExp10 ? cl_exp10 : cl_exp);
3840 auto XLCL = B.buildFMul(Ty, XL, CL, Flags);
3841
3842 Register Mad0 =
3843 getMad(B, Ty, XL.getReg(0), CH.getReg(0), XLCL.getReg(0), Flags);
3844 PL = getMad(B, Ty, XH.getReg(0), CL.getReg(0), Mad0, Flags);
3845 }
3846
3847 auto E = B.buildIntrinsicRoundeven(Ty, PH, Flags);
3848
3849 // It is unsafe to contract this fsub into the PH multiply.
3850 auto PHSubE = B.buildFSub(Ty, PH, E, FlagsNoContract);
3851 auto A = B.buildFAdd(Ty, PHSubE, PL, Flags);
3852 auto IntE = B.buildFPTOSI(LLT::scalar(32), E);
3853
3854 auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
3855 .addUse(A.getReg(0))
3856 .setMIFlags(Flags);
3857 auto R = B.buildFLdexp(Ty, Exp2, IntE, Flags);
3858
3859 auto UnderflowCheckConst =
3860 B.buildFConstant(Ty, IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f);
3861 auto Zero = B.buildFConstant(Ty, 0.0);
3862 auto Underflow =
3863 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, UnderflowCheckConst);
3864
3865 R = B.buildSelect(Ty, Underflow, Zero, R);
3866
3867 if (!(Flags & MachineInstr::FmNoInfs)) {
3868 auto OverflowCheckConst =
3869 B.buildFConstant(Ty, IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f);
3870
3871 auto Overflow =
3872 B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), X, OverflowCheckConst);
3873 auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
3874 R = B.buildSelect(Ty, Overflow, Inf, R, Flags);
3875 }
3876
3877 B.buildCopy(Dst, R);
3878 MI.eraseFromParent();
3879 return true;
3880}
3881
3883 MachineIRBuilder &B) const {
3884 Register Dst = MI.getOperand(0).getReg();
3885 Register Src0 = MI.getOperand(1).getReg();
3886 Register Src1 = MI.getOperand(2).getReg();
3887 unsigned Flags = MI.getFlags();
3888 LLT Ty = B.getMRI()->getType(Dst);
3889 const LLT F16 = LLT::float16();
3890 const LLT F32 = LLT::float32();
3891
3892 if (Ty == F32) {
3893 auto Log = B.buildFLog2(F32, Src0, Flags);
3894 auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
3895 .addUse(Log.getReg(0))
3896 .addUse(Src1)
3897 .setMIFlags(Flags);
3898 B.buildFExp2(Dst, Mul, Flags);
3899 } else if (Ty == F16) {
3900 // There's no f16 fmul_legacy, so we need to convert for it.
3901 auto Log = B.buildFLog2(F16, Src0, Flags);
3902 auto Ext0 = B.buildFPExt(F32, Log, Flags);
3903 auto Ext1 = B.buildFPExt(F32, Src1, Flags);
3904 auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
3905 .addUse(Ext0.getReg(0))
3906 .addUse(Ext1.getReg(0))
3907 .setMIFlags(Flags);
3908 B.buildFExp2(Dst, B.buildFPTrunc(F16, Mul), Flags);
3909 } else
3910 return false;
3911
3912 MI.eraseFromParent();
3913 return true;
3914}
3915
3916// Find a source register, ignoring any possible source modifiers.
3918 Register ModSrc = OrigSrc;
3919 if (MachineInstr *SrcFNeg = getOpcodeDef(AMDGPU::G_FNEG, ModSrc, MRI)) {
3920 ModSrc = SrcFNeg->getOperand(1).getReg();
3921 if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
3922 ModSrc = SrcFAbs->getOperand(1).getReg();
3923 } else if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
3924 ModSrc = SrcFAbs->getOperand(1).getReg();
3925 return ModSrc;
3926}
3927
3930 MachineIRBuilder &B) const {
3931
3932 const LLT S1 = LLT::scalar(1);
3933 const LLT F64 = LLT::float64();
3934 Register Dst = MI.getOperand(0).getReg();
3935 Register OrigSrc = MI.getOperand(1).getReg();
3936 unsigned Flags = MI.getFlags();
3937 assert(ST.hasFractBug() && MRI.getType(Dst) == F64 &&
3938 "this should not have been custom lowered");
3939
3940 // V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x))
3941 // is used instead. However, SI doesn't have V_FLOOR_F64, so the most
3942 // efficient way to implement it is using V_FRACT_F64. The workaround for the
3943 // V_FRACT bug is:
3944 // fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)
3945 //
3946 // Convert floor(x) to (x - fract(x))
3947
3948 auto Fract = B.buildIntrinsic(Intrinsic::amdgcn_fract, {F64})
3949 .addUse(OrigSrc)
3950 .setMIFlags(Flags);
3951
3952 // Give source modifier matching some assistance before obscuring a foldable
3953 // pattern.
3954
3955 // TODO: We can avoid the neg on the fract? The input sign to fract
3956 // shouldn't matter?
3957 Register ModSrc = stripAnySourceMods(OrigSrc, MRI);
3958
3959 auto Const =
3960 B.buildFConstant(F64, llvm::bit_cast<double>(0x3fefffffffffffff));
3961
3962 Register Min = MRI.createGenericVirtualRegister(F64);
3963
3964 // We don't need to concern ourselves with the snan handling difference, so
3965 // use the one which will directly select.
3966 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
3967 if (MFI->getMode().IEEE)
3968 B.buildFMinNumIEEE(Min, Fract, Const, Flags);
3969 else
3970 B.buildFMinNum(Min, Fract, Const, Flags);
3971
3972 Register CorrectedFract = Min;
3973 if (!MI.getFlag(MachineInstr::FmNoNans)) {
3974 auto IsNan = B.buildFCmp(CmpInst::FCMP_ORD, S1, ModSrc, ModSrc, Flags);
3975 CorrectedFract = B.buildSelect(F64, IsNan, ModSrc, Min, Flags).getReg(0);
3976 }
3977
3978 auto NegFract = B.buildFNeg(F64, CorrectedFract, Flags);
3979 B.buildFAdd(Dst, OrigSrc, NegFract, Flags);
3980
3981 MI.eraseFromParent();
3982 return true;
3983}
3984
3985// Turn an illegal packed v2s16 build vector into bit operations.
3986// TODO: This should probably be a bitcast action in LegalizerHelper.
3989 Register Dst = MI.getOperand(0).getReg();
3990 const LLT S32 = LLT::scalar(32);
3991 const LLT S16 = LLT::scalar(16);
3992 assert(MRI.getType(Dst) == LLT::fixed_vector(2, 16));
3993
3994 Register Src0 = MI.getOperand(1).getReg();
3995 Register Src1 = MI.getOperand(2).getReg();
3996
3997 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC) {
3998 assert(MRI.getType(Src0) == S32);
3999 Src0 = B.buildTrunc(S16, MI.getOperand(1).getReg()).getReg(0);
4000 Src1 = B.buildTrunc(S16, MI.getOperand(2).getReg()).getReg(0);
4001 }
4002
4003 auto Merge = B.buildMergeLikeInstr(S32, {Src0, Src1});
4004 B.buildBitcast(Dst, Merge);
4005
4006 MI.eraseFromParent();
4007 return true;
4008}
4009
4010// Build a big integer multiply or multiply-add using MAD_64_32 instructions.
4011//
4012// Source and accumulation registers must all be 32-bits.
4013//
4014// TODO: When the multiply is uniform, we should produce a code sequence
4015// that is better suited to instruction selection on the SALU. Instead of
4016// the outer loop going over parts of the result, the outer loop should go
4017// over parts of one of the factors. This should result in instruction
4018// selection that makes full use of S_ADDC_U32 instructions.
4021 ArrayRef<Register> Src0,
4022 ArrayRef<Register> Src1,
4023 bool UsePartialMad64_32,
4024 bool SeparateOddAlignedProducts) const {
4025 // Use (possibly empty) vectors of S1 registers to represent the set of
4026 // carries from one pair of positions to the next.
4027 using Carry = SmallVector<Register, 2>;
4028
4029 MachineIRBuilder &B = Helper.MIRBuilder;
4030 GISelValueTracking &VT = *Helper.getValueTracking();
4031
4032 const LLT S1 = LLT::scalar(1);
4033 const LLT S32 = LLT::scalar(32);
4034 const LLT S64 = LLT::scalar(64);
4035
4036 Register Zero32;
4037 Register Zero64;
4038
4039 auto getZero32 = [&]() -> Register {
4040 if (!Zero32)
4041 Zero32 = B.buildConstant(S32, 0).getReg(0);
4042 return Zero32;
4043 };
4044 auto getZero64 = [&]() -> Register {
4045 if (!Zero64)
4046 Zero64 = B.buildConstant(S64, 0).getReg(0);
4047 return Zero64;
4048 };
4049
4050 SmallVector<bool, 2> Src0KnownZeros, Src1KnownZeros;
4051 for (unsigned i = 0; i < Src0.size(); ++i) {
4052 Src0KnownZeros.push_back(VT.getKnownBits(Src0[i]).isZero());
4053 Src1KnownZeros.push_back(VT.getKnownBits(Src1[i]).isZero());
4054 }
4055
4056 // Merge the given carries into the 32-bit LocalAccum, which is modified
4057 // in-place.
4058 //
4059 // Returns the carry-out, which is a single S1 register or null.
4060 auto mergeCarry =
4061 [&](Register &LocalAccum, const Carry &CarryIn) -> Register {
4062 if (CarryIn.empty())
4063 return Register();
4064
4065 bool HaveCarryOut = true;
4066 Register CarryAccum;
4067 if (CarryIn.size() == 1) {
4068 if (!LocalAccum) {
4069 LocalAccum = B.buildZExt(S32, CarryIn[0]).getReg(0);
4070 return Register();
4071 }
4072
4073 CarryAccum = getZero32();
4074 } else {
4075 CarryAccum = B.buildZExt(S32, CarryIn[0]).getReg(0);
4076 for (unsigned i = 1; i + 1 < CarryIn.size(); ++i) {
4077 CarryAccum =
4078 B.buildUAdde(S32, S1, CarryAccum, getZero32(), CarryIn[i])
4079 .getReg(0);
4080 }
4081
4082 if (!LocalAccum) {
4083 LocalAccum = getZero32();
4084 HaveCarryOut = false;
4085 }
4086 }
4087
4088 auto Add =
4089 B.buildUAdde(S32, S1, CarryAccum, LocalAccum, CarryIn.back());
4090 LocalAccum = Add.getReg(0);
4091 return HaveCarryOut ? Add.getReg(1) : Register();
4092 };
4093
4094 // Build a multiply-add chain to compute
4095 //
4096 // LocalAccum + (partial products at DstIndex)
4097 // + (opportunistic subset of CarryIn)
4098 //
4099 // LocalAccum is an array of one or two 32-bit registers that are updated
4100 // in-place. The incoming registers may be null.
4101 //
4102 // In some edge cases, carry-ins can be consumed "for free". In that case,
4103 // the consumed carry bits are removed from CarryIn in-place.
4104 auto buildMadChain =
4105 [&](MutableArrayRef<Register> LocalAccum, unsigned DstIndex, Carry &CarryIn)
4106 -> Carry {
4107 assert((DstIndex + 1 < Accum.size() && LocalAccum.size() == 2) ||
4108 (DstIndex + 1 >= Accum.size() && LocalAccum.size() == 1));
4109
4110 Carry CarryOut;
4111 unsigned j0 = 0;
4112
4113 // Use plain 32-bit multiplication for the most significant part of the
4114 // result by default.
4115 if (LocalAccum.size() == 1 &&
4116 (!UsePartialMad64_32 || !CarryIn.empty())) {
4117 do {
4118 // Skip multiplication if one of the operands is 0
4119 unsigned j1 = DstIndex - j0;
4120 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4121 ++j0;
4122 continue;
4123 }
4124 auto Mul = B.buildMul(S32, Src0[j0], Src1[j1]);
4125 if (!LocalAccum[0] || VT.getKnownBits(LocalAccum[0]).isZero()) {
4126 LocalAccum[0] = Mul.getReg(0);
4127 } else {
4128 if (CarryIn.empty()) {
4129 LocalAccum[0] = B.buildAdd(S32, LocalAccum[0], Mul).getReg(0);
4130 } else {
4131 LocalAccum[0] =
4132 B.buildUAdde(S32, S1, LocalAccum[0], Mul, CarryIn.back())
4133 .getReg(0);
4134 CarryIn.pop_back();
4135 }
4136 }
4137 ++j0;
4138 } while (j0 <= DstIndex && (!UsePartialMad64_32 || !CarryIn.empty()));
4139 }
4140
4141 // Build full 64-bit multiplies.
4142 if (j0 <= DstIndex) {
4143 bool HaveSmallAccum = false;
4144 Register Tmp;
4145
4146 if (LocalAccum[0]) {
4147 if (LocalAccum.size() == 1) {
4148 Tmp = B.buildAnyExt(S64, LocalAccum[0]).getReg(0);
4149 HaveSmallAccum = true;
4150 } else if (LocalAccum[1]) {
4151 Tmp = B.buildMergeLikeInstr(S64, LocalAccum).getReg(0);
4152 HaveSmallAccum = false;
4153 } else {
4154 Tmp = B.buildZExt(S64, LocalAccum[0]).getReg(0);
4155 HaveSmallAccum = true;
4156 }
4157 } else {
4158 assert(LocalAccum.size() == 1 || !LocalAccum[1]);
4159 Tmp = getZero64();
4160 HaveSmallAccum = true;
4161 }
4162
4163 do {
4164 unsigned j1 = DstIndex - j0;
4165 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4166 ++j0;
4167 continue;
4168 }
4169 auto Mad = B.buildInstr(AMDGPU::G_AMDGPU_MAD_U64_U32, {S64, S1},
4170 {Src0[j0], Src1[j1], Tmp});
4171 Tmp = Mad.getReg(0);
4172 if (!HaveSmallAccum)
4173 CarryOut.push_back(Mad.getReg(1));
4174 HaveSmallAccum = false;
4175
4176 ++j0;
4177 } while (j0 <= DstIndex);
4178
4179 auto Unmerge = B.buildUnmerge(S32, Tmp);
4180 LocalAccum[0] = Unmerge.getReg(0);
4181 if (LocalAccum.size() > 1)
4182 LocalAccum[1] = Unmerge.getReg(1);
4183 }
4184
4185 return CarryOut;
4186 };
4187
4188 // Outer multiply loop, iterating over destination parts from least
4189 // significant to most significant parts.
4190 //
4191 // The columns of the following diagram correspond to the destination parts
4192 // affected by one iteration of the outer loop (ignoring boundary
4193 // conditions).
4194 //
4195 // Dest index relative to 2 * i: 1 0 -1
4196 // ------
4197 // Carries from previous iteration: e o
4198 // Even-aligned partial product sum: E E .
4199 // Odd-aligned partial product sum: O O
4200 //
4201 // 'o' is OddCarry, 'e' is EvenCarry.
4202 // EE and OO are computed from partial products via buildMadChain and use
4203 // accumulation where possible and appropriate.
4204 //
4205 Register SeparateOddCarry;
4206 Carry EvenCarry;
4207 Carry OddCarry;
4208
4209 for (unsigned i = 0; i <= Accum.size() / 2; ++i) {
4210 Carry OddCarryIn = std::move(OddCarry);
4211 Carry EvenCarryIn = std::move(EvenCarry);
4212 OddCarry.clear();
4213 EvenCarry.clear();
4214
4215 // Partial products at offset 2 * i.
4216 if (2 * i < Accum.size()) {
4217 auto LocalAccum = Accum.drop_front(2 * i).take_front(2);
4218 EvenCarry = buildMadChain(LocalAccum, 2 * i, EvenCarryIn);
4219 }
4220
4221 // Partial products at offset 2 * i - 1.
4222 if (i > 0) {
4223 if (!SeparateOddAlignedProducts) {
4224 auto LocalAccum = Accum.drop_front(2 * i - 1).take_front(2);
4225 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4226 } else {
4227 bool IsHighest = 2 * i >= Accum.size();
4228 Register SeparateOddOut[2];
4229 auto LocalAccum = MutableArrayRef(SeparateOddOut)
4230 .take_front(IsHighest ? 1 : 2);
4231 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4232
4234
4235 if (i == 1) {
4236 if (!IsHighest)
4237 Lo = B.buildUAddo(S32, S1, Accum[2 * i - 1], SeparateOddOut[0]);
4238 else
4239 Lo = B.buildAdd(S32, Accum[2 * i - 1], SeparateOddOut[0]);
4240 } else {
4241 Lo = B.buildUAdde(S32, S1, Accum[2 * i - 1], SeparateOddOut[0],
4242 SeparateOddCarry);
4243 }
4244 Accum[2 * i - 1] = Lo->getOperand(0).getReg();
4245
4246 if (!IsHighest) {
4247 auto Hi = B.buildUAdde(S32, S1, Accum[2 * i], SeparateOddOut[1],
4248 Lo->getOperand(1).getReg());
4249 Accum[2 * i] = Hi.getReg(0);
4250 SeparateOddCarry = Hi.getReg(1);
4251 }
4252 }
4253 }
4254
4255 // Add in the carries from the previous iteration
4256 if (i > 0) {
4257 if (Register CarryOut = mergeCarry(Accum[2 * i - 1], OddCarryIn))
4258 EvenCarryIn.push_back(CarryOut);
4259
4260 if (2 * i < Accum.size()) {
4261 if (Register CarryOut = mergeCarry(Accum[2 * i], EvenCarryIn))
4262 OddCarry.push_back(CarryOut);
4263 }
4264 }
4265 }
4266}
4267
4268// Custom narrowing of wide multiplies using wide multiply-add instructions.
4269//
4270// TODO: If the multiply is followed by an addition, we should attempt to
4271// integrate it to make better use of V_MAD_U64_U32's multiply-add capabilities.
4273 MachineInstr &MI) const {
4274 assert(ST.hasMad64_32());
4275 assert(MI.getOpcode() == TargetOpcode::G_MUL);
4276
4277 MachineIRBuilder &B = Helper.MIRBuilder;
4278 MachineRegisterInfo &MRI = *B.getMRI();
4279
4280 Register DstReg = MI.getOperand(0).getReg();
4281 Register Src0 = MI.getOperand(1).getReg();
4282 Register Src1 = MI.getOperand(2).getReg();
4283
4284 LLT Ty = MRI.getType(DstReg);
4285 assert(Ty.isScalar());
4286
4287 unsigned Size = Ty.getSizeInBits();
4288 if (ST.hasVectorMulU64() && Size == 64)
4289 return true;
4290
4291 unsigned NumParts = Size / 32;
4292 assert((Size % 32) == 0);
4293 assert(NumParts >= 2);
4294
4295 // Whether to use MAD_64_32 for partial products whose high half is
4296 // discarded. This avoids some ADD instructions but risks false dependency
4297 // stalls on some subtargets in some cases.
4298 const bool UsePartialMad64_32 = ST.getGeneration() < AMDGPUSubtarget::GFX10;
4299
4300 // Whether to compute odd-aligned partial products separately. This is
4301 // advisable on subtargets where the accumulator of MAD_64_32 must be placed
4302 // in an even-aligned VGPR.
4303 const bool SeparateOddAlignedProducts = ST.hasFullRate64Ops();
4304
4305 LLT S32 = LLT::scalar(32);
4306 SmallVector<Register, 2> Src0Parts, Src1Parts;
4307 for (unsigned i = 0; i < NumParts; ++i) {
4308 Src0Parts.push_back(MRI.createGenericVirtualRegister(S32));
4309 Src1Parts.push_back(MRI.createGenericVirtualRegister(S32));
4310 }
4311 B.buildUnmerge(Src0Parts, Src0);
4312 B.buildUnmerge(Src1Parts, Src1);
4313
4314 SmallVector<Register, 2> AccumRegs(NumParts);
4315 buildMultiply(Helper, AccumRegs, Src0Parts, Src1Parts, UsePartialMad64_32,
4316 SeparateOddAlignedProducts);
4317
4318 B.buildMergeLikeInstr(DstReg, AccumRegs);
4319 MI.eraseFromParent();
4320 return true;
4321}
4322
4323// Legalize ctlz/cttz to ffbh/ffbl instead of the default legalization to
4324// ctlz/cttz_zero_undef. This allows us to fix up the result for the zero input
4325// case with a single min instruction instead of a compare+select.
4328 MachineIRBuilder &B) const {
4329 Register Dst = MI.getOperand(0).getReg();
4330 Register Src = MI.getOperand(1).getReg();
4331 LLT DstTy = MRI.getType(Dst);
4332 LLT SrcTy = MRI.getType(Src);
4333
4334 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_CTLZ
4335 ? AMDGPU::G_AMDGPU_FFBH_U32
4336 : AMDGPU::G_AMDGPU_FFBL_B32;
4337 auto Tmp = B.buildInstr(NewOpc, {DstTy}, {Src});
4338 B.buildUMin(Dst, Tmp, B.buildConstant(DstTy, SrcTy.getSizeInBits()));
4339
4340 MI.eraseFromParent();
4341 return true;
4342}
4343
4346 MachineIRBuilder &B) const {
4347 Register Dst = MI.getOperand(0).getReg();
4348 Register Src = MI.getOperand(1).getReg();
4349 LLT SrcTy = MRI.getType(Src);
4350 TypeSize NumBits = SrcTy.getSizeInBits();
4351
4352 assert(NumBits < 32u);
4353
4354 auto ShiftAmt = B.buildConstant(S32, 32u - NumBits);
4355 auto Extend = B.buildAnyExt(S32, {Src}).getReg(0u);
4356 auto Shift = B.buildShl(S32, Extend, ShiftAmt);
4357 auto Ctlz = B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {S32}, {Shift});
4358 B.buildTrunc(Dst, Ctlz);
4359 MI.eraseFromParent();
4360 return true;
4361}
4362
4363// Check that this is a G_XOR x, -1
4364static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI) {
4365 if (MI.getOpcode() != TargetOpcode::G_XOR)
4366 return false;
4367 auto ConstVal = getIConstantVRegSExtVal(MI.getOperand(2).getReg(), MRI);
4368 return ConstVal == -1;
4369}
4370
4371// Return the use branch instruction, otherwise null if the usage is invalid.
4372static MachineInstr *
4374 MachineBasicBlock *&UncondBrTarget, bool &Negated) {
4375 Register CondDef = MI.getOperand(0).getReg();
4376 if (!MRI.hasOneNonDBGUse(CondDef))
4377 return nullptr;
4378
4379 MachineBasicBlock *Parent = MI.getParent();
4380 MachineInstr *UseMI = &*MRI.use_instr_nodbg_begin(CondDef);
4381
4382 if (isNot(MRI, *UseMI)) {
4383 Register NegatedCond = UseMI->getOperand(0).getReg();
4384 if (!MRI.hasOneNonDBGUse(NegatedCond))
4385 return nullptr;
4386
4387 // We're deleting the def of this value, so we need to remove it.
4388 eraseInstr(*UseMI, MRI);
4389
4390 UseMI = &*MRI.use_instr_nodbg_begin(NegatedCond);
4391 Negated = true;
4392 }
4393
4394 if (UseMI->getParent() != Parent || UseMI->getOpcode() != AMDGPU::G_BRCOND)
4395 return nullptr;
4396
4397 // Make sure the cond br is followed by a G_BR, or is the last instruction.
4398 MachineBasicBlock::iterator Next = std::next(UseMI->getIterator());
4399 if (Next == Parent->end()) {
4400 MachineFunction::iterator NextMBB = std::next(Parent->getIterator());
4401 if (NextMBB == Parent->getParent()->end()) // Illegal intrinsic use.
4402 return nullptr;
4403 UncondBrTarget = &*NextMBB;
4404 } else {
4405 if (Next->getOpcode() != AMDGPU::G_BR)
4406 return nullptr;
4407 Br = &*Next;
4408 UncondBrTarget = Br->getOperand(0).getMBB();
4409 }
4410
4411 return UseMI;
4412}
4413
4416 const ArgDescriptor *Arg,
4417 const TargetRegisterClass *ArgRC,
4418 LLT ArgTy) const {
4419 MCRegister SrcReg = Arg->getRegister();
4420 assert(SrcReg.isPhysical() && "Physical register expected");
4421 assert(DstReg.isVirtual() && "Virtual register expected");
4422
4423 Register LiveIn = getFunctionLiveInPhysReg(B.getMF(), B.getTII(), SrcReg,
4424 *ArgRC, B.getDebugLoc(), ArgTy);
4425 if (Arg->isMasked()) {
4426 // TODO: Should we try to emit this once in the entry block?
4427 const LLT S32 = LLT::scalar(32);
4428 const unsigned Mask = Arg->getMask();
4429 const unsigned Shift = llvm::countr_zero<unsigned>(Mask);
4430
4431 Register AndMaskSrc = LiveIn;
4432
4433 // TODO: Avoid clearing the high bits if we know workitem id y/z are always
4434 // 0.
4435 if (Shift != 0) {
4436 auto ShiftAmt = B.buildConstant(S32, Shift);
4437 AndMaskSrc = B.buildLShr(S32, LiveIn, ShiftAmt).getReg(0);
4438 }
4439
4440 B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(S32, Mask >> Shift));
4441 } else {
4442 B.buildCopy(DstReg, LiveIn);
4443 }
4444}
4445
4450 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
4451 Register DstReg = MI.getOperand(0).getReg();
4452 if (!ST.hasClusters()) {
4453 if (!loadInputValue(DstReg, B, WorkGroupIdPV))
4454 return false;
4455 MI.eraseFromParent();
4456 return true;
4457 }
4458
4459 // Clusters are supported. Return the global position in the grid. If clusters
4460 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
4461
4462 // WorkGroupIdXYZ = ClusterId == 0 ?
4463 // ClusterIdXYZ :
4464 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
4465 MachineRegisterInfo &MRI = *B.getMRI();
4466 const LLT S32 = LLT::scalar(32);
4467 Register ClusterIdXYZ = MRI.createGenericVirtualRegister(S32);
4468 Register ClusterMaxIdXYZ = MRI.createGenericVirtualRegister(S32);
4469 Register ClusterWorkGroupIdXYZ = MRI.createGenericVirtualRegister(S32);
4470 if (!loadInputValue(ClusterIdXYZ, B, WorkGroupIdPV) ||
4471 !loadInputValue(ClusterWorkGroupIdXYZ, B, ClusterWorkGroupIdPV) ||
4472 !loadInputValue(ClusterMaxIdXYZ, B, ClusterMaxIdPV))
4473 return false;
4474
4475 auto One = B.buildConstant(S32, 1);
4476 auto ClusterSizeXYZ = B.buildAdd(S32, ClusterMaxIdXYZ, One);
4477 auto GlobalIdXYZ = B.buildAdd(S32, ClusterWorkGroupIdXYZ,
4478 B.buildMul(S32, ClusterIdXYZ, ClusterSizeXYZ));
4479
4480 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
4481
4482 switch (MFI->getClusterDims().getKind()) {
4485 B.buildCopy(DstReg, GlobalIdXYZ);
4486 MI.eraseFromParent();
4487 return true;
4488 }
4490 B.buildCopy(DstReg, ClusterIdXYZ);
4491 MI.eraseFromParent();
4492 return true;
4493 }
4495 using namespace AMDGPU::Hwreg;
4496 unsigned ClusterIdField = HwregEncoding::encode(ID_IB_STS2, 6, 4);
4497 Register ClusterId = MRI.createGenericVirtualRegister(S32);
4498 MRI.setRegClass(ClusterId, &AMDGPU::SReg_32RegClass);
4499 B.buildInstr(AMDGPU::S_GETREG_B32_const)
4500 .addDef(ClusterId)
4501 .addImm(ClusterIdField);
4502 auto Zero = B.buildConstant(S32, 0);
4503 auto NoClusters =
4504 B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), ClusterId, Zero);
4505 B.buildSelect(DstReg, NoClusters, ClusterIdXYZ, GlobalIdXYZ);
4506 MI.eraseFromParent();
4507 return true;
4508 }
4509 }
4510
4511 llvm_unreachable("nothing should reach here");
4512}
4513
4515 Register DstReg, MachineIRBuilder &B,
4517 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
4518 const ArgDescriptor *Arg = nullptr;
4519 const TargetRegisterClass *ArgRC;
4520 LLT ArgTy;
4521
4522 CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
4523 const ArgDescriptor WorkGroupIDX =
4524 ArgDescriptor::createRegister(AMDGPU::TTMP9);
4525 // If GridZ is not programmed in an entry function then the hardware will set
4526 // it to all zeros, so there is no need to mask the GridY value in the low
4527 // order bits.
4528 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
4529 AMDGPU::TTMP7,
4530 AMDGPU::isEntryFunctionCC(CC) && !MFI->hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
4531 const ArgDescriptor WorkGroupIDZ =
4532 ArgDescriptor::createRegister(AMDGPU::TTMP7, 0xFFFF0000u);
4533 const ArgDescriptor ClusterWorkGroupIDX =
4534 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000000Fu);
4535 const ArgDescriptor ClusterWorkGroupIDY =
4536 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000000F0u);
4537 const ArgDescriptor ClusterWorkGroupIDZ =
4538 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00000F00u);
4539 const ArgDescriptor ClusterWorkGroupMaxIDX =
4540 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000F000u);
4541 const ArgDescriptor ClusterWorkGroupMaxIDY =
4542 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000F0000u);
4543 const ArgDescriptor ClusterWorkGroupMaxIDZ =
4544 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00F00000u);
4545 const ArgDescriptor ClusterWorkGroupMaxFlatID =
4546 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0F000000u);
4547
4548 auto LoadConstant = [&](unsigned N) {
4549 B.buildConstant(DstReg, N);
4550 return true;
4551 };
4552
4553 if (ST.hasArchitectedSGPRs() &&
4555 AMDGPU::ClusterDimsAttr ClusterDims = MFI->getClusterDims();
4556 bool HasFixedDims = ClusterDims.isFixedDims();
4557
4558 switch (ArgType) {
4560 Arg = &WorkGroupIDX;
4561 ArgRC = &AMDGPU::SReg_32RegClass;
4562 ArgTy = LLT::scalar(32);
4563 break;
4565 Arg = &WorkGroupIDY;
4566 ArgRC = &AMDGPU::SReg_32RegClass;
4567 ArgTy = LLT::scalar(32);
4568 break;
4570 Arg = &WorkGroupIDZ;
4571 ArgRC = &AMDGPU::SReg_32RegClass;
4572 ArgTy = LLT::scalar(32);
4573 break;
4575 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
4576 return LoadConstant(0);
4577 Arg = &ClusterWorkGroupIDX;
4578 ArgRC = &AMDGPU::SReg_32RegClass;
4579 ArgTy = LLT::scalar(32);
4580 break;
4582 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
4583 return LoadConstant(0);
4584 Arg = &ClusterWorkGroupIDY;
4585 ArgRC = &AMDGPU::SReg_32RegClass;
4586 ArgTy = LLT::scalar(32);
4587 break;
4589 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
4590 return LoadConstant(0);
4591 Arg = &ClusterWorkGroupIDZ;
4592 ArgRC = &AMDGPU::SReg_32RegClass;
4593 ArgTy = LLT::scalar(32);
4594 break;
4596 if (HasFixedDims)
4597 return LoadConstant(ClusterDims.getDims()[0] - 1);
4598 Arg = &ClusterWorkGroupMaxIDX;
4599 ArgRC = &AMDGPU::SReg_32RegClass;
4600 ArgTy = LLT::scalar(32);
4601 break;
4603 if (HasFixedDims)
4604 return LoadConstant(ClusterDims.getDims()[1] - 1);
4605 Arg = &ClusterWorkGroupMaxIDY;
4606 ArgRC = &AMDGPU::SReg_32RegClass;
4607 ArgTy = LLT::scalar(32);
4608 break;
4610 if (HasFixedDims)
4611 return LoadConstant(ClusterDims.getDims()[2] - 1);
4612 Arg = &ClusterWorkGroupMaxIDZ;
4613 ArgRC = &AMDGPU::SReg_32RegClass;
4614 ArgTy = LLT::scalar(32);
4615 break;
4617 Arg = &ClusterWorkGroupMaxFlatID;
4618 ArgRC = &AMDGPU::SReg_32RegClass;
4619 ArgTy = LLT::scalar(32);
4620 break;
4621 default:
4622 break;
4623 }
4624 }
4625
4626 if (!Arg)
4627 std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);
4628
4629 if (!Arg) {
4631 // The intrinsic may appear when we have a 0 sized kernarg segment, in
4632 // which case the pointer argument may be missing and we use null.
4633 return LoadConstant(0);
4634 }
4635
4636 // It's undefined behavior if a function marked with the amdgpu-no-*
4637 // attributes uses the corresponding intrinsic.
4638 B.buildUndef(DstReg);
4639 return true;
4640 }
4641
4642 if (!Arg->isRegister() || !Arg->getRegister().isValid())
4643 return false; // TODO: Handle these
4644 buildLoadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
4645 return true;
4646}
4647
4651 if (!loadInputValue(MI.getOperand(0).getReg(), B, ArgType))
4652 return false;
4653
4654 MI.eraseFromParent();
4655 return true;
4656}
4657
4659 int64_t C) {
4660 B.buildConstant(MI.getOperand(0).getReg(), C);
4661 MI.eraseFromParent();
4662 return true;
4663}
4664
4667 unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
4668 unsigned MaxID = ST.getMaxWorkitemID(B.getMF().getFunction(), Dim);
4669 if (MaxID == 0)
4670 return replaceWithConstant(B, MI, 0);
4671
4672 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
4673 const ArgDescriptor *Arg;
4674 const TargetRegisterClass *ArgRC;
4675 LLT ArgTy;
4676 std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);
4677
4678 Register DstReg = MI.getOperand(0).getReg();
4679 if (!Arg) {
4680 // It's undefined behavior if a function marked with the amdgpu-no-*
4681 // attributes uses the corresponding intrinsic.
4682 B.buildUndef(DstReg);
4683 MI.eraseFromParent();
4684 return true;
4685 }
4686
4687 if (Arg->isMasked()) {
4688 // Don't bother inserting AssertZext for packed IDs since we're emitting the
4689 // masking operations anyway.
4690 //
4691 // TODO: We could assert the top bit is 0 for the source copy.
4692 if (!loadInputValue(DstReg, B, ArgType))
4693 return false;
4694 } else {
4695 Register TmpReg = MRI.createGenericVirtualRegister(LLT::scalar(32));
4696 if (!loadInputValue(TmpReg, B, ArgType))
4697 return false;
4698 B.buildAssertZExt(DstReg, TmpReg, llvm::bit_width(MaxID));
4699 }
4700
4701 MI.eraseFromParent();
4702 return true;
4703}
4704
4706 int64_t Offset) const {
4708 Register KernArgReg = B.getMRI()->createGenericVirtualRegister(PtrTy);
4709
4710 // TODO: If we passed in the base kernel offset we could have a better
4711 // alignment than 4, but we don't really need it.
4712 if (!loadInputValue(KernArgReg, B,
4714 llvm_unreachable("failed to find kernarg segment ptr");
4715
4716 auto COffset = B.buildConstant(LLT::scalar(64), Offset);
4717 return B.buildObjectPtrOffset(PtrTy, KernArgReg, COffset).getReg(0);
4718}
4719
4720/// Legalize a value that's loaded from kernel arguments. This is only used by
4721/// legacy intrinsics.
4725 Align Alignment) const {
4726 Register DstReg = MI.getOperand(0).getReg();
4727
4728 assert(B.getMRI()->getType(DstReg) == LLT::scalar(32) &&
4729 "unexpected kernarg parameter type");
4730
4733 B.buildLoad(DstReg, Ptr, PtrInfo, Align(4),
4736 MI.eraseFromParent();
4737 return true;
4738}
4739
4742 MachineIRBuilder &B) const {
4743 Register Dst = MI.getOperand(0).getReg();
4744 LLT DstTy = MRI.getType(Dst);
4745 LLT S16 = LLT::scalar(16);
4746 LLT S32 = LLT::scalar(32);
4747 LLT S64 = LLT::scalar(64);
4748
4749 if (DstTy == S16)
4750 return legalizeFDIV16(MI, MRI, B);
4751 if (DstTy == S32)
4752 return legalizeFDIV32(MI, MRI, B);
4753 if (DstTy == S64)
4754 return legalizeFDIV64(MI, MRI, B);
4755
4756 return false;
4757}
4758
4760 Register DstDivReg,
4761 Register DstRemReg,
4762 Register X,
4763 Register Y) const {
4764 const LLT S1 = LLT::scalar(1);
4765 const LLT S32 = LLT::scalar(32);
4766
4767 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
4768 // algorithm used here.
4769
4770 // Initial estimate of inv(y).
4771 auto FloatY = B.buildUITOFP(S32, Y);
4772 auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {FloatY});
4773 auto Scale = B.buildFConstant(S32, llvm::bit_cast<float>(0x4f7ffffe));
4774 auto ScaledY = B.buildFMul(S32, RcpIFlag, Scale);
4775 auto Z = B.buildFPTOUI(S32, ScaledY);
4776
4777 // One round of UNR.
4778 auto NegY = B.buildSub(S32, B.buildConstant(S32, 0), Y);
4779 auto NegYZ = B.buildMul(S32, NegY, Z);
4780 Z = B.buildAdd(S32, Z, B.buildUMulH(S32, Z, NegYZ));
4781
4782 // Quotient/remainder estimate.
4783 auto Q = B.buildUMulH(S32, X, Z);
4784 auto R = B.buildSub(S32, X, B.buildMul(S32, Q, Y));
4785
4786 // First quotient/remainder refinement.
4787 auto One = B.buildConstant(S32, 1);
4788 auto Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
4789 if (DstDivReg)
4790 Q = B.buildSelect(S32, Cond, B.buildAdd(S32, Q, One), Q);
4791 R = B.buildSelect(S32, Cond, B.buildSub(S32, R, Y), R);
4792
4793 // Second quotient/remainder refinement.
4794 Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
4795 if (DstDivReg)
4796 B.buildSelect(DstDivReg, Cond, B.buildAdd(S32, Q, One), Q);
4797
4798 if (DstRemReg)
4799 B.buildSelect(DstRemReg, Cond, B.buildSub(S32, R, Y), R);
4800}
4801
4802// Build integer reciprocal sequence around V_RCP_IFLAG_F32
4803//
4804// Return lo, hi of result
4805//
4806// %cvt.lo = G_UITOFP Val.lo
4807// %cvt.hi = G_UITOFP Val.hi
4808// %mad = G_FMAD %cvt.hi, 2**32, %cvt.lo
4809// %rcp = G_AMDGPU_RCP_IFLAG %mad
4810// %mul1 = G_FMUL %rcp, 0x5f7ffffc
4811// %mul2 = G_FMUL %mul1, 2**(-32)
4812// %trunc = G_INTRINSIC_TRUNC %mul2
4813// %mad2 = G_FMAD %trunc, -(2**32), %mul1
4814// return {G_FPTOUI %mad2, G_FPTOUI %trunc}
4815static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
4816 Register Val) {
4817 const LLT S32 = LLT::scalar(32);
4818 auto Unmerge = B.buildUnmerge(S32, Val);
4819
4820 auto CvtLo = B.buildUITOFP(S32, Unmerge.getReg(0));
4821 auto CvtHi = B.buildUITOFP(S32, Unmerge.getReg(1));
4822
4823 auto Mad = B.buildFMAD(
4824 S32, CvtHi, // 2**32
4825 B.buildFConstant(S32, llvm::bit_cast<float>(0x4f800000)), CvtLo);
4826
4827 auto Rcp = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {Mad});
4828 auto Mul1 = B.buildFMul(
4829 S32, Rcp, B.buildFConstant(S32, llvm::bit_cast<float>(0x5f7ffffc)));
4830
4831 // 2**(-32)
4832 auto Mul2 = B.buildFMul(
4833 S32, Mul1, B.buildFConstant(S32, llvm::bit_cast<float>(0x2f800000)));
4834 auto Trunc = B.buildIntrinsicTrunc(S32, Mul2);
4835
4836 // -(2**32)
4837 auto Mad2 = B.buildFMAD(
4838 S32, Trunc, B.buildFConstant(S32, llvm::bit_cast<float>(0xcf800000)),
4839 Mul1);
4840
4841 auto ResultLo = B.buildFPTOUI(S32, Mad2);
4842 auto ResultHi = B.buildFPTOUI(S32, Trunc);
4843
4844 return {ResultLo.getReg(0), ResultHi.getReg(0)};
4845}
4846
4848 Register DstDivReg,
4849 Register DstRemReg,
4850 Register Numer,
4851 Register Denom) const {
4852 const LLT S32 = LLT::scalar(32);
4853 const LLT S64 = LLT::scalar(64);
4854 const LLT S1 = LLT::scalar(1);
4855 Register RcpLo, RcpHi;
4856
4857 std::tie(RcpLo, RcpHi) = emitReciprocalU64(B, Denom);
4858
4859 auto Rcp = B.buildMergeLikeInstr(S64, {RcpLo, RcpHi});
4860
4861 auto Zero64 = B.buildConstant(S64, 0);
4862 auto NegDenom = B.buildSub(S64, Zero64, Denom);
4863
4864 auto MulLo1 = B.buildMul(S64, NegDenom, Rcp);
4865 auto MulHi1 = B.buildUMulH(S64, Rcp, MulLo1);
4866
4867 auto UnmergeMulHi1 = B.buildUnmerge(S32, MulHi1);
4868 Register MulHi1_Lo = UnmergeMulHi1.getReg(0);
4869 Register MulHi1_Hi = UnmergeMulHi1.getReg(1);
4870
4871 auto Add1_Lo = B.buildUAddo(S32, S1, RcpLo, MulHi1_Lo);
4872 auto Add1_Hi = B.buildUAdde(S32, S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1));
4873 auto Add1 = B.buildMergeLikeInstr(S64, {Add1_Lo, Add1_Hi});
4874
4875 auto MulLo2 = B.buildMul(S64, NegDenom, Add1);
4876 auto MulHi2 = B.buildUMulH(S64, Add1, MulLo2);
4877 auto UnmergeMulHi2 = B.buildUnmerge(S32, MulHi2);
4878 Register MulHi2_Lo = UnmergeMulHi2.getReg(0);
4879 Register MulHi2_Hi = UnmergeMulHi2.getReg(1);
4880
4881 auto Zero32 = B.buildConstant(S32, 0);
4882 auto Add2_Lo = B.buildUAddo(S32, S1, Add1_Lo, MulHi2_Lo);
4883 auto Add2_Hi = B.buildUAdde(S32, S1, Add1_Hi, MulHi2_Hi, Add2_Lo.getReg(1));
4884 auto Add2 = B.buildMergeLikeInstr(S64, {Add2_Lo, Add2_Hi});
4885
4886 auto UnmergeNumer = B.buildUnmerge(S32, Numer);
4887 Register NumerLo = UnmergeNumer.getReg(0);
4888 Register NumerHi = UnmergeNumer.getReg(1);
4889
4890 auto MulHi3 = B.buildUMulH(S64, Numer, Add2);
4891 auto Mul3 = B.buildMul(S64, Denom, MulHi3);
4892 auto UnmergeMul3 = B.buildUnmerge(S32, Mul3);
4893 Register Mul3_Lo = UnmergeMul3.getReg(0);
4894 Register Mul3_Hi = UnmergeMul3.getReg(1);
4895 auto Sub1_Lo = B.buildUSubo(S32, S1, NumerLo, Mul3_Lo);
4896 auto Sub1_Hi = B.buildUSube(S32, S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1));
4897 auto Sub1_Mi = B.buildSub(S32, NumerHi, Mul3_Hi);
4898 auto Sub1 = B.buildMergeLikeInstr(S64, {Sub1_Lo, Sub1_Hi});
4899
4900 auto UnmergeDenom = B.buildUnmerge(S32, Denom);
4901 Register DenomLo = UnmergeDenom.getReg(0);
4902 Register DenomHi = UnmergeDenom.getReg(1);
4903
4904 auto CmpHi = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Hi, DenomHi);
4905 auto C1 = B.buildSExt(S32, CmpHi);
4906
4907 auto CmpLo = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Lo, DenomLo);
4908 auto C2 = B.buildSExt(S32, CmpLo);
4909
4910 auto CmpEq = B.buildICmp(CmpInst::ICMP_EQ, S1, Sub1_Hi, DenomHi);
4911 auto C3 = B.buildSelect(S32, CmpEq, C2, C1);
4912
4913 // TODO: Here and below portions of the code can be enclosed into if/endif.
4914 // Currently control flow is unconditional and we have 4 selects after
4915 // potential endif to substitute PHIs.
4916
4917 // if C3 != 0 ...
4918 auto Sub2_Lo = B.buildUSubo(S32, S1, Sub1_Lo, DenomLo);
4919 auto Sub2_Mi = B.buildUSube(S32, S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1));
4920 auto Sub2_Hi = B.buildUSube(S32, S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1));
4921 auto Sub2 = B.buildMergeLikeInstr(S64, {Sub2_Lo, Sub2_Hi});
4922
4923 auto One64 = B.buildConstant(S64, 1);
4924 auto Add3 = B.buildAdd(S64, MulHi3, One64);
4925
4926 auto C4 =
4927 B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Hi, DenomHi));
4928 auto C5 =
4929 B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Lo, DenomLo));
4930 auto C6 = B.buildSelect(
4931 S32, B.buildICmp(CmpInst::ICMP_EQ, S1, Sub2_Hi, DenomHi), C5, C4);
4932
4933 // if (C6 != 0)
4934 auto Add4 = B.buildAdd(S64, Add3, One64);
4935 auto Sub3_Lo = B.buildUSubo(S32, S1, Sub2_Lo, DenomLo);
4936
4937 auto Sub3_Mi = B.buildUSube(S32, S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1));
4938 auto Sub3_Hi = B.buildUSube(S32, S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1));
4939 auto Sub3 = B.buildMergeLikeInstr(S64, {Sub3_Lo, Sub3_Hi});
4940
4941 // endif C6
4942 // endif C3
4943
4944 if (DstDivReg) {
4945 auto Sel1 = B.buildSelect(
4946 S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
4947 B.buildSelect(DstDivReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
4948 Sel1, MulHi3);
4949 }
4950
4951 if (DstRemReg) {
4952 auto Sel2 = B.buildSelect(
4953 S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
4954 B.buildSelect(DstRemReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
4955 Sel2, Sub1);
4956 }
4957}
4958
4961 MachineIRBuilder &B) const {
4962 Register DstDivReg, DstRemReg;
4963 switch (MI.getOpcode()) {
4964 default:
4965 llvm_unreachable("Unexpected opcode!");
4966 case AMDGPU::G_UDIV: {
4967 DstDivReg = MI.getOperand(0).getReg();
4968 break;
4969 }
4970 case AMDGPU::G_UREM: {
4971 DstRemReg = MI.getOperand(0).getReg();
4972 break;
4973 }
4974 case AMDGPU::G_UDIVREM: {
4975 DstDivReg = MI.getOperand(0).getReg();
4976 DstRemReg = MI.getOperand(1).getReg();
4977 break;
4978 }
4979 }
4980
4981 const LLT S64 = LLT::scalar(64);
4982 const LLT S32 = LLT::scalar(32);
4983 const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
4984 Register Num = MI.getOperand(FirstSrcOpIdx).getReg();
4985 Register Den = MI.getOperand(FirstSrcOpIdx + 1).getReg();
4986 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
4987
4988 if (Ty == S32)
4989 legalizeUnsignedDIV_REM32Impl(B, DstDivReg, DstRemReg, Num, Den);
4990 else if (Ty == S64)
4991 legalizeUnsignedDIV_REM64Impl(B, DstDivReg, DstRemReg, Num, Den);
4992 else
4993 return false;
4994
4995 MI.eraseFromParent();
4996 return true;
4997}
4998
5001 MachineIRBuilder &B) const {
5002 const LLT S64 = LLT::scalar(64);
5003 const LLT S32 = LLT::scalar(32);
5004
5005 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
5006 if (Ty != S32 && Ty != S64)
5007 return false;
5008
5009 const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
5010 Register LHS = MI.getOperand(FirstSrcOpIdx).getReg();
5011 Register RHS = MI.getOperand(FirstSrcOpIdx + 1).getReg();
5012
5013 auto SignBitOffset = B.buildConstant(S32, Ty.getSizeInBits() - 1);
5014 auto LHSign = B.buildAShr(Ty, LHS, SignBitOffset);
5015 auto RHSign = B.buildAShr(Ty, RHS, SignBitOffset);
5016
5017 LHS = B.buildAdd(Ty, LHS, LHSign).getReg(0);
5018 RHS = B.buildAdd(Ty, RHS, RHSign).getReg(0);
5019
5020 LHS = B.buildXor(Ty, LHS, LHSign).getReg(0);
5021 RHS = B.buildXor(Ty, RHS, RHSign).getReg(0);
5022
5023 Register DstDivReg, DstRemReg, TmpDivReg, TmpRemReg;
5024 switch (MI.getOpcode()) {
5025 default:
5026 llvm_unreachable("Unexpected opcode!");
5027 case AMDGPU::G_SDIV: {
5028 DstDivReg = MI.getOperand(0).getReg();
5029 TmpDivReg = MRI.createGenericVirtualRegister(Ty);
5030 break;
5031 }
5032 case AMDGPU::G_SREM: {
5033 DstRemReg = MI.getOperand(0).getReg();
5034 TmpRemReg = MRI.createGenericVirtualRegister(Ty);
5035 break;
5036 }
5037 case AMDGPU::G_SDIVREM: {
5038 DstDivReg = MI.getOperand(0).getReg();
5039 DstRemReg = MI.getOperand(1).getReg();
5040 TmpDivReg = MRI.createGenericVirtualRegister(Ty);
5041 TmpRemReg = MRI.createGenericVirtualRegister(Ty);
5042 break;
5043 }
5044 }
5045
5046 if (Ty == S32)
5047 legalizeUnsignedDIV_REM32Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
5048 else
5049 legalizeUnsignedDIV_REM64Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
5050
5051 if (DstDivReg) {
5052 auto Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
5053 auto SignXor = B.buildXor(Ty, TmpDivReg, Sign).getReg(0);
5054 B.buildSub(DstDivReg, SignXor, Sign);
5055 }
5056
5057 if (DstRemReg) {
5058 auto Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
5059 auto SignXor = B.buildXor(Ty, TmpRemReg, Sign).getReg(0);
5060 B.buildSub(DstRemReg, SignXor, Sign);
5061 }
5062
5063 MI.eraseFromParent();
5064 return true;
5065}
5066
5069 MachineIRBuilder &B) const {
5070 Register Res = MI.getOperand(0).getReg();
5071 Register LHS = MI.getOperand(1).getReg();
5072 Register RHS = MI.getOperand(2).getReg();
5073 uint16_t Flags = MI.getFlags();
5074 LLT ResTy = MRI.getType(Res);
5075
5076 bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);
5077
5078 if (const auto *CLHS = getConstantFPVRegVal(LHS, MRI)) {
5079 if (!AllowInaccurateRcp && ResTy != LLT::scalar(16))
5080 return false;
5081
5082 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
5083 // the CI documentation has a worst case error of 1 ulp.
5084 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
5085 // use it as long as we aren't trying to use denormals.
5086 //
5087 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
5088
5089 // 1 / x -> RCP(x)
5090 if (CLHS->isExactlyValue(1.0)) {
5091 B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
5092 .addUse(RHS)
5093 .setMIFlags(Flags);
5094
5095 MI.eraseFromParent();
5096 return true;
5097 }
5098
5099 // -1 / x -> RCP( FNEG(x) )
5100 if (CLHS->isExactlyValue(-1.0)) {
5101 auto FNeg = B.buildFNeg(ResTy, RHS, Flags);
5102 B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
5103 .addUse(FNeg.getReg(0))
5104 .setMIFlags(Flags);
5105
5106 MI.eraseFromParent();
5107 return true;
5108 }
5109 }
5110
5111 // For f16 require afn or arcp.
5112 // For f32 require afn.
5113 if (!AllowInaccurateRcp && (ResTy != LLT::scalar(16) ||
5114 !MI.getFlag(MachineInstr::FmArcp)))
5115 return false;
5116
5117 // x / y -> x * (1.0 / y)
5118 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5119 .addUse(RHS)
5120 .setMIFlags(Flags);
5121 B.buildFMul(Res, LHS, RCP, Flags);
5122
5123 MI.eraseFromParent();
5124 return true;
5125}
5126
5129 MachineIRBuilder &B) const {
5130 Register Res = MI.getOperand(0).getReg();
5131 Register X = MI.getOperand(1).getReg();
5132 Register Y = MI.getOperand(2).getReg();
5133 uint16_t Flags = MI.getFlags();
5134 LLT ResTy = MRI.getType(Res);
5135
5136 bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);
5137
5138 if (!AllowInaccurateRcp)
5139 return false;
5140
5141 auto NegY = B.buildFNeg(ResTy, Y);
5142 auto One = B.buildFConstant(ResTy, 1.0);
5143
5144 auto R = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5145 .addUse(Y)
5146 .setMIFlags(Flags);
5147
5148 auto Tmp0 = B.buildFMA(ResTy, NegY, R, One);
5149 R = B.buildFMA(ResTy, Tmp0, R, R);
5150
5151 auto Tmp1 = B.buildFMA(ResTy, NegY, R, One);
5152 R = B.buildFMA(ResTy, Tmp1, R, R);
5153
5154 auto Ret = B.buildFMul(ResTy, X, R);
5155 auto Tmp2 = B.buildFMA(ResTy, NegY, Ret, X);
5156
5157 B.buildFMA(Res, Tmp2, R, Ret);
5158 MI.eraseFromParent();
5159 return true;
5160}
5161
5164 MachineIRBuilder &B) const {
5166 return true;
5167
5168 Register Res = MI.getOperand(0).getReg();
5169 Register LHS = MI.getOperand(1).getReg();
5170 Register RHS = MI.getOperand(2).getReg();
5171
5172 uint16_t Flags = MI.getFlags();
5173
5174 LLT S16 = LLT::scalar(16);
5175 LLT S32 = LLT::scalar(32);
5176
5177 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
5178 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
5179 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
5180 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
5181 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
5182 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
5183 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
5184 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
5185 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
5186 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
5187 // q16.u = opx(V_CVT_F16_F32, q32.u);
5188 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
5189
5190 auto LHSExt = B.buildFPExt(S32, LHS, Flags);
5191 auto RHSExt = B.buildFPExt(S32, RHS, Flags);
5192 auto NegRHSExt = B.buildFNeg(S32, RHSExt);
5193 auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32})
5194 .addUse(RHSExt.getReg(0))
5195 .setMIFlags(Flags);
5196 auto Quot = B.buildFMul(S32, LHSExt, Rcp, Flags);
5198 if (ST.hasMadMacF32Insts()) {
5199 Err = B.buildFMAD(S32, NegRHSExt, Quot, LHSExt, Flags);
5200 Quot = B.buildFMAD(S32, Err, Rcp, Quot, Flags);
5201 Err = B.buildFMAD(S32, NegRHSExt, Quot, LHSExt, Flags);
5202 } else {
5203 Err = B.buildFMA(S32, NegRHSExt, Quot, LHSExt, Flags);
5204 Quot = B.buildFMA(S32, Err, Rcp, Quot, Flags);
5205 Err = B.buildFMA(S32, NegRHSExt, Quot, LHSExt, Flags);
5206 }
5207 auto Tmp = B.buildFMul(S32, Err, Rcp, Flags);
5208 Tmp = B.buildAnd(S32, Tmp, B.buildConstant(S32, 0xff800000));
5209 Quot = B.buildFAdd(S32, Tmp, Quot, Flags);
5210 auto RDst = B.buildFPTrunc(S16, Quot, Flags);
5211 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
5212 .addUse(RDst.getReg(0))
5213 .addUse(RHS)
5214 .addUse(LHS)
5215 .setMIFlags(Flags);
5216
5217 MI.eraseFromParent();
5218 return true;
5219}
5220
5221static constexpr unsigned SPDenormModeBitField =
5223
5224// Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions
5225// to enable denorm mode. When 'Enable' is false, disable denorm mode.
5227 const GCNSubtarget &ST,
5229 // Set SP denorm mode to this value.
5230 unsigned SPDenormMode =
5231 Enable ? FP_DENORM_FLUSH_NONE : Mode.fpDenormModeSPValue();
5232
5233 if (ST.hasDenormModeInst()) {
5234 // Preserve default FP64FP16 denorm mode while updating FP32 mode.
5235 uint32_t DPDenormModeDefault = Mode.fpDenormModeDPValue();
5236
5237 uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
5238 B.buildInstr(AMDGPU::S_DENORM_MODE)
5239 .addImm(NewDenormModeValue);
5240
5241 } else {
5242 B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
5243 .addImm(SPDenormMode)
5244 .addImm(SPDenormModeBitField);
5245 }
5246}
5247
5250 MachineIRBuilder &B) const {
5252 return true;
5253
5254 Register Res = MI.getOperand(0).getReg();
5255 Register LHS = MI.getOperand(1).getReg();
5256 Register RHS = MI.getOperand(2).getReg();
5257 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5258 SIModeRegisterDefaults Mode = MFI->getMode();
5259
5260 uint16_t Flags = MI.getFlags();
5261
5262 LLT S32 = LLT::scalar(32);
5263 LLT S1 = LLT::scalar(1);
5264
5265 auto One = B.buildFConstant(S32, 1.0f);
5266
5267 auto DenominatorScaled =
5268 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1})
5269 .addUse(LHS)
5270 .addUse(RHS)
5271 .addImm(0)
5272 .setMIFlags(Flags);
5273 auto NumeratorScaled =
5274 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1})
5275 .addUse(LHS)
5276 .addUse(RHS)
5277 .addImm(1)
5278 .setMIFlags(Flags);
5279
5280 auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32})
5281 .addUse(DenominatorScaled.getReg(0))
5282 .setMIFlags(Flags);
5283 auto NegDivScale0 = B.buildFNeg(S32, DenominatorScaled, Flags);
5284
5285 const bool PreservesDenormals = Mode.FP32Denormals == DenormalMode::getIEEE();
5286 const bool HasDynamicDenormals =
5287 (Mode.FP32Denormals.Input == DenormalMode::Dynamic) ||
5288 (Mode.FP32Denormals.Output == DenormalMode::Dynamic);
5289
5290 Register SavedSPDenormMode;
5291 if (!PreservesDenormals) {
5292 if (HasDynamicDenormals) {
5293 SavedSPDenormMode = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5294 B.buildInstr(AMDGPU::S_GETREG_B32)
5295 .addDef(SavedSPDenormMode)
5296 .addImm(SPDenormModeBitField);
5297 }
5298 toggleSPDenormMode(true, B, ST, Mode);
5299 }
5300
5301 auto Fma0 = B.buildFMA(S32, NegDivScale0, ApproxRcp, One, Flags);
5302 auto Fma1 = B.buildFMA(S32, Fma0, ApproxRcp, ApproxRcp, Flags);
5303 auto Mul = B.buildFMul(S32, NumeratorScaled, Fma1, Flags);
5304 auto Fma2 = B.buildFMA(S32, NegDivScale0, Mul, NumeratorScaled, Flags);
5305 auto Fma3 = B.buildFMA(S32, Fma2, Fma1, Mul, Flags);
5306 auto Fma4 = B.buildFMA(S32, NegDivScale0, Fma3, NumeratorScaled, Flags);
5307
5308 if (!PreservesDenormals) {
5309 if (HasDynamicDenormals) {
5310 assert(SavedSPDenormMode);
5311 B.buildInstr(AMDGPU::S_SETREG_B32)
5312 .addReg(SavedSPDenormMode)
5313 .addImm(SPDenormModeBitField);
5314 } else
5315 toggleSPDenormMode(false, B, ST, Mode);
5316 }
5317
5318 auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S32})
5319 .addUse(Fma4.getReg(0))
5320 .addUse(Fma1.getReg(0))
5321 .addUse(Fma3.getReg(0))
5322 .addUse(NumeratorScaled.getReg(1))
5323 .setMIFlags(Flags);
5324
5325 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
5326 .addUse(Fmas.getReg(0))
5327 .addUse(RHS)
5328 .addUse(LHS)
5329 .setMIFlags(Flags);
5330
5331 MI.eraseFromParent();
5332 return true;
5333}
5334
5337 MachineIRBuilder &B) const {
5339 return true;
5340
5341 Register Res = MI.getOperand(0).getReg();
5342 Register LHS = MI.getOperand(1).getReg();
5343 Register RHS = MI.getOperand(2).getReg();
5344
5345 uint16_t Flags = MI.getFlags();
5346
5347 LLT S64 = LLT::scalar(64);
5348 LLT S1 = LLT::scalar(1);
5349
5350 auto One = B.buildFConstant(S64, 1.0);
5351
5352 auto DivScale0 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1})
5353 .addUse(LHS)
5354 .addUse(RHS)
5355 .addImm(0)
5356 .setMIFlags(Flags);
5357
5358 auto NegDivScale0 = B.buildFNeg(S64, DivScale0.getReg(0), Flags);
5359
5360 auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S64})
5361 .addUse(DivScale0.getReg(0))
5362 .setMIFlags(Flags);
5363
5364 auto Fma0 = B.buildFMA(S64, NegDivScale0, Rcp, One, Flags);
5365 auto Fma1 = B.buildFMA(S64, Rcp, Fma0, Rcp, Flags);
5366 auto Fma2 = B.buildFMA(S64, NegDivScale0, Fma1, One, Flags);
5367
5368 auto DivScale1 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1})
5369 .addUse(LHS)
5370 .addUse(RHS)
5371 .addImm(1)
5372 .setMIFlags(Flags);
5373
5374 auto Fma3 = B.buildFMA(S64, Fma1, Fma2, Fma1, Flags);
5375 auto Mul = B.buildFMul(S64, DivScale1.getReg(0), Fma3, Flags);
5376 auto Fma4 = B.buildFMA(S64, NegDivScale0, Mul, DivScale1.getReg(0), Flags);
5377
5378 Register Scale;
5379 if (!ST.hasUsableDivScaleConditionOutput()) {
5380 // Workaround a hardware bug on SI where the condition output from div_scale
5381 // is not usable.
5382
5383 LLT S32 = LLT::scalar(32);
5384
5385 auto NumUnmerge = B.buildUnmerge(S32, LHS);
5386 auto DenUnmerge = B.buildUnmerge(S32, RHS);
5387 auto Scale0Unmerge = B.buildUnmerge(S32, DivScale0);
5388 auto Scale1Unmerge = B.buildUnmerge(S32, DivScale1);
5389
5390 auto CmpNum = B.buildICmp(ICmpInst::ICMP_EQ, S1, NumUnmerge.getReg(1),
5391 Scale1Unmerge.getReg(1));
5392 auto CmpDen = B.buildICmp(ICmpInst::ICMP_EQ, S1, DenUnmerge.getReg(1),
5393 Scale0Unmerge.getReg(1));
5394 Scale = B.buildXor(S1, CmpNum, CmpDen).getReg(0);
5395 } else {
5396 Scale = DivScale1.getReg(1);
5397 }
5398
5399 auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S64})
5400 .addUse(Fma4.getReg(0))
5401 .addUse(Fma3.getReg(0))
5402 .addUse(Mul.getReg(0))
5403 .addUse(Scale)
5404 .setMIFlags(Flags);
5405
5406 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, ArrayRef(Res))
5407 .addUse(Fmas.getReg(0))
5408 .addUse(RHS)
5409 .addUse(LHS)
5410 .setMIFlags(Flags);
5411
5412 MI.eraseFromParent();
5413 return true;
5414}
5415
5418 MachineIRBuilder &B) const {
5419 Register Res0 = MI.getOperand(0).getReg();
5420 Register Res1 = MI.getOperand(1).getReg();
5421 Register Val = MI.getOperand(2).getReg();
5422 uint16_t Flags = MI.getFlags();
5423
5424 LLT Ty = MRI.getType(Res0);
5425 LLT InstrExpTy = Ty == LLT::scalar(16) ? LLT::scalar(16) : LLT::scalar(32);
5426
5427 auto Mant = B.buildIntrinsic(Intrinsic::amdgcn_frexp_mant, {Ty})
5428 .addUse(Val)
5429 .setMIFlags(Flags);
5430 auto Exp = B.buildIntrinsic(Intrinsic::amdgcn_frexp_exp, {InstrExpTy})
5431 .addUse(Val)
5432 .setMIFlags(Flags);
5433
5434 if (ST.hasFractBug()) {
5435 auto Fabs = B.buildFAbs(Ty, Val);
5436 auto Inf = B.buildFConstant(Ty, APFloat::getInf(getFltSemanticForLLT(Ty)));
5437 auto IsFinite =
5438 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
5439 auto Zero = B.buildConstant(InstrExpTy, 0);
5440 Exp = B.buildSelect(InstrExpTy, IsFinite, Exp, Zero);
5441 Mant = B.buildSelect(Ty, IsFinite, Mant, Val);
5442 }
5443
5444 B.buildCopy(Res0, Mant);
5445 B.buildSExtOrTrunc(Res1, Exp);
5446
5447 MI.eraseFromParent();
5448 return true;
5449}
5450
5453 MachineIRBuilder &B) const {
5454 Register Res = MI.getOperand(0).getReg();
5455 Register LHS = MI.getOperand(2).getReg();
5456 Register RHS = MI.getOperand(3).getReg();
5457 uint16_t Flags = MI.getFlags();
5458
5459 LLT S32 = LLT::scalar(32);
5460 LLT S1 = LLT::scalar(1);
5461
5462 auto Abs = B.buildFAbs(S32, RHS, Flags);
5463 const APFloat C0Val(1.0f);
5464
5465 auto C0 = B.buildFConstant(S32, 0x1p+96f);
5466 auto C1 = B.buildFConstant(S32, 0x1p-32f);
5467 auto C2 = B.buildFConstant(S32, 1.0f);
5468
5469 auto CmpRes = B.buildFCmp(CmpInst::FCMP_OGT, S1, Abs, C0, Flags);
5470 auto Sel = B.buildSelect(S32, CmpRes, C1, C2, Flags);
5471
5472 auto Mul0 = B.buildFMul(S32, RHS, Sel, Flags);
5473
5474 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32})
5475 .addUse(Mul0.getReg(0))
5476 .setMIFlags(Flags);
5477
5478 auto Mul1 = B.buildFMul(S32, LHS, RCP, Flags);
5479
5480 B.buildFMul(Res, Sel, Mul1, Flags);
5481
5482 MI.eraseFromParent();
5483 return true;
5484}
5485
5488 MachineIRBuilder &B) const {
5489 // Bypass the correct expansion a standard promotion through G_FSQRT would
5490 // get. The f32 op is accurate enough for the f16 cas.
5491 unsigned Flags = MI.getFlags();
5492 assert(!ST.has16BitInsts());
5493 const LLT F32 = LLT::scalar(32);
5494 auto Ext = B.buildFPExt(F32, MI.getOperand(1), Flags);
5495 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_sqrt, {F32})
5496 .addUse(Ext.getReg(0))
5497 .setMIFlags(Flags);
5498 B.buildFPTrunc(MI.getOperand(0), Log2, Flags);
5499 MI.eraseFromParent();
5500 return true;
5501}
5502
5505 MachineIRBuilder &B) const {
5506 MachineFunction &MF = B.getMF();
5507 Register Dst = MI.getOperand(0).getReg();
5508 Register X = MI.getOperand(1).getReg();
5509 const unsigned Flags = MI.getFlags();
5510 const LLT S1 = LLT::scalar(1);
5511 const LLT F32 = LLT::scalar(32);
5512 const LLT I32 = LLT::scalar(32);
5513
5514 if (allowApproxFunc(MF, Flags)) {
5515 B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({Dst}))
5516 .addUse(X)
5517 .setMIFlags(Flags);
5518 MI.eraseFromParent();
5519 return true;
5520 }
5521
5522 auto ScaleThreshold = B.buildFConstant(F32, 0x1.0p-96f);
5523 auto NeedScale = B.buildFCmp(CmpInst::FCMP_OGT, S1, ScaleThreshold, X, Flags);
5524 auto ScaleUpFactor = B.buildFConstant(F32, 0x1.0p+32f);
5525 auto ScaledX = B.buildFMul(F32, X, ScaleUpFactor, Flags);
5526 auto SqrtX = B.buildSelect(F32, NeedScale, ScaledX, X, Flags);
5527
5528 Register SqrtS = MRI.createGenericVirtualRegister(F32);
5529 if (needsDenormHandlingF32(MF, X, Flags)) {
5530 B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({SqrtS}))
5531 .addUse(SqrtX.getReg(0))
5532 .setMIFlags(Flags);
5533
5534 auto NegOne = B.buildConstant(I32, -1);
5535 auto SqrtSNextDown = B.buildAdd(I32, SqrtS, NegOne);
5536
5537 auto NegSqrtSNextDown = B.buildFNeg(F32, SqrtSNextDown, Flags);
5538 auto SqrtVP = B.buildFMA(F32, NegSqrtSNextDown, SqrtS, SqrtX, Flags);
5539
5540 auto PosOne = B.buildConstant(I32, 1);
5541 auto SqrtSNextUp = B.buildAdd(I32, SqrtS, PosOne);
5542
5543 auto NegSqrtSNextUp = B.buildFNeg(F32, SqrtSNextUp, Flags);
5544 auto SqrtVS = B.buildFMA(F32, NegSqrtSNextUp, SqrtS, SqrtX, Flags);
5545
5546 auto Zero = B.buildFConstant(F32, 0.0f);
5547 auto SqrtVPLE0 = B.buildFCmp(CmpInst::FCMP_OLE, S1, SqrtVP, Zero, Flags);
5548
5549 SqrtS =
5550 B.buildSelect(F32, SqrtVPLE0, SqrtSNextDown, SqrtS, Flags).getReg(0);
5551
5552 auto SqrtVPVSGT0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, SqrtVS, Zero, Flags);
5553 SqrtS =
5554 B.buildSelect(F32, SqrtVPVSGT0, SqrtSNextUp, SqrtS, Flags).getReg(0);
5555 } else {
5556 auto SqrtR =
5557 B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F32}).addReg(SqrtX.getReg(0));
5558 B.buildFMul(SqrtS, SqrtX, SqrtR, Flags);
5559
5560 auto Half = B.buildFConstant(F32, 0.5f);
5561 auto SqrtH = B.buildFMul(F32, SqrtR, Half, Flags);
5562 auto NegSqrtH = B.buildFNeg(F32, SqrtH, Flags);
5563 auto SqrtE = B.buildFMA(F32, NegSqrtH, SqrtS, Half, Flags);
5564 SqrtH = B.buildFMA(F32, SqrtH, SqrtE, SqrtH, Flags);
5565 SqrtS = B.buildFMA(F32, SqrtS, SqrtE, SqrtS, Flags).getReg(0);
5566 auto NegSqrtS = B.buildFNeg(F32, SqrtS, Flags);
5567 auto SqrtD = B.buildFMA(F32, NegSqrtS, SqrtS, SqrtX, Flags);
5568 SqrtS = B.buildFMA(F32, SqrtD, SqrtH, SqrtS, Flags).getReg(0);
5569 }
5570
5571 auto ScaleDownFactor = B.buildFConstant(F32, 0x1.0p-16f);
5572
5573 auto ScaledDown = B.buildFMul(F32, SqrtS, ScaleDownFactor, Flags);
5574
5575 SqrtS = B.buildSelect(F32, NeedScale, ScaledDown, SqrtS, Flags).getReg(0);
5576
5577 auto IsZeroOrInf = B.buildIsFPClass(LLT::scalar(1), SqrtX, fcZero | fcPosInf);
5578 B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtS, Flags);
5579
5580 MI.eraseFromParent();
5581 return true;
5582}
5583
5586 MachineIRBuilder &B) const {
5587 // For double type, the SQRT and RSQ instructions don't have required
5588 // precision, we apply Goldschmidt's algorithm to improve the result:
5589 //
5590 // y0 = rsq(x)
5591 // g0 = x * y0
5592 // h0 = 0.5 * y0
5593 //
5594 // r0 = 0.5 - h0 * g0
5595 // g1 = g0 * r0 + g0
5596 // h1 = h0 * r0 + h0
5597 //
5598 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
5599 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
5600 // h2 = h1 * r1 + h1
5601 //
5602 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
5603 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
5604 //
5605 // sqrt(x) = g3
5606
5607 const LLT S1 = LLT::scalar(1);
5608 const LLT S32 = LLT::scalar(32);
5609 const LLT F64 = LLT::scalar(64);
5610
5611 Register Dst = MI.getOperand(0).getReg();
5612 assert(MRI.getType(Dst) == F64 && "only expect to lower f64 sqrt");
5613
5614 Register X = MI.getOperand(1).getReg();
5615 unsigned Flags = MI.getFlags();
5616
5617 auto ScaleConstant = B.buildFConstant(F64, 0x1.0p-767);
5618
5619 auto ZeroInt = B.buildConstant(S32, 0);
5620 auto Scaling = B.buildFCmp(FCmpInst::FCMP_OLT, S1, X, ScaleConstant);
5621
5622 // Scale up input if it is too small.
5623 auto ScaleUpFactor = B.buildConstant(S32, 256);
5624 auto ScaleUp = B.buildSelect(S32, Scaling, ScaleUpFactor, ZeroInt);
5625 auto SqrtX = B.buildFLdexp(F64, X, ScaleUp, Flags);
5626
5627 auto SqrtY =
5628 B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F64}).addReg(SqrtX.getReg(0));
5629
5630 auto Half = B.buildFConstant(F64, 0.5);
5631 auto SqrtH0 = B.buildFMul(F64, SqrtY, Half);
5632 auto SqrtS0 = B.buildFMul(F64, SqrtX, SqrtY);
5633
5634 auto NegSqrtH0 = B.buildFNeg(F64, SqrtH0);
5635 auto SqrtR0 = B.buildFMA(F64, NegSqrtH0, SqrtS0, Half);
5636
5637 auto SqrtS1 = B.buildFMA(F64, SqrtS0, SqrtR0, SqrtS0);
5638 auto SqrtH1 = B.buildFMA(F64, SqrtH0, SqrtR0, SqrtH0);
5639
5640 auto NegSqrtS1 = B.buildFNeg(F64, SqrtS1);
5641 auto SqrtD0 = B.buildFMA(F64, NegSqrtS1, SqrtS1, SqrtX);
5642
5643 auto SqrtS2 = B.buildFMA(F64, SqrtD0, SqrtH1, SqrtS1);
5644
5645 auto NegSqrtS2 = B.buildFNeg(F64, SqrtS2);
5646 auto SqrtD1 = B.buildFMA(F64, NegSqrtS2, SqrtS2, SqrtX);
5647
5648 auto SqrtRet = B.buildFMA(F64, SqrtD1, SqrtH1, SqrtS2);
5649
5650 // Scale down the result.
5651 auto ScaleDownFactor = B.buildConstant(S32, -128);
5652 auto ScaleDown = B.buildSelect(S32, Scaling, ScaleDownFactor, ZeroInt);
5653 SqrtRet = B.buildFLdexp(F64, SqrtRet, ScaleDown, Flags);
5654
5655 // TODO: Switch to fcmp oeq 0 for finite only. Can't fully remove this check
5656 // with finite only or nsz because rsq(+/-0) = +/-inf
5657
5658 // TODO: Check for DAZ and expand to subnormals
5659 auto IsZeroOrInf = B.buildIsFPClass(LLT::scalar(1), SqrtX, fcZero | fcPosInf);
5660
5661 // If x is +INF, +0, or -0, use its original value
5662 B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtRet, Flags);
5663
5664 MI.eraseFromParent();
5665 return true;
5666}
5667
5670 MachineIRBuilder &B) const {
5671 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
5672 if (Ty == LLT::scalar(32))
5673 return legalizeFSQRTF32(MI, MRI, B);
5674 if (Ty == LLT::scalar(64))
5675 return legalizeFSQRTF64(MI, MRI, B);
5676 if (Ty == LLT::scalar(16))
5677 return legalizeFSQRTF16(MI, MRI, B);
5678 return false;
5679}
5680
5681// Expand llvm.amdgcn.rsq.clamp on targets that don't support the instruction.
5682// FIXME: Why do we handle this one but not other removed instructions?
5683//
5684// Reciprocal square root. The clamp prevents infinite results, clamping
5685// infinities to max_float. D.f = 1.0 / sqrt(S0.f), result clamped to
5686// +-max_float.
5689 MachineIRBuilder &B) const {
5690 if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
5691 return true;
5692
5693 Register Dst = MI.getOperand(0).getReg();
5694 Register Src = MI.getOperand(2).getReg();
5695 auto Flags = MI.getFlags();
5696
5697 LLT Ty = MRI.getType(Dst);
5698
5699 const fltSemantics *FltSemantics;
5700 if (Ty == LLT::scalar(32))
5701 FltSemantics = &APFloat::IEEEsingle();
5702 else if (Ty == LLT::scalar(64))
5703 FltSemantics = &APFloat::IEEEdouble();
5704 else
5705 return false;
5706
5707 auto Rsq = B.buildIntrinsic(Intrinsic::amdgcn_rsq, {Ty})
5708 .addUse(Src)
5709 .setMIFlags(Flags);
5710
5711 // We don't need to concern ourselves with the snan handling difference, since
5712 // the rsq quieted (or not) so use the one which will directly select.
5713 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5714 const bool UseIEEE = MFI->getMode().IEEE;
5715
5716 auto MaxFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics));
5717 auto ClampMax = UseIEEE ? B.buildFMinNumIEEE(Ty, Rsq, MaxFlt, Flags) :
5718 B.buildFMinNum(Ty, Rsq, MaxFlt, Flags);
5719
5720 auto MinFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics, true));
5721
5722 if (UseIEEE)
5723 B.buildFMaxNumIEEE(Dst, ClampMax, MinFlt, Flags);
5724 else
5725 B.buildFMaxNum(Dst, ClampMax, MinFlt, Flags);
5726 MI.eraseFromParent();
5727 return true;
5728}
5729
5730// TODO: Fix pointer type handling
5733 Intrinsic::ID IID) const {
5734
5735 MachineIRBuilder &B = Helper.MIRBuilder;
5736 MachineRegisterInfo &MRI = *B.getMRI();
5737
5738 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
5739 IID == Intrinsic::amdgcn_permlanex16;
5740 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
5741 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
5742
5743 auto createLaneOp = [&IID, &B, &MI](Register Src0, Register Src1,
5744 Register Src2, LLT VT) -> Register {
5745 auto LaneOp = B.buildIntrinsic(IID, {VT}).addUse(Src0);
5746 switch (IID) {
5747 case Intrinsic::amdgcn_readfirstlane:
5748 case Intrinsic::amdgcn_permlane64:
5749 return LaneOp.getReg(0);
5750 case Intrinsic::amdgcn_readlane:
5751 case Intrinsic::amdgcn_set_inactive:
5752 case Intrinsic::amdgcn_set_inactive_chain_arg:
5753 return LaneOp.addUse(Src1).getReg(0);
5754 case Intrinsic::amdgcn_writelane:
5755 return LaneOp.addUse(Src1).addUse(Src2).getReg(0);
5756 case Intrinsic::amdgcn_permlane16:
5757 case Intrinsic::amdgcn_permlanex16: {
5758 Register Src3 = MI.getOperand(5).getReg();
5759 int64_t Src4 = MI.getOperand(6).getImm();
5760 int64_t Src5 = MI.getOperand(7).getImm();
5761 return LaneOp.addUse(Src1)
5762 .addUse(Src2)
5763 .addUse(Src3)
5764 .addImm(Src4)
5765 .addImm(Src5)
5766 .getReg(0);
5767 }
5768 case Intrinsic::amdgcn_mov_dpp8:
5769 return LaneOp.addImm(MI.getOperand(3).getImm()).getReg(0);
5770 case Intrinsic::amdgcn_update_dpp:
5771 return LaneOp.addUse(Src1)
5772 .addImm(MI.getOperand(4).getImm())
5773 .addImm(MI.getOperand(5).getImm())
5774 .addImm(MI.getOperand(6).getImm())
5775 .addImm(MI.getOperand(7).getImm())
5776 .getReg(0);
5777 default:
5778 llvm_unreachable("unhandled lane op");
5779 }
5780 };
5781
5782 Register DstReg = MI.getOperand(0).getReg();
5783 Register Src0 = MI.getOperand(2).getReg();
5784 Register Src1, Src2;
5785 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
5786 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
5787 Src1 = MI.getOperand(3).getReg();
5788 if (IID == Intrinsic::amdgcn_writelane || IsPermLane16) {
5789 Src2 = MI.getOperand(4).getReg();
5790 }
5791 }
5792
5793 LLT Ty = MRI.getType(DstReg);
5794 unsigned Size = Ty.getSizeInBits();
5795
5796 unsigned SplitSize = 32;
5797 if (IID == Intrinsic::amdgcn_update_dpp && (Size % 64 == 0) &&
5798 ST.hasDPALU_DPP() &&
5799 AMDGPU::isLegalDPALU_DPPControl(ST, MI.getOperand(4).getImm()))
5800 SplitSize = 64;
5801
5802 if (Size == SplitSize) {
5803 // Already legal
5804 return true;
5805 }
5806
5807 if (Size < 32) {
5808 Src0 = B.buildAnyExt(S32, Src0).getReg(0);
5809
5810 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
5811 Src1 = B.buildAnyExt(LLT::scalar(32), Src1).getReg(0);
5812
5813 if (IID == Intrinsic::amdgcn_writelane)
5814 Src2 = B.buildAnyExt(LLT::scalar(32), Src2).getReg(0);
5815
5816 Register LaneOpDst = createLaneOp(Src0, Src1, Src2, S32);
5817 B.buildTrunc(DstReg, LaneOpDst);
5818 MI.eraseFromParent();
5819 return true;
5820 }
5821
5822 if (Size % SplitSize != 0)
5823 return false;
5824
5825 LLT PartialResTy = LLT::scalar(SplitSize);
5826 bool NeedsBitcast = false;
5827 if (Ty.isVector()) {
5828 LLT EltTy = Ty.getElementType();
5829 unsigned EltSize = EltTy.getSizeInBits();
5830 if (EltSize == SplitSize) {
5831 PartialResTy = EltTy;
5832 } else if (EltSize == 16 || EltSize == 32) {
5833 unsigned NElem = SplitSize / EltSize;
5834 PartialResTy = Ty.changeElementCount(ElementCount::getFixed(NElem));
5835 } else {
5836 // Handle all other cases via S32/S64 pieces
5837 NeedsBitcast = true;
5838 }
5839 }
5840
5841 SmallVector<Register, 4> PartialRes;
5842 unsigned NumParts = Size / SplitSize;
5843 MachineInstrBuilder Src0Parts = B.buildUnmerge(PartialResTy, Src0);
5844 MachineInstrBuilder Src1Parts, Src2Parts;
5845
5846 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
5847 Src1Parts = B.buildUnmerge(PartialResTy, Src1);
5848
5849 if (IID == Intrinsic::amdgcn_writelane)
5850 Src2Parts = B.buildUnmerge(PartialResTy, Src2);
5851
5852 for (unsigned i = 0; i < NumParts; ++i) {
5853 Src0 = Src0Parts.getReg(i);
5854
5855 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
5856 Src1 = Src1Parts.getReg(i);
5857
5858 if (IID == Intrinsic::amdgcn_writelane)
5859 Src2 = Src2Parts.getReg(i);
5860
5861 PartialRes.push_back(createLaneOp(Src0, Src1, Src2, PartialResTy));
5862 }
5863
5864 if (NeedsBitcast)
5865 B.buildBitcast(DstReg, B.buildMergeLikeInstr(
5866 LLT::scalar(Ty.getSizeInBits()), PartialRes));
5867 else
5868 B.buildMergeLikeInstr(DstReg, PartialRes);
5869
5870 MI.eraseFromParent();
5871 return true;
5872}
5873
5876 MachineIRBuilder &B) const {
5878 ST.getTargetLowering()->getImplicitParameterOffset(
5880 LLT DstTy = MRI.getType(DstReg);
5881 LLT IdxTy = LLT::scalar(DstTy.getSizeInBits());
5882
5883 Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
5884 if (!loadInputValue(KernargPtrReg, B,
5886 return false;
5887
5888 B.buildObjectPtrOffset(DstReg, KernargPtrReg,
5889 B.buildConstant(IdxTy, Offset).getReg(0));
5890 return true;
5891}
5892
5893/// To create a buffer resource from a 64-bit pointer, mask off the upper 32
5894/// bits of the pointer and replace them with the stride argument, then
5895/// merge_values everything together. In the common case of a raw buffer (the
5896/// stride component is 0), we can just AND off the upper half.
5899 Register Result = MI.getOperand(0).getReg();
5900 Register Pointer = MI.getOperand(2).getReg();
5901 Register Stride = MI.getOperand(3).getReg();
5902 Register NumRecords = MI.getOperand(4).getReg();
5903 Register Flags = MI.getOperand(5).getReg();
5904
5905 LLT S32 = LLT::scalar(32);
5906 LLT S64 = LLT::scalar(64);
5907
5908 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
5909
5910 auto ExtStride = B.buildAnyExt(S32, Stride);
5911
5912 if (ST.has45BitNumRecordsBufferResource()) {
5913 Register Zero = B.buildConstant(S32, 0).getReg(0);
5914 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
5915 // num_records.
5916 LLT PtrIntTy = LLT::scalar(MRI.getType(Pointer).getSizeInBits());
5917 auto PointerInt = B.buildPtrToInt(PtrIntTy, Pointer);
5918 auto ExtPointer = B.buildAnyExtOrTrunc(S64, PointerInt);
5919 auto NumRecordsLHS = B.buildShl(S64, NumRecords, B.buildConstant(S32, 57));
5920 Register LowHalf = B.buildOr(S64, ExtPointer, NumRecordsLHS).getReg(0);
5921
5922 // Build the higher 64-bit value, which has the higher 38-bit num_records,
5923 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
5924 auto NumRecordsRHS = B.buildLShr(S64, NumRecords, B.buildConstant(S32, 7));
5925 auto ShiftedStride = B.buildShl(S32, ExtStride, B.buildConstant(S32, 12));
5926 auto ExtShiftedStride =
5927 B.buildMergeValues(S64, {Zero, ShiftedStride.getReg(0)});
5928 auto ShiftedFlags = B.buildShl(S32, Flags, B.buildConstant(S32, 28));
5929 auto ExtShiftedFlags =
5930 B.buildMergeValues(S64, {Zero, ShiftedFlags.getReg(0)});
5931 auto CombinedFields = B.buildOr(S64, NumRecordsRHS, ExtShiftedStride);
5932 Register HighHalf =
5933 B.buildOr(S64, CombinedFields, ExtShiftedFlags).getReg(0);
5934 B.buildMergeValues(Result, {LowHalf, HighHalf});
5935 } else {
5936 NumRecords = B.buildTrunc(S32, NumRecords).getReg(0);
5937 auto Unmerge = B.buildUnmerge(S32, Pointer);
5938 auto LowHalf = Unmerge.getReg(0);
5939 auto HighHalf = Unmerge.getReg(1);
5940
5941 auto AndMask = B.buildConstant(S32, 0x0000ffff);
5942 auto Masked = B.buildAnd(S32, HighHalf, AndMask);
5943 auto ShiftConst = B.buildConstant(S32, 16);
5944 auto ShiftedStride = B.buildShl(S32, ExtStride, ShiftConst);
5945 auto NewHighHalf = B.buildOr(S32, Masked, ShiftedStride);
5946 Register NewHighHalfReg = NewHighHalf.getReg(0);
5947 B.buildMergeValues(Result, {LowHalf, NewHighHalfReg, NumRecords, Flags});
5948 }
5949
5950 MI.eraseFromParent();
5951 return true;
5952}
5953
5956 MachineIRBuilder &B) const {
5957 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5958 if (!MFI->isEntryFunction()) {
5961 }
5962
5963 Register DstReg = MI.getOperand(0).getReg();
5964 if (!getImplicitArgPtr(DstReg, MRI, B))
5965 return false;
5966
5967 MI.eraseFromParent();
5968 return true;
5969}
5970
5973 MachineIRBuilder &B) const {
5974 Function &F = B.getMF().getFunction();
5975 std::optional<uint32_t> KnownSize =
5977 if (KnownSize.has_value())
5978 B.buildConstant(DstReg, *KnownSize);
5979 return false;
5980}
5981
5984 MachineIRBuilder &B) const {
5985
5986 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5987 if (!MFI->isEntryFunction()) {
5990 }
5991
5992 Register DstReg = MI.getOperand(0).getReg();
5993 if (!getLDSKernelId(DstReg, MRI, B))
5994 return false;
5995
5996 MI.eraseFromParent();
5997 return true;
5998}
5999
6003 unsigned AddrSpace) const {
6004 const LLT S32 = LLT::scalar(32);
6005 auto Unmerge = B.buildUnmerge(S32, MI.getOperand(2).getReg());
6006 Register Hi32 = Unmerge.getReg(1);
6007
6008 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS &&
6009 ST.hasGloballyAddressableScratch()) {
6010 Register FlatScratchBaseHi =
6011 B.buildInstr(AMDGPU::S_MOV_B32, {S32},
6012 {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)})
6013 .getReg(0);
6014 MRI.setRegClass(FlatScratchBaseHi, &AMDGPU::SReg_32RegClass);
6015 // Test bits 63..58 against the aperture address.
6016 Register XOR = B.buildXor(S32, Hi32, FlatScratchBaseHi).getReg(0);
6017 B.buildICmp(ICmpInst::ICMP_ULT, MI.getOperand(0), XOR,
6018 B.buildConstant(S32, 1u << 26));
6019 } else {
6020 Register ApertureReg = getSegmentAperture(AddrSpace, MRI, B);
6021 B.buildICmp(ICmpInst::ICMP_EQ, MI.getOperand(0), Hi32, ApertureReg);
6022 }
6023 MI.eraseFromParent();
6024 return true;
6025}
6026
6027// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
6028// offset (the offset that is included in bounds checking and swizzling, to be
6029// split between the instruction's voffset and immoffset fields) and soffset
6030// (the offset that is excluded from bounds checking and swizzling, to go in
6031// the instruction's soffset field). This function takes the first kind of
6032// offset and figures out how to split it between voffset and immoffset.
6033std::pair<Register, unsigned>
6035 Register OrigOffset) const {
6036 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST);
6037 Register BaseReg;
6038 unsigned ImmOffset;
6039 const LLT S32 = LLT::scalar(32);
6040 MachineRegisterInfo &MRI = *B.getMRI();
6041
6042 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
6043 // being added, so we can only safely match a 32-bit addition with no unsigned
6044 // overflow.
6045 bool CheckNUW = AMDGPU::isGFX1250(ST);
6046 std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
6047 MRI, OrigOffset, /*KnownBits=*/nullptr, CheckNUW);
6048
6049 // If BaseReg is a pointer, convert it to int.
6050 if (MRI.getType(BaseReg).isPointer())
6051 BaseReg = B.buildPtrToInt(MRI.getType(OrigOffset), BaseReg).getReg(0);
6052
6053 // If the immediate value is too big for the immoffset field, put only bits
6054 // that would normally fit in the immoffset field. The remaining value that
6055 // is copied/added for the voffset field is a large power of 2, and it
6056 // stands more chance of being CSEd with the copy/add for another similar
6057 // load/store.
6058 // However, do not do that rounding down if that is a negative
6059 // number, as it appears to be illegal to have a negative offset in the
6060 // vgpr, even if adding the immediate offset makes it positive.
6061 unsigned Overflow = ImmOffset & ~MaxImm;
6062 ImmOffset -= Overflow;
6063 if ((int32_t)Overflow < 0) {
6064 Overflow += ImmOffset;
6065 ImmOffset = 0;
6066 }
6067
6068 if (Overflow != 0) {
6069 if (!BaseReg) {
6070 BaseReg = B.buildConstant(S32, Overflow).getReg(0);
6071 } else {
6072 auto OverflowVal = B.buildConstant(S32, Overflow);
6073 BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
6074 }
6075 }
6076
6077 if (!BaseReg)
6078 BaseReg = B.buildConstant(S32, 0).getReg(0);
6079
6080 return std::pair(BaseReg, ImmOffset);
6081}
6082
6083/// Handle register layout difference for f16 images for some subtargets.
6086 Register Reg,
6087 bool ImageStore) const {
6088 const LLT S16 = LLT::scalar(16);
6089 const LLT S32 = LLT::scalar(32);
6090 LLT StoreVT = MRI.getType(Reg);
6091 assert(StoreVT.isVector() && StoreVT.getElementType() == S16);
6092
6093 if (ST.hasUnpackedD16VMem()) {
6094 auto Unmerge = B.buildUnmerge(S16, Reg);
6095
6096 SmallVector<Register, 4> WideRegs;
6097 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6098 WideRegs.push_back(B.buildAnyExt(S32, Unmerge.getReg(I)).getReg(0));
6099
6100 int NumElts = StoreVT.getNumElements();
6101
6102 return B.buildBuildVector(LLT::fixed_vector(NumElts, S32), WideRegs)
6103 .getReg(0);
6104 }
6105
6106 if (ImageStore && ST.hasImageStoreD16Bug()) {
6107 if (StoreVT.getNumElements() == 2) {
6108 SmallVector<Register, 4> PackedRegs;
6109 Reg = B.buildBitcast(S32, Reg).getReg(0);
6110 PackedRegs.push_back(Reg);
6111 PackedRegs.resize(2, B.buildUndef(S32).getReg(0));
6112 return B.buildBuildVector(LLT::fixed_vector(2, S32), PackedRegs)
6113 .getReg(0);
6114 }
6115
6116 if (StoreVT.getNumElements() == 3) {
6117 SmallVector<Register, 4> PackedRegs;
6118 auto Unmerge = B.buildUnmerge(S16, Reg);
6119 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6120 PackedRegs.push_back(Unmerge.getReg(I));
6121 PackedRegs.resize(6, B.buildUndef(S16).getReg(0));
6122 Reg = B.buildBuildVector(LLT::fixed_vector(6, S16), PackedRegs).getReg(0);
6123 return B.buildBitcast(LLT::fixed_vector(3, S32), Reg).getReg(0);
6124 }
6125
6126 if (StoreVT.getNumElements() == 4) {
6127 SmallVector<Register, 4> PackedRegs;
6128 Reg = B.buildBitcast(LLT::fixed_vector(2, S32), Reg).getReg(0);
6129 auto Unmerge = B.buildUnmerge(S32, Reg);
6130 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6131 PackedRegs.push_back(Unmerge.getReg(I));
6132 PackedRegs.resize(4, B.buildUndef(S32).getReg(0));
6133 return B.buildBuildVector(LLT::fixed_vector(4, S32), PackedRegs)
6134 .getReg(0);
6135 }
6136
6137 llvm_unreachable("invalid data type");
6138 }
6139
6140 if (StoreVT == LLT::fixed_vector(3, S16)) {
6141 Reg = B.buildPadVectorWithUndefElements(LLT::fixed_vector(4, S16), Reg)
6142 .getReg(0);
6143 }
6144 return Reg;
6145}
6146
6148 Register VData, LLT MemTy,
6149 bool IsFormat) const {
6150 MachineRegisterInfo *MRI = B.getMRI();
6151 LLT Ty = MRI->getType(VData);
6152
6153 const LLT S16 = LLT::scalar(16);
6154
6155 // Fixup buffer resources themselves needing to be v4i128.
6157 return castBufferRsrcToV4I32(VData, B);
6158
6159 if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
6160 Ty = getBitcastRegisterType(Ty);
6161 VData = B.buildBitcast(Ty, VData).getReg(0);
6162 }
6163 // Fixup illegal register types for i8 stores.
6164 if (Ty == LLT::scalar(8) || Ty == S16) {
6165 Register AnyExt = B.buildAnyExt(LLT::scalar(32), VData).getReg(0);
6166 return AnyExt;
6167 }
6168
6169 if (Ty.isVector()) {
6170 if (Ty.getElementType() == S16 && Ty.getNumElements() <= 4) {
6171 if (IsFormat)
6172 return handleD16VData(B, *MRI, VData);
6173 }
6174 }
6175
6176 return VData;
6177}
6178
6180 LegalizerHelper &Helper,
6181 bool IsTyped,
6182 bool IsFormat) const {
6183 MachineIRBuilder &B = Helper.MIRBuilder;
6184 MachineRegisterInfo &MRI = *B.getMRI();
6185
6186 Register VData = MI.getOperand(1).getReg();
6187 LLT Ty = MRI.getType(VData);
6188 LLT EltTy = Ty.getScalarType();
6189 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
6190 const LLT S32 = LLT::scalar(32);
6191
6192 MachineMemOperand *MMO = *MI.memoperands_begin();
6193 const int MemSize = MMO->getSize().getValue();
6194 LLT MemTy = MMO->getMemoryType();
6195
6196 VData = fixStoreSourceType(B, VData, MemTy, IsFormat);
6197
6199 Register RSrc = MI.getOperand(2).getReg();
6200
6201 unsigned ImmOffset;
6202
6203 // The typed intrinsics add an immediate after the registers.
6204 const unsigned NumVIndexOps = IsTyped ? 8 : 7;
6205
6206 // The struct intrinsic variants add one additional operand over raw.
6207 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
6208 Register VIndex;
6209 int OpOffset = 0;
6210 if (HasVIndex) {
6211 VIndex = MI.getOperand(3).getReg();
6212 OpOffset = 1;
6213 } else {
6214 VIndex = B.buildConstant(S32, 0).getReg(0);
6215 }
6216
6217 Register VOffset = MI.getOperand(3 + OpOffset).getReg();
6218 Register SOffset = MI.getOperand(4 + OpOffset).getReg();
6219
6220 unsigned Format = 0;
6221 if (IsTyped) {
6222 Format = MI.getOperand(5 + OpOffset).getImm();
6223 ++OpOffset;
6224 }
6225
6226 unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
6227
6228 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
6229
6230 unsigned Opc;
6231 if (IsTyped) {
6232 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
6233 AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
6234 } else if (IsFormat) {
6235 Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
6236 AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
6237 } else {
6238 switch (MemSize) {
6239 case 1:
6240 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
6241 break;
6242 case 2:
6243 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
6244 break;
6245 default:
6246 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
6247 break;
6248 }
6249 }
6250
6251 auto MIB = B.buildInstr(Opc)
6252 .addUse(VData) // vdata
6253 .addUse(RSrc) // rsrc
6254 .addUse(VIndex) // vindex
6255 .addUse(VOffset) // voffset
6256 .addUse(SOffset) // soffset
6257 .addImm(ImmOffset); // offset(imm)
6258
6259 if (IsTyped)
6260 MIB.addImm(Format);
6261
6262 MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
6263 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
6264 .addMemOperand(MMO);
6265
6266 MI.eraseFromParent();
6267 return true;
6268}
6269
6270static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc,
6271 Register VIndex, Register VOffset, Register SOffset,
6272 unsigned ImmOffset, unsigned Format,
6273 unsigned AuxiliaryData, MachineMemOperand *MMO,
6274 bool IsTyped, bool HasVIndex, MachineIRBuilder &B) {
6275 auto MIB = B.buildInstr(Opc)
6276 .addDef(LoadDstReg) // vdata
6277 .addUse(RSrc) // rsrc
6278 .addUse(VIndex) // vindex
6279 .addUse(VOffset) // voffset
6280 .addUse(SOffset) // soffset
6281 .addImm(ImmOffset); // offset(imm)
6282
6283 if (IsTyped)
6284 MIB.addImm(Format);
6285
6286 MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
6287 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
6288 .addMemOperand(MMO);
6289}
6290
6292 LegalizerHelper &Helper,
6293 bool IsFormat,
6294 bool IsTyped) const {
6295 MachineIRBuilder &B = Helper.MIRBuilder;
6296 MachineRegisterInfo &MRI = *B.getMRI();
6297 GISelChangeObserver &Observer = Helper.Observer;
6298
6299 // FIXME: Verifier should enforce 1 MMO for these intrinsics.
6300 MachineMemOperand *MMO = *MI.memoperands_begin();
6301 const LLT MemTy = MMO->getMemoryType();
6302 const LLT S32 = LLT::scalar(32);
6303
6304 Register Dst = MI.getOperand(0).getReg();
6305
6306 Register StatusDst;
6307 int OpOffset = 0;
6308 assert(MI.getNumExplicitDefs() == 1 || MI.getNumExplicitDefs() == 2);
6309 bool IsTFE = MI.getNumExplicitDefs() == 2;
6310 if (IsTFE) {
6311 StatusDst = MI.getOperand(1).getReg();
6312 ++OpOffset;
6313 }
6314
6315 castBufferRsrcArgToV4I32(MI, B, 2 + OpOffset);
6316 Register RSrc = MI.getOperand(2 + OpOffset).getReg();
6317
6318 // The typed intrinsics add an immediate after the registers.
6319 const unsigned NumVIndexOps = IsTyped ? 8 : 7;
6320
6321 // The struct intrinsic variants add one additional operand over raw.
6322 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps + OpOffset;
6323 Register VIndex;
6324 if (HasVIndex) {
6325 VIndex = MI.getOperand(3 + OpOffset).getReg();
6326 ++OpOffset;
6327 } else {
6328 VIndex = B.buildConstant(S32, 0).getReg(0);
6329 }
6330
6331 Register VOffset = MI.getOperand(3 + OpOffset).getReg();
6332 Register SOffset = MI.getOperand(4 + OpOffset).getReg();
6333
6334 unsigned Format = 0;
6335 if (IsTyped) {
6336 Format = MI.getOperand(5 + OpOffset).getImm();
6337 ++OpOffset;
6338 }
6339
6340 unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
6341 unsigned ImmOffset;
6342
6343 LLT Ty = MRI.getType(Dst);
6344 // Make addrspace 8 pointers loads into 4xs32 loads here, so the rest of the
6345 // logic doesn't have to handle that case.
6346 if (hasBufferRsrcWorkaround(Ty)) {
6347 Observer.changingInstr(MI);
6348 Ty = castBufferRsrcFromV4I32(MI, B, MRI, 0);
6349 Observer.changedInstr(MI);
6350 Dst = MI.getOperand(0).getReg();
6351 B.setInsertPt(B.getMBB(), MI);
6352 }
6353 if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
6354 Ty = getBitcastRegisterType(Ty);
6355 Observer.changingInstr(MI);
6356 Helper.bitcastDst(MI, Ty, 0);
6357 Observer.changedInstr(MI);
6358 Dst = MI.getOperand(0).getReg();
6359 B.setInsertPt(B.getMBB(), MI);
6360 }
6361
6362 LLT EltTy = Ty.getScalarType();
6363 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
6364 const bool Unpacked = ST.hasUnpackedD16VMem();
6365
6366 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
6367
6368 unsigned Opc;
6369
6370 // TODO: Support TFE for typed and narrow loads.
6371 if (IsTyped) {
6372 if (IsTFE)
6373 return false;
6374 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
6375 AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
6376 } else if (IsFormat) {
6377 if (IsD16) {
6378 if (IsTFE)
6379 return false;
6380 Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
6381 } else {
6382 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
6383 : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
6384 }
6385 } else {
6386 switch (MemTy.getSizeInBits()) {
6387 case 8:
6388 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE
6389 : AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
6390 break;
6391 case 16:
6392 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE
6393 : AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
6394 break;
6395 default:
6396 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE
6397 : AMDGPU::G_AMDGPU_BUFFER_LOAD;
6398 break;
6399 }
6400 }
6401
6402 if (IsTFE) {
6403 unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
6404 unsigned NumLoadDWords = NumValueDWords + 1;
6405 LLT LoadTy = LLT::fixed_vector(NumLoadDWords, S32);
6406 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
6407 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
6408 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6409 if (MemTy.getSizeInBits() < 32) {
6410 Register ExtDst = B.getMRI()->createGenericVirtualRegister(S32);
6411 B.buildUnmerge({ExtDst, StatusDst}, LoadDstReg);
6412 B.buildTrunc(Dst, ExtDst);
6413 } else if (NumValueDWords == 1) {
6414 B.buildUnmerge({Dst, StatusDst}, LoadDstReg);
6415 } else {
6416 SmallVector<Register, 5> LoadElts;
6417 for (unsigned I = 0; I != NumValueDWords; ++I)
6418 LoadElts.push_back(B.getMRI()->createGenericVirtualRegister(S32));
6419 LoadElts.push_back(StatusDst);
6420 B.buildUnmerge(LoadElts, LoadDstReg);
6421 LoadElts.truncate(NumValueDWords);
6422 B.buildMergeLikeInstr(Dst, LoadElts);
6423 }
6424 } else if ((!IsD16 && MemTy.getSizeInBits() < 32) ||
6425 (IsD16 && !Ty.isVector())) {
6426 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(S32);
6427 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
6428 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6429 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
6430 B.buildTrunc(Dst, LoadDstReg);
6431 } else if (Unpacked && IsD16 && Ty.isVector()) {
6432 LLT UnpackedTy = Ty.changeElementSize(32);
6433 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(UnpackedTy);
6434 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
6435 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6436 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
6437 // FIXME: G_TRUNC should work, but legalization currently fails
6438 auto Unmerge = B.buildUnmerge(S32, LoadDstReg);
6440 for (unsigned I = 0, N = Unmerge->getNumOperands() - 1; I != N; ++I)
6441 Repack.push_back(B.buildTrunc(EltTy, Unmerge.getReg(I)).getReg(0));
6442 B.buildMergeLikeInstr(Dst, Repack);
6443 } else {
6444 buildBufferLoad(Opc, Dst, RSrc, VIndex, VOffset, SOffset, ImmOffset, Format,
6445 AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6446 }
6447
6448 MI.eraseFromParent();
6449 return true;
6450}
6451
6452static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
6453 switch (IntrID) {
6454 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
6455 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
6456 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
6457 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
6458 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
6459 case Intrinsic::amdgcn_raw_buffer_atomic_add:
6460 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
6461 case Intrinsic::amdgcn_struct_buffer_atomic_add:
6462 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
6463 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
6464 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
6465 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
6466 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
6467 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
6468 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
6469 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
6470 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
6471 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
6472 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
6473 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
6474 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
6475 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
6476 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
6477 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
6478 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
6479 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
6480 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
6481 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
6482 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
6483 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
6484 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
6485 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
6486 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
6487 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
6488 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
6489 case Intrinsic::amdgcn_raw_buffer_atomic_and:
6490 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
6491 case Intrinsic::amdgcn_struct_buffer_atomic_and:
6492 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
6493 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
6494 case Intrinsic::amdgcn_raw_buffer_atomic_or:
6495 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
6496 case Intrinsic::amdgcn_struct_buffer_atomic_or:
6497 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
6498 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
6499 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
6500 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
6501 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
6502 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
6503 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
6504 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
6505 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
6506 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
6507 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
6508 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
6509 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
6510 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
6511 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
6512 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
6513 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
6514 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
6515 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
6516 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
6517 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
6518 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
6519 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
6520 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
6521 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
6522 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
6523 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD;
6524 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
6525 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
6526 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
6527 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
6528 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN;
6529 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
6530 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
6531 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
6532 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
6533 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX;
6534 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
6535 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
6536 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32;
6537 default:
6538 llvm_unreachable("unhandled atomic opcode");
6539 }
6540}
6541
6544 Intrinsic::ID IID) const {
6545 const bool IsCmpSwap =
6546 IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
6547 IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap ||
6548 IID == Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap ||
6549 IID == Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap;
6550
6551 Register Dst = MI.getOperand(0).getReg();
6552 // Since we don't have 128-bit atomics, we don't need to handle the case of
6553 // p8 argmunents to the atomic itself
6554 Register VData = MI.getOperand(2).getReg();
6555
6556 Register CmpVal;
6557 int OpOffset = 0;
6558
6559 if (IsCmpSwap) {
6560 CmpVal = MI.getOperand(3).getReg();
6561 ++OpOffset;
6562 }
6563
6564 castBufferRsrcArgToV4I32(MI, B, 3 + OpOffset);
6565 Register RSrc = MI.getOperand(3 + OpOffset).getReg();
6566 const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
6567
6568 // The struct intrinsic variants add one additional operand over raw.
6569 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
6570 Register VIndex;
6571 if (HasVIndex) {
6572 VIndex = MI.getOperand(4 + OpOffset).getReg();
6573 ++OpOffset;
6574 } else {
6575 VIndex = B.buildConstant(LLT::scalar(32), 0).getReg(0);
6576 }
6577
6578 Register VOffset = MI.getOperand(4 + OpOffset).getReg();
6579 Register SOffset = MI.getOperand(5 + OpOffset).getReg();
6580 unsigned AuxiliaryData = MI.getOperand(6 + OpOffset).getImm();
6581
6582 MachineMemOperand *MMO = *MI.memoperands_begin();
6583
6584 unsigned ImmOffset;
6585 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
6586
6587 auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
6588 .addDef(Dst)
6589 .addUse(VData); // vdata
6590
6591 if (IsCmpSwap)
6592 MIB.addReg(CmpVal);
6593
6594 MIB.addUse(RSrc) // rsrc
6595 .addUse(VIndex) // vindex
6596 .addUse(VOffset) // voffset
6597 .addUse(SOffset) // soffset
6598 .addImm(ImmOffset) // offset(imm)
6599 .addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
6600 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
6601 .addMemOperand(MMO);
6602
6603 MI.eraseFromParent();
6604 return true;
6605}
6606
6607/// Turn a set of s16 typed registers in \p AddrRegs into a dword sized
6608/// vector with s16 typed elements.
6610 SmallVectorImpl<Register> &PackedAddrs,
6611 unsigned ArgOffset,
6613 bool IsA16, bool IsG16) {
6614 const LLT S16 = LLT::scalar(16);
6615 const LLT V2S16 = LLT::fixed_vector(2, 16);
6616 auto EndIdx = Intr->VAddrEnd;
6617
6618 for (unsigned I = Intr->VAddrStart; I < EndIdx; I++) {
6619 MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
6620 if (!SrcOp.isReg())
6621 continue; // _L to _LZ may have eliminated this.
6622
6623 Register AddrReg = SrcOp.getReg();
6624
6625 if ((I < Intr->GradientStart) ||
6626 (I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) ||
6627 (I >= Intr->CoordStart && !IsA16)) {
6628 if ((I < Intr->GradientStart) && IsA16 &&
6629 (B.getMRI()->getType(AddrReg) == S16)) {
6630 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
6631 // Special handling of bias when A16 is on. Bias is of type half but
6632 // occupies full 32-bit.
6633 PackedAddrs.push_back(
6634 B.buildBuildVector(V2S16, {AddrReg, B.buildUndef(S16).getReg(0)})
6635 .getReg(0));
6636 } else {
6637 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
6638 "Bias needs to be converted to 16 bit in A16 mode");
6639 // Handle any gradient or coordinate operands that should not be packed
6640 AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0);
6641 PackedAddrs.push_back(AddrReg);
6642 }
6643 } else {
6644 // Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D,
6645 // derivatives dx/dh and dx/dv are packed with undef.
6646 if (((I + 1) >= EndIdx) ||
6647 ((Intr->NumGradients / 2) % 2 == 1 &&
6648 (I == static_cast<unsigned>(Intr->GradientStart +
6649 (Intr->NumGradients / 2) - 1) ||
6650 I == static_cast<unsigned>(Intr->GradientStart +
6651 Intr->NumGradients - 1))) ||
6652 // Check for _L to _LZ optimization
6653 !MI.getOperand(ArgOffset + I + 1).isReg()) {
6654 PackedAddrs.push_back(
6655 B.buildBuildVector(V2S16, {AddrReg, B.buildUndef(S16).getReg(0)})
6656 .getReg(0));
6657 } else {
6658 PackedAddrs.push_back(
6659 B.buildBuildVector(
6660 V2S16, {AddrReg, MI.getOperand(ArgOffset + I + 1).getReg()})
6661 .getReg(0));
6662 ++I;
6663 }
6664 }
6665 }
6666}
6667
6668/// Convert from separate vaddr components to a single vector address register,
6669/// and replace the remaining operands with $noreg.
6671 int DimIdx, int NumVAddrs) {
6672 const LLT S32 = LLT::scalar(32);
6673 (void)S32;
6674 SmallVector<Register, 8> AddrRegs;
6675 for (int I = 0; I != NumVAddrs; ++I) {
6676 MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
6677 if (SrcOp.isReg()) {
6678 AddrRegs.push_back(SrcOp.getReg());
6679 assert(B.getMRI()->getType(SrcOp.getReg()) == S32);
6680 }
6681 }
6682
6683 int NumAddrRegs = AddrRegs.size();
6684 if (NumAddrRegs != 1) {
6685 auto VAddr =
6686 B.buildBuildVector(LLT::fixed_vector(NumAddrRegs, 32), AddrRegs);
6687 MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
6688 }
6689
6690 for (int I = 1; I != NumVAddrs; ++I) {
6691 MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
6692 if (SrcOp.isReg())
6693 MI.getOperand(DimIdx + I).setReg(AMDGPU::NoRegister);
6694 }
6695}
6696
6697/// Rewrite image intrinsics to use register layouts expected by the subtarget.
6698///
6699/// Depending on the subtarget, load/store with 16-bit element data need to be
6700/// rewritten to use the low half of 32-bit registers, or directly use a packed
6701/// layout. 16-bit addresses should also sometimes be packed into 32-bit
6702/// registers.
6703///
6704/// We don't want to directly select image instructions just yet, but also want
6705/// to exposes all register repacking to the legalizer/combiners. We also don't
6706/// want a selected instruction entering RegBankSelect. In order to avoid
6707/// defining a multitude of intermediate image instructions, directly hack on
6708/// the intrinsic's arguments. In cases like a16 addresses, this requires
6709/// padding now unnecessary arguments with $noreg.
6712 const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
6713
6714 const MachineFunction &MF = *MI.getMF();
6715 const unsigned NumDefs = MI.getNumExplicitDefs();
6716 const unsigned ArgOffset = NumDefs + 1;
6717 bool IsTFE = NumDefs == 2;
6718 // We are only processing the operands of d16 image operations on subtargets
6719 // that use the unpacked register layout, or need to repack the TFE result.
6720
6721 // TODO: Do we need to guard against already legalized intrinsics?
6722 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
6724
6725 MachineRegisterInfo *MRI = B.getMRI();
6726 const LLT S32 = LLT::scalar(32);
6727 const LLT S16 = LLT::scalar(16);
6728 const LLT V2S16 = LLT::fixed_vector(2, 16);
6729
6730 unsigned DMask = 0;
6731 Register VData;
6732 LLT Ty;
6733
6734 if (!BaseOpcode->NoReturn || BaseOpcode->Store) {
6735 VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
6736 Ty = MRI->getType(VData);
6737 }
6738
6739 const bool IsAtomicPacked16Bit =
6740 (BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
6741 BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16);
6742
6743 // Check for 16 bit addresses and pack if true.
6744 LLT GradTy =
6745 MRI->getType(MI.getOperand(ArgOffset + Intr->GradientStart).getReg());
6746 LLT AddrTy =
6747 MRI->getType(MI.getOperand(ArgOffset + Intr->CoordStart).getReg());
6748 const bool IsG16 =
6749 ST.hasG16() ? (BaseOpcode->Gradients && GradTy == S16) : GradTy == S16;
6750 const bool IsA16 = AddrTy == S16;
6751 const bool IsD16 = !IsAtomicPacked16Bit && Ty.getScalarType() == S16;
6752
6753 int DMaskLanes = 0;
6754 if (!BaseOpcode->Atomic) {
6755 DMask = MI.getOperand(ArgOffset + Intr->DMaskIndex).getImm();
6756 if (BaseOpcode->Gather4) {
6757 DMaskLanes = 4;
6758 } else if (DMask != 0) {
6759 DMaskLanes = llvm::popcount(DMask);
6760 } else if (!IsTFE && !BaseOpcode->Store) {
6761 // If dmask is 0, this is a no-op load. This can be eliminated.
6762 B.buildUndef(MI.getOperand(0));
6763 MI.eraseFromParent();
6764 return true;
6765 }
6766 }
6767
6768 Observer.changingInstr(MI);
6769 auto ChangedInstr = make_scope_exit([&] { Observer.changedInstr(MI); });
6770
6771 const unsigned StoreOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16
6772 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
6773 const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
6774 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
6775 unsigned NewOpcode = LoadOpcode;
6776 if (BaseOpcode->Store)
6777 NewOpcode = StoreOpcode;
6778 else if (BaseOpcode->NoReturn)
6779 NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;
6780
6781 // Track that we legalized this
6782 MI.setDesc(B.getTII().get(NewOpcode));
6783
6784 // Expecting to get an error flag since TFC is on - and dmask is 0 Force
6785 // dmask to be at least 1 otherwise the instruction will fail
6786 if (IsTFE && DMask == 0) {
6787 DMask = 0x1;
6788 DMaskLanes = 1;
6789 MI.getOperand(ArgOffset + Intr->DMaskIndex).setImm(DMask);
6790 }
6791
6792 if (BaseOpcode->Atomic) {
6793 Register VData0 = MI.getOperand(2).getReg();
6794 LLT Ty = MRI->getType(VData0);
6795
6796 // TODO: Allow atomic swap and bit ops for v2s16/v4s16
6797 if (Ty.isVector() && !IsAtomicPacked16Bit)
6798 return false;
6799
6800 if (BaseOpcode->AtomicX2) {
6801 Register VData1 = MI.getOperand(3).getReg();
6802 // The two values are packed in one register.
6803 LLT PackedTy = LLT::fixed_vector(2, Ty);
6804 auto Concat = B.buildBuildVector(PackedTy, {VData0, VData1});
6805 MI.getOperand(2).setReg(Concat.getReg(0));
6806 MI.getOperand(3).setReg(AMDGPU::NoRegister);
6807 }
6808 }
6809
6810 unsigned CorrectedNumVAddrs = Intr->NumVAddrs;
6811
6812 // Rewrite the addressing register layout before doing anything else.
6813 if (BaseOpcode->Gradients && !ST.hasG16() && (IsA16 != IsG16)) {
6814 // 16 bit gradients are supported, but are tied to the A16 control
6815 // so both gradients and addresses must be 16 bit
6816 return false;
6817 }
6818
6819 if (IsA16 && !ST.hasA16()) {
6820 // A16 not supported
6821 return false;
6822 }
6823
6824 const unsigned NSAMaxSize = ST.getNSAMaxSize(BaseOpcode->Sampler);
6825 const unsigned HasPartialNSA = ST.hasPartialNSAEncoding();
6826
6827 if (IsA16 || IsG16) {
6828 // Even if NumVAddrs == 1 we should pack it into a 32-bit value, because the
6829 // instructions expect VGPR_32
6830 SmallVector<Register, 4> PackedRegs;
6831
6832 packImage16bitOpsToDwords(B, MI, PackedRegs, ArgOffset, Intr, IsA16, IsG16);
6833
6834 // See also below in the non-a16 branch
6835 const bool UseNSA = ST.hasNSAEncoding() &&
6836 PackedRegs.size() >= ST.getNSAThreshold(MF) &&
6837 (PackedRegs.size() <= NSAMaxSize || HasPartialNSA);
6838 const bool UsePartialNSA =
6839 UseNSA && HasPartialNSA && PackedRegs.size() > NSAMaxSize;
6840
6841 if (UsePartialNSA) {
6842 // Pack registers that would go over NSAMaxSize into last VAddr register
6843 LLT PackedAddrTy =
6844 LLT::fixed_vector(2 * (PackedRegs.size() - NSAMaxSize + 1), 16);
6845 auto Concat = B.buildConcatVectors(
6846 PackedAddrTy, ArrayRef(PackedRegs).slice(NSAMaxSize - 1));
6847 PackedRegs[NSAMaxSize - 1] = Concat.getReg(0);
6848 PackedRegs.resize(NSAMaxSize);
6849 } else if (!UseNSA && PackedRegs.size() > 1) {
6850 LLT PackedAddrTy = LLT::fixed_vector(2 * PackedRegs.size(), 16);
6851 auto Concat = B.buildConcatVectors(PackedAddrTy, PackedRegs);
6852 PackedRegs[0] = Concat.getReg(0);
6853 PackedRegs.resize(1);
6854 }
6855
6856 const unsigned NumPacked = PackedRegs.size();
6857 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
6858 MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
6859 if (!SrcOp.isReg()) {
6860 assert(SrcOp.isImm() && SrcOp.getImm() == 0);
6861 continue;
6862 }
6863
6864 assert(SrcOp.getReg() != AMDGPU::NoRegister);
6865
6866 if (I - Intr->VAddrStart < NumPacked)
6867 SrcOp.setReg(PackedRegs[I - Intr->VAddrStart]);
6868 else
6869 SrcOp.setReg(AMDGPU::NoRegister);
6870 }
6871 } else {
6872 // If the register allocator cannot place the address registers contiguously
6873 // without introducing moves, then using the non-sequential address encoding
6874 // is always preferable, since it saves VALU instructions and is usually a
6875 // wash in terms of code size or even better.
6876 //
6877 // However, we currently have no way of hinting to the register allocator
6878 // that MIMG addresses should be placed contiguously when it is possible to
6879 // do so, so force non-NSA for the common 2-address case as a heuristic.
6880 //
6881 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
6882 // allocation when possible.
6883 //
6884 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
6885 // set of the remaining addresses.
6886 const bool UseNSA = ST.hasNSAEncoding() &&
6887 CorrectedNumVAddrs >= ST.getNSAThreshold(MF) &&
6888 (CorrectedNumVAddrs <= NSAMaxSize || HasPartialNSA);
6889 const bool UsePartialNSA =
6890 UseNSA && HasPartialNSA && CorrectedNumVAddrs > NSAMaxSize;
6891
6892 if (UsePartialNSA) {
6894 ArgOffset + Intr->VAddrStart + NSAMaxSize - 1,
6895 Intr->NumVAddrs - NSAMaxSize + 1);
6896 } else if (!UseNSA && Intr->NumVAddrs > 1) {
6897 convertImageAddrToPacked(B, MI, ArgOffset + Intr->VAddrStart,
6898 Intr->NumVAddrs);
6899 }
6900 }
6901
6902 int Flags = 0;
6903 if (IsA16)
6904 Flags |= 1;
6905 if (IsG16)
6906 Flags |= 2;
6907 MI.addOperand(MachineOperand::CreateImm(Flags));
6908
6909 if (BaseOpcode->NoReturn) { // No TFE for stores?
6910 // TODO: Handle dmask trim
6911 if (!Ty.isVector() || !IsD16)
6912 return true;
6913
6914 Register RepackedReg = handleD16VData(B, *MRI, VData, true);
6915 if (RepackedReg != VData) {
6916 MI.getOperand(1).setReg(RepackedReg);
6917 }
6918
6919 return true;
6920 }
6921
6922 Register DstReg = MI.getOperand(0).getReg();
6923 const LLT EltTy = Ty.getScalarType();
6924 const int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;
6925
6926 // Confirm that the return type is large enough for the dmask specified
6927 if (NumElts < DMaskLanes)
6928 return false;
6929
6930 if (NumElts > 4 || DMaskLanes > 4)
6931 return false;
6932
6933 // Image atomic instructions are using DMask to specify how many bits
6934 // input/output data will have. 32-bits (s32, v2s16) or 64-bits (s64, v4s16).
6935 // DMaskLanes for image atomic has default value '0'.
6936 // We must be sure that atomic variants (especially packed) will not be
6937 // truncated from v2s16 or v4s16 to s16 type.
6938 //
6939 // ChangeElementCount will be needed for image load where Ty is always scalar.
6940 const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
6941 const LLT AdjustedTy =
6942 DMaskLanes == 0
6943 ? Ty
6944 : Ty.changeElementCount(ElementCount::getFixed(AdjustedNumElts));
6945
6946 // The raw dword aligned data component of the load. The only legal cases
6947 // where this matters should be when using the packed D16 format, for
6948 // s16 -> <2 x s16>, and <3 x s16> -> <4 x s16>,
6949 LLT RoundedTy;
6950
6951 // S32 vector to cover all data, plus TFE result element.
6952 LLT TFETy;
6953
6954 // Register type to use for each loaded component. Will be S32 or V2S16.
6955 LLT RegTy;
6956
6957 if (IsD16 && ST.hasUnpackedD16VMem()) {
6958 RoundedTy =
6959 LLT::scalarOrVector(ElementCount::getFixed(AdjustedNumElts), 32);
6960 TFETy = LLT::fixed_vector(AdjustedNumElts + 1, 32);
6961 RegTy = S32;
6962 } else {
6963 unsigned EltSize = EltTy.getSizeInBits();
6964 unsigned RoundedElts = (AdjustedTy.getSizeInBits() + 31) / 32;
6965 unsigned RoundedSize = 32 * RoundedElts;
6966 RoundedTy = LLT::scalarOrVector(
6967 ElementCount::getFixed(RoundedSize / EltSize), EltSize);
6968 TFETy = LLT::fixed_vector(RoundedSize / 32 + 1, S32);
6969 RegTy = !IsTFE && EltSize == 16 ? V2S16 : S32;
6970 }
6971
6972 // The return type does not need adjustment.
6973 // TODO: Should we change s16 case to s32 or <2 x s16>?
6974 if (!IsTFE && (RoundedTy == Ty || !Ty.isVector()))
6975 return true;
6976
6977 Register Dst1Reg;
6978
6979 // Insert after the instruction.
6980 B.setInsertPt(*MI.getParent(), ++MI.getIterator());
6981
6982 // TODO: For TFE with d16, if we used a TFE type that was a multiple of <2 x
6983 // s16> instead of s32, we would only need 1 bitcast instead of multiple.
6984 const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy;
6985 const int ResultNumRegs = LoadResultTy.getSizeInBits() / 32;
6986
6987 Register NewResultReg = MRI->createGenericVirtualRegister(LoadResultTy);
6988
6989 MI.getOperand(0).setReg(NewResultReg);
6990
6991 // In the IR, TFE is supposed to be used with a 2 element struct return
6992 // type. The instruction really returns these two values in one contiguous
6993 // register, with one additional dword beyond the loaded data. Rewrite the
6994 // return type to use a single register result.
6995
6996 if (IsTFE) {
6997 Dst1Reg = MI.getOperand(1).getReg();
6998 if (MRI->getType(Dst1Reg) != S32)
6999 return false;
7000
7001 // TODO: Make sure the TFE operand bit is set.
7002 MI.removeOperand(1);
7003
7004 // Handle the easy case that requires no repack instructions.
7005 if (Ty == S32) {
7006 B.buildUnmerge({DstReg, Dst1Reg}, NewResultReg);
7007 return true;
7008 }
7009 }
7010
7011 // Now figure out how to copy the new result register back into the old
7012 // result.
7013 SmallVector<Register, 5> ResultRegs(ResultNumRegs, Dst1Reg);
7014
7015 const int NumDataRegs = IsTFE ? ResultNumRegs - 1 : ResultNumRegs;
7016
7017 if (ResultNumRegs == 1) {
7018 assert(!IsTFE);
7019 ResultRegs[0] = NewResultReg;
7020 } else {
7021 // We have to repack into a new vector of some kind.
7022 for (int I = 0; I != NumDataRegs; ++I)
7023 ResultRegs[I] = MRI->createGenericVirtualRegister(RegTy);
7024 B.buildUnmerge(ResultRegs, NewResultReg);
7025
7026 // Drop the final TFE element to get the data part. The TFE result is
7027 // directly written to the right place already.
7028 if (IsTFE)
7029 ResultRegs.resize(NumDataRegs);
7030 }
7031
7032 // For an s16 scalar result, we form an s32 result with a truncate regardless
7033 // of packed vs. unpacked.
7034 if (IsD16 && !Ty.isVector()) {
7035 B.buildTrunc(DstReg, ResultRegs[0]);
7036 return true;
7037 }
7038
7039 // Avoid a build/concat_vector of 1 entry.
7040 if (Ty == V2S16 && NumDataRegs == 1 && !ST.hasUnpackedD16VMem()) {
7041 B.buildBitcast(DstReg, ResultRegs[0]);
7042 return true;
7043 }
7044
7045 assert(Ty.isVector());
7046
7047 if (IsD16) {
7048 // For packed D16 results with TFE enabled, all the data components are
7049 // S32. Cast back to the expected type.
7050 //
7051 // TODO: We don't really need to use load s32 elements. We would only need one
7052 // cast for the TFE result if a multiple of v2s16 was used.
7053 if (RegTy != V2S16 && !ST.hasUnpackedD16VMem()) {
7054 for (Register &Reg : ResultRegs)
7055 Reg = B.buildBitcast(V2S16, Reg).getReg(0);
7056 } else if (ST.hasUnpackedD16VMem()) {
7057 for (Register &Reg : ResultRegs)
7058 Reg = B.buildTrunc(S16, Reg).getReg(0);
7059 }
7060 }
7061
7062 auto padWithUndef = [&](LLT Ty, int NumElts) {
7063 if (NumElts == 0)
7064 return;
7065 Register Undef = B.buildUndef(Ty).getReg(0);
7066 for (int I = 0; I != NumElts; ++I)
7067 ResultRegs.push_back(Undef);
7068 };
7069
7070 // Pad out any elements eliminated due to the dmask.
7071 LLT ResTy = MRI->getType(ResultRegs[0]);
7072 if (!ResTy.isVector()) {
7073 padWithUndef(ResTy, NumElts - ResultRegs.size());
7074 B.buildBuildVector(DstReg, ResultRegs);
7075 return true;
7076 }
7077
7078 assert(!ST.hasUnpackedD16VMem() && ResTy == V2S16);
7079 const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;
7080
7081 // Deal with the one annoying legal case.
7082 const LLT V3S16 = LLT::fixed_vector(3, 16);
7083 if (Ty == V3S16) {
7084 if (IsTFE) {
7085 if (ResultRegs.size() == 1) {
7086 NewResultReg = ResultRegs[0];
7087 } else if (ResultRegs.size() == 2) {
7088 LLT V4S16 = LLT::fixed_vector(4, 16);
7089 NewResultReg = B.buildConcatVectors(V4S16, ResultRegs).getReg(0);
7090 } else {
7091 return false;
7092 }
7093 }
7094
7095 if (MRI->getType(DstReg).getNumElements() <
7096 MRI->getType(NewResultReg).getNumElements()) {
7097 B.buildDeleteTrailingVectorElements(DstReg, NewResultReg);
7098 } else {
7099 B.buildPadVectorWithUndefElements(DstReg, NewResultReg);
7100 }
7101 return true;
7102 }
7103
7104 padWithUndef(ResTy, RegsToCover - ResultRegs.size());
7105 B.buildConcatVectors(DstReg, ResultRegs);
7106 return true;
7107}
7108
7110 MachineInstr &MI) const {
7111 MachineIRBuilder &B = Helper.MIRBuilder;
7112 GISelChangeObserver &Observer = Helper.Observer;
7113
7114 Register OrigDst = MI.getOperand(0).getReg();
7115 Register Dst;
7116 LLT Ty = B.getMRI()->getType(OrigDst);
7117 unsigned Size = Ty.getSizeInBits();
7118 MachineFunction &MF = B.getMF();
7119 unsigned Opc = 0;
7120 if (Size < 32 && ST.hasScalarSubwordLoads()) {
7121 assert(Size == 8 || Size == 16);
7122 Opc = Size == 8 ? AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE
7123 : AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT;
7124 // The 8-bit and 16-bit scalar buffer load instructions have 32-bit
7125 // destination register.
7126 Dst = B.getMRI()->createGenericVirtualRegister(LLT::scalar(32));
7127 } else {
7128 Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
7129 Dst = OrigDst;
7130 }
7131
7132 Observer.changingInstr(MI);
7133
7134 // Handle needing to s.buffer.load() a p8 value.
7135 if (hasBufferRsrcWorkaround(Ty)) {
7136 Ty = castBufferRsrcFromV4I32(MI, B, *B.getMRI(), 0);
7137 B.setInsertPt(B.getMBB(), MI);
7138 }
7140 Ty = getBitcastRegisterType(Ty);
7141 Helper.bitcastDst(MI, Ty, 0);
7142 B.setInsertPt(B.getMBB(), MI);
7143 }
7144
7145 // FIXME: We don't really need this intermediate instruction. The intrinsic
7146 // should be fixed to have a memory operand. Since it's readnone, we're not
7147 // allowed to add one.
7148 MI.setDesc(B.getTII().get(Opc));
7149 MI.removeOperand(1); // Remove intrinsic ID
7150
7151 // FIXME: When intrinsic definition is fixed, this should have an MMO already.
7152 const unsigned MemSize = (Size + 7) / 8;
7153 const Align MemAlign = B.getDataLayout().getABITypeAlign(
7159 MemSize, MemAlign);
7160 MI.addMemOperand(MF, MMO);
7161 if (Dst != OrigDst) {
7162 MI.getOperand(0).setReg(Dst);
7163 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
7164 B.buildTrunc(OrigDst, Dst);
7165 }
7166
7167 // If we don't have 96-bit result scalar loads, widening to 128-bit should
7168 // always be legal. We may need to restore this to a 96-bit result if it turns
7169 // out this needs to be converted to a vector load during RegBankSelect.
7170 if (!isPowerOf2_32(Size) && (Size != 96 || !ST.hasScalarDwordx3Loads())) {
7171 if (Ty.isVector())
7173 else
7174 Helper.widenScalarDst(MI, getPow2ScalarType(Ty), 0);
7175 }
7176
7177 Observer.changedInstr(MI);
7178 return true;
7179}
7180
7182 MachineInstr &MI) const {
7183 MachineIRBuilder &B = Helper.MIRBuilder;
7184 GISelChangeObserver &Observer = Helper.Observer;
7185 Observer.changingInstr(MI);
7186 MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH));
7187 MI.removeOperand(0); // Remove intrinsic ID
7189 Observer.changedInstr(MI);
7190 return true;
7191}
7192
7193// TODO: Move to selection
7196 MachineIRBuilder &B) const {
7197 if (!ST.isTrapHandlerEnabled() ||
7198 ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
7199 return legalizeTrapEndpgm(MI, MRI, B);
7200
7201 return ST.supportsGetDoorbellID() ?
7203}
7204
7207 const DebugLoc &DL = MI.getDebugLoc();
7208 MachineBasicBlock &BB = B.getMBB();
7209 MachineFunction *MF = BB.getParent();
7210
7211 if (BB.succ_empty() && std::next(MI.getIterator()) == BB.end()) {
7212 BuildMI(BB, BB.end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
7213 .addImm(0);
7214 MI.eraseFromParent();
7215 return true;
7216 }
7217
7218 // We need a block split to make the real endpgm a terminator. We also don't
7219 // want to break phis in successor blocks, so we can't just delete to the
7220 // end of the block.
7221 BB.splitAt(MI, false /*UpdateLiveIns*/);
7223 MF->push_back(TrapBB);
7224 BuildMI(*TrapBB, TrapBB->end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
7225 .addImm(0);
7226 BuildMI(BB, &MI, DL, B.getTII().get(AMDGPU::S_CBRANCH_EXECNZ))
7227 .addMBB(TrapBB);
7228
7229 BB.addSuccessor(TrapBB);
7230 MI.eraseFromParent();
7231 return true;
7232}
7233
7236 MachineFunction &MF = B.getMF();
7237 const LLT S64 = LLT::scalar(64);
7238
7239 Register SGPR01(AMDGPU::SGPR0_SGPR1);
7240 // For code object version 5, queue_ptr is passed through implicit kernarg.
7246 ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);
7247
7248 Register KernargPtrReg = MRI.createGenericVirtualRegister(
7250
7251 if (!loadInputValue(KernargPtrReg, B,
7253 return false;
7254
7255 // TODO: can we be smarter about machine pointer info?
7258 PtrInfo,
7262
7263 // Pointer address
7264 Register LoadAddr = MRI.createGenericVirtualRegister(
7266 B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
7267 B.buildConstant(LLT::scalar(64), Offset).getReg(0));
7268 // Load address
7269 Register Temp = B.buildLoad(S64, LoadAddr, *MMO).getReg(0);
7270 B.buildCopy(SGPR01, Temp);
7271 B.buildInstr(AMDGPU::S_TRAP)
7272 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
7273 .addReg(SGPR01, RegState::Implicit);
7274 MI.eraseFromParent();
7275 return true;
7276 }
7277
7278 // Pass queue pointer to trap handler as input, and insert trap instruction
7279 // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
7280 Register LiveIn =
7281 MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
7283 return false;
7284
7285 B.buildCopy(SGPR01, LiveIn);
7286 B.buildInstr(AMDGPU::S_TRAP)
7287 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
7288 .addReg(SGPR01, RegState::Implicit);
7289
7290 MI.eraseFromParent();
7291 return true;
7292}
7293
7296 MachineIRBuilder &B) const {
7297 // We need to simulate the 's_trap 2' instruction on targets that run in
7298 // PRIV=1 (where it is treated as a nop).
7299 if (ST.hasPrivEnabledTrap2NopBug()) {
7300 ST.getInstrInfo()->insertSimulatedTrap(MRI, B.getMBB(), MI,
7301 MI.getDebugLoc());
7302 MI.eraseFromParent();
7303 return true;
7304 }
7305
7306 B.buildInstr(AMDGPU::S_TRAP)
7307 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
7308 MI.eraseFromParent();
7309 return true;
7310}
7311
7314 MachineIRBuilder &B) const {
7315 // Is non-HSA path or trap-handler disabled? Then, report a warning
7316 // accordingly
7317 if (!ST.isTrapHandlerEnabled() ||
7318 ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
7319 Function &Fn = B.getMF().getFunction();
7321 Fn, "debugtrap handler not supported", MI.getDebugLoc(), DS_Warning));
7322 } else {
7323 // Insert debug-trap instruction
7324 B.buildInstr(AMDGPU::S_TRAP)
7325 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap));
7326 }
7327
7328 MI.eraseFromParent();
7329 return true;
7330}
7331
7333 MachineInstr &MI, MachineIRBuilder &B) const {
7334 MachineRegisterInfo &MRI = *B.getMRI();
7335 const LLT S16 = LLT::scalar(16);
7336 const LLT S32 = LLT::scalar(32);
7337 const LLT V2S16 = LLT::fixed_vector(2, 16);
7338 const LLT V3S32 = LLT::fixed_vector(3, 32);
7339
7340 Register DstReg = MI.getOperand(0).getReg();
7341 Register NodePtr = MI.getOperand(2).getReg();
7342 Register RayExtent = MI.getOperand(3).getReg();
7343 Register RayOrigin = MI.getOperand(4).getReg();
7344 Register RayDir = MI.getOperand(5).getReg();
7345 Register RayInvDir = MI.getOperand(6).getReg();
7346 Register TDescr = MI.getOperand(7).getReg();
7347
7348 if (!ST.hasGFX10_AEncoding()) {
7349 Function &Fn = B.getMF().getFunction();
7351 Fn, "intrinsic not supported on subtarget", MI.getDebugLoc()));
7352 return false;
7353 }
7354
7355 const bool IsGFX11 = AMDGPU::isGFX11(ST);
7356 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(ST);
7357 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(ST);
7358 const bool IsA16 = MRI.getType(RayDir).getElementType().getSizeInBits() == 16;
7359 const bool Is64 = MRI.getType(NodePtr).getSizeInBits() == 64;
7360 const unsigned NumVDataDwords = 4;
7361 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
7362 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
7363 const bool UseNSA =
7364 IsGFX12Plus || (ST.hasNSAEncoding() && NumVAddrs <= ST.getNSAMaxSize());
7365
7366 const unsigned BaseOpcodes[2][2] = {
7367 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
7368 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
7369 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
7370 int Opcode;
7371 if (UseNSA) {
7372 Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
7373 IsGFX12Plus ? AMDGPU::MIMGEncGfx12
7374 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
7375 : AMDGPU::MIMGEncGfx10NSA,
7376 NumVDataDwords, NumVAddrDwords);
7377 } else {
7378 assert(!IsGFX12Plus);
7379 Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
7380 IsGFX11 ? AMDGPU::MIMGEncGfx11Default
7381 : AMDGPU::MIMGEncGfx10Default,
7382 NumVDataDwords, NumVAddrDwords);
7383 }
7384 assert(Opcode != -1);
7385
7387 if (UseNSA && IsGFX11Plus) {
7388 auto packLanes = [&Ops, &S32, &V3S32, &B](Register Src) {
7389 auto Unmerge = B.buildUnmerge({S32, S32, S32}, Src);
7390 auto Merged = B.buildMergeLikeInstr(
7391 V3S32, {Unmerge.getReg(0), Unmerge.getReg(1), Unmerge.getReg(2)});
7392 Ops.push_back(Merged.getReg(0));
7393 };
7394
7395 Ops.push_back(NodePtr);
7396 Ops.push_back(RayExtent);
7397 packLanes(RayOrigin);
7398
7399 if (IsA16) {
7400 auto UnmergeRayDir = B.buildUnmerge({S16, S16, S16}, RayDir);
7401 auto UnmergeRayInvDir = B.buildUnmerge({S16, S16, S16}, RayInvDir);
7402 auto MergedDir = B.buildMergeLikeInstr(
7403 V3S32,
7404 {B.buildBitcast(
7405 S32, B.buildMergeLikeInstr(V2S16, {UnmergeRayInvDir.getReg(0),
7406 UnmergeRayDir.getReg(0)}))
7407 .getReg(0),
7408 B.buildBitcast(
7409 S32, B.buildMergeLikeInstr(V2S16, {UnmergeRayInvDir.getReg(1),
7410 UnmergeRayDir.getReg(1)}))
7411 .getReg(0),
7412 B.buildBitcast(
7413 S32, B.buildMergeLikeInstr(V2S16, {UnmergeRayInvDir.getReg(2),
7414 UnmergeRayDir.getReg(2)}))
7415 .getReg(0)});
7416 Ops.push_back(MergedDir.getReg(0));
7417 } else {
7418 packLanes(RayDir);
7419 packLanes(RayInvDir);
7420 }
7421 } else {
7422 if (Is64) {
7423 auto Unmerge = B.buildUnmerge({S32, S32}, NodePtr);
7424 Ops.push_back(Unmerge.getReg(0));
7425 Ops.push_back(Unmerge.getReg(1));
7426 } else {
7427 Ops.push_back(NodePtr);
7428 }
7429 Ops.push_back(RayExtent);
7430
7431 auto packLanes = [&Ops, &S32, &B](Register Src) {
7432 auto Unmerge = B.buildUnmerge({S32, S32, S32}, Src);
7433 Ops.push_back(Unmerge.getReg(0));
7434 Ops.push_back(Unmerge.getReg(1));
7435 Ops.push_back(Unmerge.getReg(2));
7436 };
7437
7438 packLanes(RayOrigin);
7439 if (IsA16) {
7440 auto UnmergeRayDir = B.buildUnmerge({S16, S16, S16}, RayDir);
7441 auto UnmergeRayInvDir = B.buildUnmerge({S16, S16, S16}, RayInvDir);
7442 Register R1 = MRI.createGenericVirtualRegister(S32);
7443 Register R2 = MRI.createGenericVirtualRegister(S32);
7444 Register R3 = MRI.createGenericVirtualRegister(S32);
7445 B.buildMergeLikeInstr(R1,
7446 {UnmergeRayDir.getReg(0), UnmergeRayDir.getReg(1)});
7447 B.buildMergeLikeInstr(
7448 R2, {UnmergeRayDir.getReg(2), UnmergeRayInvDir.getReg(0)});
7449 B.buildMergeLikeInstr(
7450 R3, {UnmergeRayInvDir.getReg(1), UnmergeRayInvDir.getReg(2)});
7451 Ops.push_back(R1);
7452 Ops.push_back(R2);
7453 Ops.push_back(R3);
7454 } else {
7455 packLanes(RayDir);
7456 packLanes(RayInvDir);
7457 }
7458 }
7459
7460 if (!UseNSA) {
7461 // Build a single vector containing all the operands so far prepared.
7462 LLT OpTy = LLT::fixed_vector(Ops.size(), 32);
7463 Register MergedOps = B.buildMergeLikeInstr(OpTy, Ops).getReg(0);
7464 Ops.clear();
7465 Ops.push_back(MergedOps);
7466 }
7467
7468 auto MIB = B.buildInstr(AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY)
7469 .addDef(DstReg)
7470 .addImm(Opcode);
7471
7472 for (Register R : Ops) {
7473 MIB.addUse(R);
7474 }
7475
7476 MIB.addUse(TDescr)
7477 .addImm(IsA16 ? 1 : 0)
7478 .cloneMemRefs(MI);
7479
7480 MI.eraseFromParent();
7481 return true;
7482}
7483
7485 MachineInstr &MI, MachineIRBuilder &B) const {
7486 const LLT S32 = LLT::scalar(32);
7487 const LLT V2S32 = LLT::fixed_vector(2, 32);
7488
7489 Register DstReg = MI.getOperand(0).getReg();
7490 Register DstOrigin = MI.getOperand(1).getReg();
7491 Register DstDir = MI.getOperand(2).getReg();
7492 Register NodePtr = MI.getOperand(4).getReg();
7493 Register RayExtent = MI.getOperand(5).getReg();
7494 Register InstanceMask = MI.getOperand(6).getReg();
7495 Register RayOrigin = MI.getOperand(7).getReg();
7496 Register RayDir = MI.getOperand(8).getReg();
7497 Register Offsets = MI.getOperand(9).getReg();
7498 Register TDescr = MI.getOperand(10).getReg();
7499
7500 if (!ST.hasBVHDualAndBVH8Insts()) {
7501 Function &Fn = B.getMF().getFunction();
7503 Fn, "intrinsic not supported on subtarget", MI.getDebugLoc()));
7504 return false;
7505 }
7506
7507 bool IsBVH8 = cast<GIntrinsic>(MI).getIntrinsicID() ==
7508 Intrinsic::amdgcn_image_bvh8_intersect_ray;
7509 const unsigned NumVDataDwords = 10;
7510 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
7511 int Opcode = AMDGPU::getMIMGOpcode(
7512 IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
7513 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
7514 AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords);
7515 assert(Opcode != -1);
7516
7517 auto RayExtentInstanceMaskVec = B.buildMergeLikeInstr(
7518 V2S32, {RayExtent, B.buildAnyExt(S32, InstanceMask)});
7519
7520 B.buildInstr(IsBVH8 ? AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY
7521 : AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY)
7522 .addDef(DstReg)
7523 .addDef(DstOrigin)
7524 .addDef(DstDir)
7525 .addImm(Opcode)
7526 .addUse(NodePtr)
7527 .addUse(RayExtentInstanceMaskVec.getReg(0))
7528 .addUse(RayOrigin)
7529 .addUse(RayDir)
7530 .addUse(Offsets)
7531 .addUse(TDescr)
7532 .cloneMemRefs(MI);
7533
7534 MI.eraseFromParent();
7535 return true;
7536}
7537
7539 MachineIRBuilder &B) const {
7540 const SITargetLowering *TLI = ST.getTargetLowering();
7542 Register DstReg = MI.getOperand(0).getReg();
7543 B.buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {DstReg}, {StackPtr});
7544 MI.eraseFromParent();
7545 return true;
7546}
7547
7549 MachineIRBuilder &B) const {
7550 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
7551 if (!ST.hasArchitectedSGPRs())
7552 return false;
7553 LLT S32 = LLT::scalar(32);
7554 Register DstReg = MI.getOperand(0).getReg();
7555 auto TTMP8 = B.buildCopy(S32, Register(AMDGPU::TTMP8));
7556 auto LSB = B.buildConstant(S32, 25);
7557 auto Width = B.buildConstant(S32, 5);
7558 B.buildUbfx(DstReg, TTMP8, LSB, Width);
7559 MI.eraseFromParent();
7560 return true;
7561}
7562
7565 AMDGPU::Hwreg::Id HwReg,
7566 unsigned LowBit,
7567 unsigned Width) const {
7568 MachineRegisterInfo &MRI = *B.getMRI();
7569 Register DstReg = MI.getOperand(0).getReg();
7570 if (!MRI.getRegClassOrNull(DstReg))
7571 MRI.setRegClass(DstReg, &AMDGPU::SReg_32RegClass);
7572 B.buildInstr(AMDGPU::S_GETREG_B32_const)
7573 .addDef(DstReg)
7574 .addImm(AMDGPU::Hwreg::HwregEncoding::encode(HwReg, LowBit, Width));
7575 MI.eraseFromParent();
7576 return true;
7577}
7578
7579static constexpr unsigned FPEnvModeBitField =
7581
7582static constexpr unsigned FPEnvTrapBitField =
7584
7587 MachineIRBuilder &B) const {
7588 Register Src = MI.getOperand(0).getReg();
7589 if (MRI.getType(Src) != S64)
7590 return false;
7591
7592 auto ModeReg =
7593 B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {S32},
7594 /*HasSideEffects=*/true, /*isConvergent=*/false)
7595 .addImm(FPEnvModeBitField);
7596 auto TrapReg =
7597 B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {S32},
7598 /*HasSideEffects=*/true, /*isConvergent=*/false)
7599 .addImm(FPEnvTrapBitField);
7600 B.buildMergeLikeInstr(Src, {ModeReg, TrapReg});
7601 MI.eraseFromParent();
7602 return true;
7603}
7604
7607 MachineIRBuilder &B) const {
7608 Register Src = MI.getOperand(0).getReg();
7609 if (MRI.getType(Src) != S64)
7610 return false;
7611
7612 auto Unmerge = B.buildUnmerge({S32, S32}, MI.getOperand(0));
7613 B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
7614 /*HasSideEffects=*/true, /*isConvergent=*/false)
7615 .addImm(static_cast<int16_t>(FPEnvModeBitField))
7616 .addReg(Unmerge.getReg(0));
7617 B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
7618 /*HasSideEffects=*/true, /*isConvergent=*/false)
7619 .addImm(static_cast<int16_t>(FPEnvTrapBitField))
7620 .addReg(Unmerge.getReg(1));
7621 MI.eraseFromParent();
7622 return true;
7623}
7624
7626 MachineInstr &MI) const {
7627 MachineIRBuilder &B = Helper.MIRBuilder;
7628 MachineRegisterInfo &MRI = *B.getMRI();
7629
7630 // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
7631 auto IntrID = cast<GIntrinsic>(MI).getIntrinsicID();
7632 switch (IntrID) {
7633 case Intrinsic::amdgcn_if:
7634 case Intrinsic::amdgcn_else: {
7635 MachineInstr *Br = nullptr;
7636 MachineBasicBlock *UncondBrTarget = nullptr;
7637 bool Negated = false;
7638 if (MachineInstr *BrCond =
7639 verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
7640 const SIRegisterInfo *TRI
7641 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
7642
7643 Register Def = MI.getOperand(1).getReg();
7644 Register Use = MI.getOperand(3).getReg();
7645
7646 MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
7647
7648 if (Negated)
7649 std::swap(CondBrTarget, UncondBrTarget);
7650
7651 B.setInsertPt(B.getMBB(), BrCond->getIterator());
7652 if (IntrID == Intrinsic::amdgcn_if) {
7653 B.buildInstr(AMDGPU::SI_IF)
7654 .addDef(Def)
7655 .addUse(Use)
7656 .addMBB(UncondBrTarget);
7657 } else {
7658 B.buildInstr(AMDGPU::SI_ELSE)
7659 .addDef(Def)
7660 .addUse(Use)
7661 .addMBB(UncondBrTarget);
7662 }
7663
7664 if (Br) {
7665 Br->getOperand(0).setMBB(CondBrTarget);
7666 } else {
7667 // The IRTranslator skips inserting the G_BR for fallthrough cases, but
7668 // since we're swapping branch targets it needs to be reinserted.
7669 // FIXME: IRTranslator should probably not do this
7670 B.buildBr(*CondBrTarget);
7671 }
7672
7673 MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
7674 MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
7675 MI.eraseFromParent();
7676 BrCond->eraseFromParent();
7677 return true;
7678 }
7679
7680 return false;
7681 }
7682 case Intrinsic::amdgcn_loop: {
7683 MachineInstr *Br = nullptr;
7684 MachineBasicBlock *UncondBrTarget = nullptr;
7685 bool Negated = false;
7686 if (MachineInstr *BrCond =
7687 verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
7688 const SIRegisterInfo *TRI
7689 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
7690
7691 MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
7692 Register Reg = MI.getOperand(2).getReg();
7693
7694 if (Negated)
7695 std::swap(CondBrTarget, UncondBrTarget);
7696
7697 B.setInsertPt(B.getMBB(), BrCond->getIterator());
7698 B.buildInstr(AMDGPU::SI_LOOP)
7699 .addUse(Reg)
7700 .addMBB(UncondBrTarget);
7701
7702 if (Br)
7703 Br->getOperand(0).setMBB(CondBrTarget);
7704 else
7705 B.buildBr(*CondBrTarget);
7706
7707 MI.eraseFromParent();
7708 BrCond->eraseFromParent();
7709 MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
7710 return true;
7711 }
7712
7713 return false;
7714 }
7715 case Intrinsic::amdgcn_addrspacecast_nonnull:
7716 return legalizeAddrSpaceCast(MI, MRI, B);
7717 case Intrinsic::amdgcn_make_buffer_rsrc:
7719 case Intrinsic::amdgcn_kernarg_segment_ptr:
7720 if (!AMDGPU::isKernel(B.getMF().getFunction().getCallingConv())) {
7721 // This only makes sense to call in a kernel, so just lower to null.
7722 B.buildConstant(MI.getOperand(0).getReg(), 0);
7723 MI.eraseFromParent();
7724 return true;
7725 }
7726
7729 case Intrinsic::amdgcn_implicitarg_ptr:
7730 return legalizeImplicitArgPtr(MI, MRI, B);
7731 case Intrinsic::amdgcn_workitem_id_x:
7732 return legalizeWorkitemIDIntrinsic(MI, MRI, B, 0,
7734 case Intrinsic::amdgcn_workitem_id_y:
7735 return legalizeWorkitemIDIntrinsic(MI, MRI, B, 1,
7737 case Intrinsic::amdgcn_workitem_id_z:
7738 return legalizeWorkitemIDIntrinsic(MI, MRI, B, 2,
7740 case Intrinsic::amdgcn_workgroup_id_x:
7741 return legalizeWorkGroupId(
7745 case Intrinsic::amdgcn_workgroup_id_y:
7746 return legalizeWorkGroupId(
7750 case Intrinsic::amdgcn_workgroup_id_z:
7751 return legalizeWorkGroupId(
7755 case Intrinsic::amdgcn_cluster_id_x:
7756 return ST.hasClusters() &&
7759 case Intrinsic::amdgcn_cluster_id_y:
7760 return ST.hasClusters() &&
7763 case Intrinsic::amdgcn_cluster_id_z:
7764 return ST.hasClusters() &&
7767 case Intrinsic::amdgcn_cluster_workgroup_id_x:
7768 return ST.hasClusters() &&
7771 case Intrinsic::amdgcn_cluster_workgroup_id_y:
7772 return ST.hasClusters() &&
7775 case Intrinsic::amdgcn_cluster_workgroup_id_z:
7776 return ST.hasClusters() &&
7779 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
7780 return ST.hasClusters() &&
7782 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
7783 return ST.hasClusters() &&
7786 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
7787 return ST.hasClusters() &&
7790 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
7791 return ST.hasClusters() &&
7794 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
7795 return ST.hasClusters() &&
7797 MI, MRI, B,
7799 case Intrinsic::amdgcn_wave_id:
7800 return legalizeWaveID(MI, B);
7801 case Intrinsic::amdgcn_lds_kernel_id:
7804 case Intrinsic::amdgcn_dispatch_ptr:
7807 case Intrinsic::amdgcn_queue_ptr:
7810 case Intrinsic::amdgcn_implicit_buffer_ptr:
7813 case Intrinsic::amdgcn_dispatch_id:
7816 case Intrinsic::r600_read_ngroups_x:
7817 // TODO: Emit error for hsa
7820 case Intrinsic::r600_read_ngroups_y:
7823 case Intrinsic::r600_read_ngroups_z:
7826 case Intrinsic::r600_read_local_size_x:
7827 // TODO: Could insert G_ASSERT_ZEXT from s16
7829 case Intrinsic::r600_read_local_size_y:
7830 // TODO: Could insert G_ASSERT_ZEXT from s16
7832 // TODO: Could insert G_ASSERT_ZEXT from s16
7833 case Intrinsic::r600_read_local_size_z:
7836 case Intrinsic::amdgcn_fdiv_fast:
7837 return legalizeFDIVFastIntrin(MI, MRI, B);
7838 case Intrinsic::amdgcn_is_shared:
7840 case Intrinsic::amdgcn_is_private:
7842 case Intrinsic::amdgcn_wavefrontsize: {
7843 B.buildConstant(MI.getOperand(0), ST.getWavefrontSize());
7844 MI.eraseFromParent();
7845 return true;
7846 }
7847 case Intrinsic::amdgcn_s_buffer_load:
7848 return legalizeSBufferLoad(Helper, MI);
7849 case Intrinsic::amdgcn_raw_buffer_store:
7850 case Intrinsic::amdgcn_raw_ptr_buffer_store:
7851 case Intrinsic::amdgcn_struct_buffer_store:
7852 case Intrinsic::amdgcn_struct_ptr_buffer_store:
7853 return legalizeBufferStore(MI, Helper, false, false);
7854 case Intrinsic::amdgcn_raw_buffer_store_format:
7855 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
7856 case Intrinsic::amdgcn_struct_buffer_store_format:
7857 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
7858 return legalizeBufferStore(MI, Helper, false, true);
7859 case Intrinsic::amdgcn_raw_tbuffer_store:
7860 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
7861 case Intrinsic::amdgcn_struct_tbuffer_store:
7862 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
7863 return legalizeBufferStore(MI, Helper, true, true);
7864 case Intrinsic::amdgcn_raw_buffer_load:
7865 case Intrinsic::amdgcn_raw_ptr_buffer_load:
7866 case Intrinsic::amdgcn_raw_atomic_buffer_load:
7867 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
7868 case Intrinsic::amdgcn_struct_buffer_load:
7869 case Intrinsic::amdgcn_struct_ptr_buffer_load:
7870 case Intrinsic::amdgcn_struct_atomic_buffer_load:
7871 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
7872 return legalizeBufferLoad(MI, Helper, false, false);
7873 case Intrinsic::amdgcn_raw_buffer_load_format:
7874 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
7875 case Intrinsic::amdgcn_struct_buffer_load_format:
7876 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
7877 return legalizeBufferLoad(MI, Helper, true, false);
7878 case Intrinsic::amdgcn_raw_tbuffer_load:
7879 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
7880 case Intrinsic::amdgcn_struct_tbuffer_load:
7881 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
7882 return legalizeBufferLoad(MI, Helper, true, true);
7883 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
7884 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
7885 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
7886 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
7887 case Intrinsic::amdgcn_raw_buffer_atomic_add:
7888 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
7889 case Intrinsic::amdgcn_struct_buffer_atomic_add:
7890 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
7891 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
7892 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
7893 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
7894 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
7895 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
7896 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
7897 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
7898 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
7899 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
7900 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
7901 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
7902 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
7903 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
7904 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
7905 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
7906 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
7907 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
7908 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
7909 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
7910 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
7911 case Intrinsic::amdgcn_raw_buffer_atomic_and:
7912 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
7913 case Intrinsic::amdgcn_struct_buffer_atomic_and:
7914 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
7915 case Intrinsic::amdgcn_raw_buffer_atomic_or:
7916 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
7917 case Intrinsic::amdgcn_struct_buffer_atomic_or:
7918 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
7919 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
7920 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
7921 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
7922 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
7923 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
7924 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
7925 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
7926 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
7927 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
7928 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
7929 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
7930 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
7931 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
7932 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
7933 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
7934 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
7935 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
7936 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
7937 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
7938 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
7939 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
7940 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
7941 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
7942 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
7943 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
7944 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
7945 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
7946 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
7947 return legalizeBufferAtomic(MI, B, IntrID);
7948 case Intrinsic::amdgcn_rsq_clamp:
7950 case Intrinsic::amdgcn_image_bvh_intersect_ray:
7952 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
7953 case Intrinsic::amdgcn_image_bvh8_intersect_ray:
7955 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
7956 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
7957 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
7958 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
7959 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
7960 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
7961 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
7962 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
7963 Register Index = MI.getOperand(5).getReg();
7964 LLT S64 = LLT::scalar(64);
7965 if (MRI.getType(Index) != S64)
7966 MI.getOperand(5).setReg(B.buildAnyExt(S64, Index).getReg(0));
7967 return true;
7968 }
7969 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
7970 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
7971 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
7972 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
7973 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
7974 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
7975 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
7976 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
7977 Register Index = MI.getOperand(5).getReg();
7978 LLT S32 = LLT::scalar(32);
7979 if (MRI.getType(Index) != S32)
7980 MI.getOperand(5).setReg(B.buildAnyExt(S32, Index).getReg(0));
7981 return true;
7982 }
7983 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
7984 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
7985 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
7986 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
7987 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
7988 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
7989 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
7990 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
7991 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
7992 Register Index = MI.getOperand(7).getReg();
7993 LLT IdxTy = IntrID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
7994 ? LLT::scalar(64)
7995 : LLT::scalar(32);
7996 if (MRI.getType(Index) != IdxTy)
7997 MI.getOperand(7).setReg(B.buildAnyExt(IdxTy, Index).getReg(0));
7998 return true;
7999 }
8000
8001 case Intrinsic::amdgcn_fmed3: {
8002 GISelChangeObserver &Observer = Helper.Observer;
8003
8004 // FIXME: This is to workaround the inability of tablegen match combiners to
8005 // match intrinsics in patterns.
8006 Observer.changingInstr(MI);
8007 MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_FMED3));
8008 MI.removeOperand(1);
8009 Observer.changedInstr(MI);
8010 return true;
8011 }
8012 case Intrinsic::amdgcn_readlane:
8013 case Intrinsic::amdgcn_writelane:
8014 case Intrinsic::amdgcn_readfirstlane:
8015 case Intrinsic::amdgcn_permlane16:
8016 case Intrinsic::amdgcn_permlanex16:
8017 case Intrinsic::amdgcn_permlane64:
8018 case Intrinsic::amdgcn_set_inactive:
8019 case Intrinsic::amdgcn_set_inactive_chain_arg:
8020 case Intrinsic::amdgcn_mov_dpp8:
8021 case Intrinsic::amdgcn_update_dpp:
8022 return legalizeLaneOp(Helper, MI, IntrID);
8023 case Intrinsic::amdgcn_s_buffer_prefetch_data:
8024 return legalizeSBufferPrefetch(Helper, MI);
8025 case Intrinsic::amdgcn_dead: {
8026 // TODO: Use poison instead of undef
8027 for (const MachineOperand &Def : MI.defs())
8028 B.buildUndef(Def);
8029 MI.eraseFromParent();
8030 return true;
8031 }
8032 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
8033 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
8034 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
8035 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8036 B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
8037 MI.eraseFromParent();
8038 return true;
8039 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
8040 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
8041 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
8042 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8043 B.buildStore(MI.getOperand(2), MI.getOperand(1), **MI.memoperands_begin());
8044 MI.eraseFromParent();
8045 return true;
8046 default: {
8047 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
8049 return legalizeImageIntrinsic(MI, B, Helper.Observer, ImageDimIntr);
8050 return true;
8051 }
8052 }
8053
8054 return true;
8055}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, SelectionDAG &DAG)
static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, SDValue Y, SDValue C, SDNodeFlags Flags=SDNodeFlags())
static bool valueIsKnownNeverF32Denorm(SDValue Src)
Return true if it's known that Src can never be an f32 denormal value.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI, SmallVectorImpl< Register > &PackedAddrs, unsigned ArgOffset, const AMDGPU::ImageDimIntrinsicInfo *Intr, bool IsA16, bool IsG16)
Turn a set of s16 typed registers in AddrRegs into a dword sized vector with s16 typed elements.
static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID)
static LLT getBufferRsrcScalarType(const LLT Ty)
static LegalityPredicate isIllegalRegisterType(const GCNSubtarget &ST, unsigned TypeIdx)
static cl::opt< bool > EnableNewLegality("amdgpu-global-isel-new-legality", cl::desc("Use GlobalISel desired legality, rather than try to use" "rules compatible with selection patterns"), cl::init(false), cl::ReallyHidden)
static bool needsDenormHandlingF32(const MachineFunction &MF, Register Src, unsigned Flags)
constexpr std::initializer_list< LLT > AllVectors
static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx)
static LegalityPredicate isSmallOddVector(unsigned TypeIdx)
static LegalizeMutation oneMoreElement(unsigned TypeIdx)
constexpr LLT F64
static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size)
constexpr LLT V2S8
static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags)
constexpr LLT V4S128
constexpr LLT S16
constexpr LLT S1
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty, const LLT MemTy)
Return true if a load or store of the type should be lowered with a bitcast to a different type.
constexpr LLT S1024
static constexpr unsigned FPEnvModeBitField
constexpr LLT V7S64
static LegalizeMutation getScalarTypeFromMemDesc(unsigned TypeIdx)
static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size)
static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy, uint64_t AlignInBits, unsigned AddrSpace, unsigned Opcode)
Return true if we should legalize a load by widening an odd sized memory access up to the alignment.
static bool isRegisterVectorElementType(LLT EltTy)
static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx)
static LegalityPredicate isWideVec16(unsigned TypeIdx)
constexpr std::initializer_list< LLT > AllScalarTypes
static LegalityPredicate isTruncStoreToSizePowerOf2(unsigned TypeIdx)
constexpr LLT V2S16
constexpr LLT V8S16
constexpr LLT V9S32
constexpr std::initializer_list< LLT > AllS32Vectors
constexpr LLT S224
static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx)
constexpr LLT S512
constexpr LLT MaxScalar
static Register castBufferRsrcToV4I32(Register Pointer, MachineIRBuilder &B)
Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is the form in which the valu...
constexpr LLT V11S32
static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty)
constexpr LLT V6S64
constexpr LLT V2S64
static std::pair< Register, Register > emitReciprocalU64(MachineIRBuilder &B, Register Val)
static LLT getBitcastRegisterType(const LLT Ty)
static LLT getBufferRsrcRegisterType(const LLT Ty)
constexpr LLT S32
constexpr LLT V2F16
static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx)
static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI)
constexpr LLT V8S32
constexpr LLT V2BF16
constexpr LLT S192
static LLT castBufferRsrcFromV4I32(MachineInstr &MI, MachineIRBuilder &B, MachineRegisterInfo &MRI, unsigned Idx)
Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial type of the operand idx an...
static bool replaceWithConstant(MachineIRBuilder &B, MachineInstr &MI, int64_t C)
static constexpr unsigned SPDenormModeBitField
constexpr LLT F32
static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS, bool IsLoad, bool IsAtomic)
constexpr LLT V6S32
static bool isLoadStoreSizeLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
constexpr LLT S160
static MachineInstr * verifyCFIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br, MachineBasicBlock *&UncondBrTarget, bool &Negated)
constexpr LLT V4S16
constexpr LLT V2S128
constexpr LLT V10S16
static LegalityPredicate numElementsNotEven(unsigned TypeIdx)
constexpr LLT V4S32
constexpr LLT V3S32
constexpr LLT V6S16
constexpr std::initializer_list< LLT > AllS64Vectors
constexpr LLT S256
static void castBufferRsrcArgToV4I32(MachineInstr &MI, MachineIRBuilder &B, unsigned Idx)
constexpr LLT V4S64
static constexpr unsigned FPEnvTrapBitField
constexpr LLT V10S32
constexpr LLT V16S32
static constexpr unsigned MaxRegisterSize
constexpr LLT V7S32
constexpr LLT S96
constexpr LLT V12S16
constexpr LLT V16S64
static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size)
static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx)
static bool hasBufferRsrcWorkaround(const LLT Ty)
constexpr LLT V32S32
static void toggleSPDenormMode(bool Enable, MachineIRBuilder &B, const GCNSubtarget &ST, SIModeRegisterDefaults Mode)
constexpr LLT S64
constexpr std::initializer_list< LLT > AllS16Vectors
static bool loadStoreBitcastWorkaround(const LLT Ty)
static LLT widenToNextPowerOf2(LLT Ty)
static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI)
constexpr LLT V16S16
static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI, int DimIdx, int NumVAddrs)
Convert from separate vaddr components to a single vector address register, and replace the remaining...
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx)
constexpr LLT V5S32
constexpr LLT V5S64
constexpr LLT V3S64
static LLT getPow2VectorType(LLT Ty)
static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc, Register VIndex, Register VOffset, Register SOffset, unsigned ImmOffset, unsigned Format, unsigned AuxiliaryData, MachineMemOperand *MMO, bool IsTyped, bool HasVIndex, MachineIRBuilder &B)
constexpr LLT V8S64
static LLT getPow2ScalarType(LLT Ty)
static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx)
constexpr LLT V2S32
static bool isRegisterVectorType(LLT Ty)
constexpr LLT V12S32
constexpr LLT S128
static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx)
constexpr LLT S8
static bool isRegisterType(const GCNSubtarget &ST, LLT Ty)
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
Return true if the value is a known valid address, such that a null check is not necessary.
This file declares the targeting of the Machinelegalizer class for AMDGPU.
Provides AMDGPU specific target descriptions.
The AMDGPU TargetMachine interface definition for hw codegen targets.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Error unsupported(const char *Str, const Triple &T)
Definition MachO.cpp:71
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
@ Enable
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
IRTranslator LLVM IR MI
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Interface for Targets to specify which operations they can successfully select and how the others sho...
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Register const TargetRegisterInfo * TRI
#define R2(n)
Promote Memory to Register
Definition Mem2Reg.cpp:110
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
ppc ctr loops verify
R600 Clause Merge
const SmallVectorImpl< MachineOperand > & Cond
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
#define CH(x, y, z)
Definition SHA256.cpp:34
#define FP_DENORM_FLUSH_NONE
Definition SIDefines.h:1254
Interface definition for SIInstrInfo.
Interface definition for SIRegisterInfo.
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static constexpr int Concat[]
bool legalizeConstHwRegRead(MachineInstr &MI, MachineIRBuilder &B, AMDGPU::Hwreg::Id HwReg, unsigned LowBit, unsigned Width) const
void buildMultiply(LegalizerHelper &Helper, MutableArrayRef< Register > Accum, ArrayRef< Register > Src0, ArrayRef< Register > Src1, bool UsePartialMad64_32, bool SeparateOddAlignedProducts) const
bool legalizeGlobalValue(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF16(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeIntrinsicTrunc(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
std::pair< Register, unsigned > splitBufferOffsets(MachineIRBuilder &B, Register OrigOffset) const
bool legalizeBVHIntersectRayIntrinsic(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeIsAddrSpace(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, unsigned AddrSpace) const
bool legalizeUnsignedDIV_REM(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF32(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeAtomicCmpXChg(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBufferStore(MachineInstr &MI, LegalizerHelper &Helper, bool IsTyped, bool IsFormat) const
bool legalizeMul(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFFREXP(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
Register getSegmentAperture(unsigned AddrSpace, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizePointerAsRsrcIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
To create a buffer resource from a 64-bit pointer, mask off the upper 32 bits of the pointer and repl...
bool legalizeFlogCommon(MachineInstr &MI, MachineIRBuilder &B) const
bool getLDSKernelId(Register DstReg, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFExp2(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeTrap(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B, Intrinsic::ID IID) const
void legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B, Register DstDivReg, Register DstRemReg, Register Num, Register Den) const
Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI, Register Reg, bool ImageStore=false) const
Handle register layout difference for f16 images for some subtargets.
bool legalizeCTLZ_CTTZ(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBuildVector(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFFloor(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
AMDGPULegalizerInfo(const GCNSubtarget &ST, const GCNTargetMachine &TM)
bool legalizeFDIV32(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFMad(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeSBufferPrefetch(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFExp(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
bool legalizeFrem(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizePreloadedArgIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool buildPCRelGlobalAddress(Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV, int64_t Offset, unsigned GAFlags=SIInstrInfo::MO_NONE) const
bool legalizeFDIV16(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeRsqClampIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
std::pair< Register, Register > getScaledLogInput(MachineIRBuilder &B, Register Src, unsigned Flags) const
bool legalizeFDIVFastIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool loadInputValue(Register DstReg, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
bool legalizeBVHDualOrBVH8IntersectRayIntrinsic(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeInsertVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFExpUnsafe(MachineIRBuilder &B, Register Dst, Register Src, unsigned Flags) const
bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBufferLoad(MachineInstr &MI, LegalizerHelper &Helper, bool IsFormat, bool IsTyped) const
bool legalizeImplicitArgPtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeMinNumMaxNum(LegalizerHelper &Helper, MachineInstr &MI) const
void legalizeUnsignedDIV_REM64Impl(MachineIRBuilder &B, Register DstDivReg, Register DstRemReg, Register Num, Register Den) const
bool legalizeDebugTrap(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFastUnsafeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeSinCos(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWaveID(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeLDSKernelId(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWorkGroupId(MachineInstr &MI, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ClusterIdPV, AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV, AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const
bool legalizeSignedDIV_REM(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeITOFP(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, bool Signed) const
bool legalizeFPow(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFastUnsafeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFPTOI(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, bool Signed) const
bool legalizeStackSave(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFlogUnsafe(MachineIRBuilder &B, Register Dst, Register Src, bool IsLog10, unsigned Flags) const
bool legalizeKernargMemParameter(MachineInstr &MI, MachineIRBuilder &B, uint64_t Offset, Align Alignment=Align(4)) const
Legalize a value that's loaded from kernel arguments.
bool legalizeImageIntrinsic(MachineInstr &MI, MachineIRBuilder &B, GISelChangeObserver &Observer, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const
Rewrite image intrinsics to use register layouts expected by the subtarget.
void buildAbsGlobalAddress(Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV, MachineRegisterInfo &MRI) const
bool legalizeGetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool getImplicitArgPtr(Register DstReg, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRT(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
Register getKernargParameterPtr(MachineIRBuilder &B, int64_t Offset) const
bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFceil(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeExtractVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
Register fixStoreSourceType(MachineIRBuilder &B, Register VData, LLT MemTy, bool IsFormat) const
bool legalizeLaneOp(LegalizerHelper &Helper, MachineInstr &MI, Intrinsic::ID IID) const
bool legalizeSetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWorkitemIDIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
void buildLoadInputValue(Register DstReg, MachineIRBuilder &B, const ArgDescriptor *Arg, const TargetRegisterClass *ArgRC, LLT ArgTy) const
bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFlog2(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
static std::optional< uint32_t > getLDSKernelIdMetadata(const Function &F)
void setDynLDSAlign(const Function &F, const GlobalVariable &GV)
unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV)
const std::array< unsigned, 3 > & getDims() const
static const fltSemantics & IEEEsingle()
Definition APFloat.h:296
static const fltSemantics & IEEEdouble()
Definition APFloat.h:297
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition APFloat.h:1140
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition APFloat.h:1120
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1080
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
@ ICMP_SLT
signed less than
Definition InstrTypes.h:705
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:682
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:680
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:700
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:703
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:684
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:701
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:683
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:685
@ ICMP_NE
not equal
Definition InstrTypes.h:698
This is the shared class of boolean and integer constants.
Definition Constants.h:87
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:169
A debug info location.
Definition DebugLoc.h:124
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:310
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:359
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
Simple wrapper observer that takes several observers, and calls each one for each event.
KnownBits getKnownBits(Register R)
bool hasExternalLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
Type * getValueType() const
static constexpr LLT float64()
Get a 64-bit IEEE double value.
constexpr unsigned getScalarSizeInBits() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
static constexpr LLT float16()
Get a 16-bit IEEE half value.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr LLT getScalarType() const
static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy)
static constexpr LLT float32()
Get a 32-bit IEEE float value.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
LLVM_ABI void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & unsupported()
The instruction is unsupported.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & bitcastIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
The specified type index is coerced if predicate is true.
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & unsupportedFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & moreElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Add more elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & lowerFor(std::initializer_list< LLT > Types)
The instruction is lowered when type index 0 is any type in the given list.
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, unsigned NumElts)
Express EltTy vectors strictly using vectors with NumElts elements (or scalars when NumElts equals 1)...
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & alwaysLegal()
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & widenScalarToNextMultipleOf(unsigned TypeIdx, unsigned Size)
Widen the scalar to the next multiple of Size.
LLVM_ABI LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI)
LLVM_ABI void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Def by performing it with addition...
GISelValueTracking * getValueTracking() const
@ Legalized
Instruction has been legalized and the MachineFunction changed.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
LLVM_ABI void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a def by inserting a G_BITCAST from ...
LLVM_ABI LegalizeResult lowerFMad(MachineInstr &MI)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LLVM_ABI void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx=0, unsigned TruncOpcode=TargetOpcode::G_TRUNC)
Legalize a single operand OpIdx of the machine instruction MI as a Def by extending the operand's typ...
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
TypeSize getValue() const
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:33
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition MCRegister.h:64
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Helper class to build MachineInstr.
MachineFunction & getMF()
Getter for the function we currently build.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
MachineBasicBlock * getMBB() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
void setMBB(MachineBasicBlock *MBB)
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition ArrayRef.h:303
MutableArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition ArrayRef.h:392
Wrapper class representing virtual and physical registers.
Definition Register.h:19
constexpr bool isValid() const
Definition Register.h:107
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:74
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
AMDGPU::ClusterDimsAttr getClusterDims() const
SIModeRegisterDefaults getMode() const
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
bool allowsMisalignedMemoryAccessesImpl(unsigned Size, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const
bool shouldEmitFixup(const GlobalValue *GV) const
bool shouldUseLDSConstAddress(const GlobalValue *GV) const
bool shouldEmitPCReloc(const GlobalValue *GV) const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void truncate(size_type N)
Like resize, but requires that N is less than size().
void resize(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
int64_t getImm() const
Register getReg() const
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BUFFER_STRIDED_POINTER
Address space for 192-bit fat buffer pointers with an additional index.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords)
bool isFlatGlobalAddrSpace(unsigned AS)
bool isGFX12Plus(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
unsigned getAMDHSACodeObjectVersion(const Module &M)
LLVM_READNONE constexpr bool isKernel(CallingConv::ID CC)
LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)
LLVM_READNONE constexpr bool isCompute(CallingConv::ID CC)
TargetExtType * isNamedBarrier(const GlobalVariable &GV)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isGFX1250(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
std::pair< Register, unsigned > getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg, GISelValueTracking *ValueTracking=nullptr, bool CheckNUW=false)
Returns base register and constant offset.
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ MaxID
The highest possible ID. Must be some 2^k - 1.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or a vector with an element type that's wider than the ...
LLVM_ABI LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LLVM_ABI LegalityPredicate isPointer(unsigned TypeIdx)
True iff the specified type index is a pointer (with any address space).
LLVM_ABI LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LLVM_ABI LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LLVM_ABI LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a larger total bit size than second type index.
LLVM_ABI LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy)
True if the type index is a vector with element type EltTy.
LLVM_ABI LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the specified type indices are both the same bit size.
LLVM_ABI LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or vector with an element type that's narrower than the...
LLVM_ABI LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size)
True if the total bitwidth of the specified type index is Size bits.
LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type)
True iff the given type index is not the specified type.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LLVM_ABI LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
LLVM_ABI LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar that's narrower than the given size.
LLVM_ABI LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LLVM_ABI LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LLVM_ABI LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
Invariant opcodes: All instruction sets have these as their low opcodes.
initializer< Ty > init(const Ty &Val)
constexpr double inv_pi
constexpr double ln2
constexpr double ln10
constexpr float log2ef
Definition MathExtras.h:51
constexpr double log2e
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
Definition Utils.cpp:915
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:346
@ Offset
Definition DWP.cpp:477
LLVM_ABI Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Get the type back from LLT.
Definition Utils.cpp:2032
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition Utils.cpp:651
LLVM_ABI const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
Definition Utils.cpp:459
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:165
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition ScopeExit.h:59
LLVM_ABI const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
std::function< std::pair< unsigned, LLT >(const LegalityQuery &)> LegalizeMutation
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:289
void * PointerTy
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:284
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition MathExtras.h:387
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition Utils.cpp:314
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:186
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:147
std::function< bool(const LegalityQuery &)> LegalityPredicate
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:279
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:396
To bit_cast(const From &from) noexcept
Definition bit.h:90
@ Mul
Product of integers.
@ FMul
Product of floats.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:144
FunctionAddr VTableAddr Next
Definition InstrProf.h:141
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI void eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI, LostDebugLocObserver *LocObserver=nullptr)
Definition Utils.cpp:1719
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:433
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1897
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:201
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:197
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:316
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:154
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:375
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:869
#define N
static constexpr uint64_t encode(Fields... Values)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:77
MCRegister getRegister() const
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ Dynamic
Denormals have unknown treatment.
static constexpr DenormalMode getPreserveSign()
static constexpr DenormalMode getIEEE()
bool isZero() const
Returns true if value is all zero.
Definition KnownBits.h:80
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
ArrayRef< LLT > Types
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
DenormalMode FP64FP16Denormals
If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...
bool IEEE
Floating point opcodes that support exception flag gathering quiet and propagate signaling NaN inputs...
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.